UT / clsync

Go to a project
Toggle navigation Toggle navigation pinning
redmine
157791c1 1 parent 1f343cb5

Options added 

* Completed untested version of --modification-signature
* Added option --cancel-syscalls
Inline Side-by-side
Showing 15 changed files with 197 additions and 54 deletions
... ... @@ -33,7 +33,6 @@
#ifdef CLUSTER_SUPPORT
#include "common.h"
#include "port-hacks.h"
#include "indexes.h"
#include "error.h"
#include "cluster.h"
... ...
... ... @@ -75,6 +75,7 @@
#endif
#include "clsync.h"
#include "port-hacks.h"
#include "ctx.h"
#include "program.h"
... ...
... ... @@ -117,6 +117,7 @@ enum flags_enum {
PERMIT_MPROTECT = 41|OPTION_LONGOPTONLY,
SHM_MPROTECT = 42|OPTION_LONGOPTONLY,
MODSIGN = 43|OPTION_LONGOPTONLY,
CANCEL_SYSCALLS = 44|OPTION_LONGOPTONLY,
};
typedef enum flags_enum flags_t;
... ... @@ -254,6 +255,30 @@ enum state_enum {
};
typedef enum state_enum state_t;
enum stat_fields {
STAT_FIELD_RESET = 0x0000,
STAT_FIELD_DEV = 0x0001,
STAT_FIELD_INO = 0x0002,
STAT_FIELD_MODE = 0x0004,
STAT_FIELD_NLINK = 0x0008,
STAT_FIELD_UID = 0x0010,
STAT_FIELD_GID = 0x0020,
STAT_FIELD_RDEV = 0x0040,
STAT_FIELD_SIZE = 0x0080,
STAT_FIELD_BLKSIZE = 0x0100,
STAT_FIELD_BLOCKS = 0x0200,
STAT_FIELD_ATIME = 0x0400,
STAT_FIELD_MTIME = 0x0800,
STAT_FIELD_CTIME = 0x1000,
STAT_FIELD_ALL = 0x1ff7,
};
enum syscall_bitmask {
CSC_RESET = 0x00,
CSC_MON_STAT = 0x01,
};
#define CAP_PRESERVE_TRY (1<<16)
struct ctx {
... ... @@ -357,5 +382,11 @@ struct ctx {
};
typedef struct ctx ctx_t;
struct fileinfo {
stat64_t lstat;
};
typedef struct fileinfo fileinfo_t;
#endif
... ...
... ... @@ -22,6 +22,8 @@
* to be slow but convenient functions.
*/
#include "common.h"
#include <stdlib.h>
#include <execinfo.h>
#include <stdio.h>
... ... @@ -31,7 +33,6 @@
#include <syslog.h>
#include <pthread.h> /* pthread_self() */
#include "error.h"
#include "common.h"
#include "pthreadex.h" /* pthread_*_shared() */
static int zero = 0;
... ...
... ... @@ -19,8 +19,6 @@
#include "common.h"
#include "port-hacks.h"
#include "error.h"
#include "malloc.h"
... ... @@ -207,3 +205,36 @@ int open_dir(int *fd_p, const char *const dir_path) {
}
*/
uint32_t stat_diff(stat64_t *a, stat64_t *b) {
uint32_t difference;
#ifdef PARANOID
critical_on (a == NULL);
critical_on (b == NULL);
#endif
difference = 0x0000;
#define STAT_COMPARE(bit, field) \
if (a->field != b->field) \
difference |= bit;
STAT_COMPARE(STAT_FIELD_DEV, st_dev);
STAT_COMPARE(STAT_FIELD_INO, st_ino);
STAT_COMPARE(STAT_FIELD_MODE, st_mode);
STAT_COMPARE(STAT_FIELD_NLINK, st_nlink);
STAT_COMPARE(STAT_FIELD_UID, st_uid);
STAT_COMPARE(STAT_FIELD_GID, st_gid);
STAT_COMPARE(STAT_FIELD_RDEV, st_rdev);
STAT_COMPARE(STAT_FIELD_SIZE, st_size);
STAT_COMPARE(STAT_FIELD_BLKSIZE,st_blksize);
STAT_COMPARE(STAT_FIELD_BLOCKS, st_blocks);
STAT_COMPARE(STAT_FIELD_ATIME, st_atime);
STAT_COMPARE(STAT_FIELD_MTIME, st_mtime);
STAT_COMPARE(STAT_FIELD_CTIME, st_ctime);
#undef STAT_COMPARE
return difference;
}
... ...
... ... @@ -22,4 +22,5 @@ extern char *fd2fpath_malloc(int fd);
extern int fileutils_copy(const char *path_from, const char *path_to);
extern short int fileutils_calcdirlevel(const char *path);
extern int mkdirat_open(const char *const dir_path, int dirfd_parent, mode_t dir_mode);
extern uint32_t stat_diff(stat64_t *a, stat64_t *b);
... ...
... ... @@ -35,6 +35,7 @@ struct indexes {
GHashTable *fpath2ei_coll_ht[QUEUE_MAX]; // "file path -> event information" aggregation hashtable for every queue
GHashTable *out_lines_aggr_ht; // output lines aggregation hashtable
GHashTable *nonthreaded_syncing_fpath2ei_ht; // events that are synchronized in signle-mode (non threaded)
GHashTable *fileinfo_ht; // to search "fileinfo" structures (that contains secondary sorts of things about any files/dirs)
};
typedef struct indexes indexes_t;
... ... @@ -164,5 +165,21 @@ static inline int indexes_outaggr_add(indexes_t *indexes_p, char *outline, event
return 0;
}
static inline fileinfo_t *indexes_fileinfo(indexes_t *indexes_p, const char *fpath) {
return (fileinfo_t *)g_hash_table_lookup(indexes_p->fileinfo_ht, fpath);
}
static inline int indexes_fileinfo_add(indexes_t *indexes_p, const char *fpath_const, fileinfo_t *fi) {
size_t fpathlen = strlen(fpath_const);
debug(4, "indexes_add_wd(indexes_p, \"%s\", %p)", fpath_const, fpathlen);
char *fpath = xmalloc(fpathlen+1);
memcpy(fpath, fpath_const, fpathlen+1);
g_hash_table_insert(indexes_p->fileinfo_ht, fpath, fi);
return 0;
}
#endif
... ...
... ... @@ -25,8 +25,6 @@
# include <sys/prctl.h> // for prctl() for --preserve-fil-access
#endif
#include "port-hacks.h"
#include <pwd.h> // getpwnam()
#include <grp.h> // getgrnam()
... ... @@ -136,6 +134,7 @@ static const struct option long_options[] =
{"delay-collect", required_argument, NULL, DELAY},
{"delay-collect-bigfile",required_argument, NULL, BFILEDELAY},
{"threshold-bigfile", required_argument, NULL, BFILETHRESHOLD},
{"cancel-syscalls", required_argument, NULL, CANCEL_SYSCALLS},
{"lists-dir", required_argument, NULL, OUTLISTSDIR},
{"have-recursive-sync", optional_argument, NULL, HAVERECURSIVESYNC},
{"synclist-simplify", optional_argument, NULL, SYNCLISTSIMPLIFY},
... ... @@ -184,25 +183,6 @@ static char *const pivotrootways[] = {
#ifdef CAPABILITIES_SUPPORT
enum stat_fields {
STAT_FIELD_RESET = 0x0000,
STAT_FIELD_DEV = 0x0001,
STAT_FIELD_INO = 0x0002,
STAT_FIELD_MODE = 0x0004,
STAT_FIELD_NLINK = 0x0008,
STAT_FIELD_UID = 0x0010,
STAT_FIELD_GID = 0x0020,
STAT_FIELD_RDEV = 0x0040,
STAT_FIELD_SIZE = 0x0080,
STAT_FIELD_BLKSIZE = 0x0100,
STAT_FIELD_BLOCKS = 0x0200,
STAT_FIELD_ATIME = 0x0400,
STAT_FIELD_MTIME = 0x0800,
STAT_FIELD_CTIME = 0x1000,
STAT_FIELD_ALL = 0x1ff7,
};
static char *const stat_fields[] = {
[STAT_FIELD_RESET] = "",
[STAT_FIELD_DEV] = "dev",
... ... @@ -221,6 +201,12 @@ static char *const stat_fields[] = {
NULL
};
static char *const syscalls_bitmask[] = {
[CSC_RESET] = "",
[CSC_MON_STAT] = "mon_stat", // disable {l,}stat{,64}()-s in mon_*.c
NULL
};
enum x_capabilities {
X_CAP_RESET = 0,
X_CAP_DAC_READ_SEARCH,
... ... @@ -1277,6 +1263,23 @@ int parse_parameter(ctx_t *ctx_p, uint16_t param_id, char *arg, paramsource_t pa
case BFILETHRESHOLD:
ctx_p->bfilethreshold = (unsigned long)atol(arg);
break;
case CANCEL_SYSCALLS: {
char *subopts = arg;
while (*subopts != 0) {
char *value;
typeof(ctx_p->flags[CANCEL_SYSCALLS]) syscall_bitmask = getsubopt(&subopts, syscalls_bitmask, &value);
debug(4, "cancel syscall == 0x%x", syscall_bitmask);
if (syscall_bitmask == CSC_RESET) {
ctx_p->flags[CANCEL_SYSCALLS] = 0;
continue;
}
ctx_p->flags[CANCEL_SYSCALLS] |= syscall_bitmask;
}
break;
}
case MONITOR: {
char *value, *arg_orig = arg;
if (paramsource == PS_CONTROL) {
... ...
... ... @@ -17,6 +17,8 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "common.h"
#include <stdlib.h>
#include <string.h>
#ifdef CAPABILITIES_SUPPORT
... ... @@ -31,7 +33,6 @@
#include <sys/ipc.h> // shmget()
#include <sys/shm.h> // shmget()
#include "common.h"
#include "malloc.h"
#include "error.h"
#include "configuration.h"
... ...
... ... @@ -651,16 +651,13 @@ If you're going to setup bi\-directional syncing then you may use \-\-modificati
.RE
.B Not enough CPU
.RS
If rsync eats to many CPU with rechecking hashsums of files on their dry open()/close() due to some hacky script (for example "chown -R www-data:www-data" in cron) then you can use \-\-modification\-signature "dev,ino,mode,uid,gid,rdev,size,atime,mtime" (without "blksize", "blocks", "nlink" and "ctime")
If rsync eats too many CPU with rechecking hashsums of files on their dry open()/close() due to some hacky script (for example "chown -R www-data:www-data" in cron) then you can use \-\-modification\-signature "dev,ino,mode,uid,gid,rdev,size,atime,mtime" (without "blksize", "blocks", "nlink" and "ctime")
.RE
.RE
.B Warning!
This option requires a
.BR lstat (2)
syscall on every file and directory. This may essentially reduce the
.B clsync
performance.
.B Warning! This option may eat a lot of memory on huge file trees.
This option cannot be used together with "\-\-cancel\-syscalls=mon_stat"
To disable file/dir modification rechecking use empty value — "".
... ... @@ -723,6 +720,31 @@ IO resources.
The default value is "134217728" ["128 MiB"].
.RE
.B \-\-cancel\-syscalls
.I syscalls\-mask
.RS
Sets syscalls to be bypassed. This may be used for to squeeze more
performance.
Possible values:
.RS
.B mon_stat
.RS
Skip lstat() calls while handling files/dirs events. This makes unpossible to
determine files sizes (that is used by
.B \-\-threshold\-bigfile
option) and to use option
.BR \-\-modification\-signature .
.RE
.RE
You can combine this values using commas.
To disable this option just use empty value — "".
The default value is "".
.RE
.PP
.B \-L, \-\-lists\-dir
.I tmpdir\-path
... ...
... ... @@ -427,7 +427,7 @@ int bsm_handle(struct ctx *ctx_p, struct indexes *indexes_p) {
do {
struct recognize_event_return r = {{0}};
char *path_stat;
struct stat st;
struct stat st, *st_p;
mode_t st_mode;
size_t st_size;
... ... @@ -445,20 +445,24 @@ int bsm_handle(struct ctx *ctx_p, struct indexes *indexes_p) {
else
path_stat = event_p->path;
if (lstat(path_stat, &st)) {
if ((ctx_p->flags[CANCEL_SYSCALLS]&CSC_MON_STAT) || lstat(path_stat, &st)) {
debug(2, "Cannot lstat64(\"%s\", st). Seems, that the object disappeared.", path_stat);
if(r.f.objtype_old == EOT_DIR || r.f.objtype_new == EOT_DIR)
st_mode = S_IFDIR;
else
st_mode = S_IFREG;
st_size = 0;
st_p = NULL;
} else {
st_mode = st.st_mode;
st_size = st.st_size;
st_p = &st;
}
if (*event_p->path) {
if (sync_prequeue_loadmark(1, ctx_p, indexes_p, event_p->path, NULL, r.f.objtype_old, r.f.objtype_new, event_p->type, event_p->w_id, st_mode, st_size, &path_rel, &path_rel_len, NULL)) {
if (sync_prequeue_loadmark(1, ctx_p, indexes_p, event_p->path, NULL, st_p, r.f.objtype_old, r.f.objtype_new, event_p->type, event_p->w_id, st_mode, st_size, &path_rel, &path_rel_len, NULL)) {
error("Got error while load_mark-ing into pre-queue \"%s\"", event_p->path);
count = -1;
*event_p->path = 0;
... ...
... ... @@ -18,7 +18,6 @@
*/
#include "common.h"
#include "port-hacks.h"
#include "error.h"
#include "sync.h"
#include "indexes.h"
... ... @@ -140,25 +139,27 @@ int inotify_handle(ctx_t *ctx_p, indexes_t *indexes_p) {
// Getting infomation about file/dir/etc
stat64_t lstat;
stat64_t lstat, *lstat_p;
mode_t st_mode;
size_t st_size;
if (lstat64(path_full, &lstat)) {
if ((ctx_p->flags[CANCEL_SYSCALLS]&CSC_MON_STAT) || lstat64(path_full, &lstat)) {
debug(2, "Cannot lstat64(\"%s\", lstat). Seems, that the object disappeared.", path_full);
if(event->mask & IN_ISDIR)
st_mode = S_IFDIR;
else
st_mode = S_IFREG;
st_size = 0;
lstat_p = NULL;
} else {
st_mode = lstat.st_mode;
st_size = lstat.st_size;
lstat_p = &lstat;
}
struct recognize_event_return r = {0};
recognize_event(&r, event->mask);
if (sync_prequeue_loadmark(1, ctx_p, indexes_p, path_full, NULL, r.objtype_old, r.objtype_new, event->mask, event->wd, st_mode, st_size, &path_rel, &path_rel_len, NULL)) {
if (sync_prequeue_loadmark(1, ctx_p, indexes_p, path_full, NULL, lstat_p, r.objtype_old, r.objtype_new, event->mask, event->wd, st_mode, st_size, &path_rel, &path_rel_len, NULL)) {
count = -1;
goto l_inotify_handle_end;
}
... ...
... ... @@ -18,7 +18,6 @@
*/
#include "common.h"
#include "port-hacks.h"
#include <search.h>
... ... @@ -449,7 +448,7 @@ char *kqueue_getpath(ctx_t *ctx_p, indexes_t *indexes_p, monobj_t *obj_p) {
}
int kqueue_sync(ctx_t *ctx_p, indexes_t *indexes_p, struct kevent *ev_p, monobj_t *obj_p) {
stat64_t lstat;
stat64_t lstat, *lstat_p;
char *path_full = kqueue_getpath(ctx_p, indexes_p, obj_p);
#ifdef PARANOID
... ... @@ -462,16 +461,18 @@ int kqueue_sync(ctx_t *ctx_p, indexes_t *indexes_p, struct kevent *ev_p, monobj_
mode_t st_mode;
size_t st_size;
if (lstat64(path_full, &lstat)) {
debug(2, "Cannot lstat64(\"%s\", lstat). Seems, that the object disappeared.", path_full);
if ((ctx_p->flags[CANCEL_SYSCALLS]&CSC_MON_STAT) || lstat64(path_full, &lstat)) {
debug(2, "Cannot or cancelled lstat64(\"%s\", lstat). The object disappeared or option \"--cancel-syscalls=mon_stat\" is set.", path_full);
if(obj_p->type == DT_DIR)
st_mode = S_IFDIR;
else
st_mode = S_IFREG;
st_size = 0;
lstat_p = NULL;
} else {
st_mode = lstat.st_mode;
st_size = lstat.st_size;
st_mode = lstat.st_mode;
st_size = lstat.st_size;
lstat_p = &lstat;
}
{
... ... @@ -480,7 +481,7 @@ int kqueue_sync(ctx_t *ctx_p, indexes_t *indexes_p, struct kevent *ev_p, monobj_
struct recognize_event_return r;
r.u.i = recognize_event(ev_p->fflags, obj_p->type == DT_DIR);
int ret = sync_prequeue_loadmark(1, ctx_p, indexes_p, path_full, NULL, r.u.v.objtype_old, r.u.v.objtype_new, ev_p->fflags, ev_p->ident, st_mode, st_size, &path_rel, &path_rel_len, NULL);
int ret = sync_prequeue_loadmark(1, ctx_p, indexes_p, path_full, NULL, lstat_p, r.u.v.objtype_old, r.u.v.objtype_new, ev_p->fflags, ev_p->ident, st_mode, st_size, &path_rel, &path_rel_len, NULL);
if (path_rel != NULL)
free(path_rel);
... ...
... ... @@ -1896,6 +1896,8 @@ int sync_prequeue_loadmark
const char *path_full,
const char *path_rel,
stat64_t *lstat_p,
eventobjtype_t objtype_old,
eventobjtype_t objtype_new,
... ... @@ -1951,11 +1953,13 @@ int sync_prequeue_loadmark
if (is_dir) {
if (is_created) {
int ret;
if (lstat_p != NULL && ctx_p->flags[MODSIGN]) {
}
if (perm & RA_WALK) {
if (path_full == NULL) {
*path_buf_p = sync_path_rel2abs(ctx_p, path_rel, -1, path_buf_len_p, *path_buf_p);
path_full = *path_buf_p;
path_full = *path_buf_p;
}
if (monitored) {
... ... @@ -1985,6 +1989,28 @@ int sync_prequeue_loadmark
return 0;
}
if (lstat_p != NULL && ctx_p->flags[MODSIGN]) {
fileinfo_t *finfo = indexes_fileinfo(indexes_p, path_rel);
if (finfo != NULL) {
if (!(stat_diff(&finfo->lstat, lstat_p) & ctx_p->flags[MODSIGN]))
return 0; // Skip file syncing if it's metadata not changed enough (according to "--modification-signature" setting)
if (is_deleted) {
// Removing file/dir information
indexes_fileinfo_add(indexes_p, path_rel, NULL);
free(finfo);
} else {
// Updating file/dir information
memcpy(&finfo->lstat, lstat_p, sizeof(finfo->lstat));
}
} else {
// Adding file/dir information
finfo = xmalloc(sizeof(*finfo));
memcpy(&finfo->lstat, lstat_p, sizeof(finfo->lstat));
indexes_fileinfo_add(indexes_p, path_rel, finfo);
}
}
switch (ctx_p->flags[MODE]) {
case MODE_SIMPLE:
return SAFE(sync_dosync(path_rel, event_mask, ctx_p, indexes_p), debug(1, "fpath == \"%s\"; evmask == 0x%o", path_rel, event_mask); return -1;);
... ... @@ -2208,7 +2234,7 @@ gboolean sync_trylocked(gpointer fpath_gp, gpointer evinfo_gp, gpointer arg_gp)
struct trylocked_arg *data = arg_p->data;
if (!sync_islocked(fpath)) {
if (sync_prequeue_loadmark(0, ctx_p, indexes_p, NULL, fpath,
if (sync_prequeue_loadmark(0, ctx_p, indexes_p, NULL, fpath, NULL,
evinfo->evmask,
evinfo->objtype_old,
evinfo->objtype_new,
... ... @@ -3615,11 +3641,12 @@ int sync_run(ctx_t *ctx_p) {
ctx_p->indexes_p = &indexes;
indexes.wd2fpath_ht = g_hash_table_new_full(g_direct_hash, g_direct_equal, 0, 0);
indexes.fpath2wd_ht = g_hash_table_new_full(g_str_hash, g_str_equal, free, 0);
indexes.fpath2ei_ht = g_hash_table_new_full(g_str_hash, g_str_equal, free, free);
indexes.exc_fpath_ht = g_hash_table_new_full(g_str_hash, g_str_equal, free, 0);
indexes.out_lines_aggr_ht = g_hash_table_new_full(g_str_hash, g_str_equal, free, 0);
indexes.wd2fpath_ht = g_hash_table_new_full(g_direct_hash, g_direct_equal, 0, 0);
indexes.fpath2wd_ht = g_hash_table_new_full(g_str_hash, g_str_equal, free, 0);
indexes.fpath2ei_ht = g_hash_table_new_full(g_str_hash, g_str_equal, free, free);
indexes.exc_fpath_ht = g_hash_table_new_full(g_str_hash, g_str_equal, free, 0);
indexes.out_lines_aggr_ht = g_hash_table_new_full(g_str_hash, g_str_equal, free, 0);
indexes.fileinfo_ht = g_hash_table_new_full(g_str_hash, g_str_equal, free, free);
i=0;
while (i<QUEUE_MAX) {
switch (i) {
... ... @@ -3863,6 +3890,7 @@ int sync_run(ctx_t *ctx_p) {
g_hash_table_destroy(indexes.fpath2ei_ht);
g_hash_table_destroy(indexes.exc_fpath_ht);
g_hash_table_destroy(indexes.out_lines_aggr_ht);
g_hash_table_destroy(indexes.fileinfo_ht);
i = 0;
while (i<QUEUE_MAX) {
switch (i) {
... ...
... ... @@ -33,6 +33,8 @@ extern int sync_prequeue_loadmark
const char *path_full,
const char *path_rel,
stat64_t *lstat_p,
eventobjtype_t objtype_old,
eventobjtype_t objtype_new,
... ...