redmine

Added option "-RR" to sync with rsync bypass any scripts

clsync
======
Contents
--------
... ... @@ -129,20 +130,23 @@ directory "example/testdir/to" ;)
First of all, recommended and not recommended options are notices in the
manpage.
However let's describe 3 situations:
However let's describe 4 situations:
- The simpliest usage (syncing from "/tmp/fromdir" to "/tmp/todir" with delay about 30 seconds):
```clsync -RR -d /dev/shm /tmp/fromdir $(which rsync) /dev/zero /tmp/todir```
- You're backing-up over very slow channel:
clsync -l backup -R -d /dev/shm -t 600 -T 3600 -B $[1024 * 1024 * 16] /home/user /home/clsync/bin/clsync-actionscript.sh /home/clsync/clsync-rules
```clsync -l backup -R -d /dev/shm -t 600 -T 3600 -B $[1024 * 1024 * 16] /home/user /home/clsync/bin/clsync-actionscript.sh /home/clsync/clsync-rules```
This will minimize network traffic. And pthread-ing is removed due to rarely
updating.
- You're syncing ordinary web-server over 1Gbs channel:
clsync -l mirror -p -R -d /dev/shm /var/www /home/clsync/bin/clsync-actionscript.sh /home/clsync/clsync-rules
```clsync -l mirror -p -R -d /dev/shm /var/www /home/clsync/bin/clsync-actionscript.sh /home/clsync/clsync-rules```
- You're syncing only few files from huge file tree (with a great lot of
excludes):
clsync -l mirror -p -R -d /dev/shm -I /home/user /home/clsync/bin/clsync-actionscript.sh /home/clsync/clsync-rules
```clsync -l mirror -p -R -d /dev/shm -I /home/user /home/clsync/bin/clsync-actionscript.sh /home/clsync/clsync-rules```
_____________________________________________
... ...
... ... @@ -131,7 +131,15 @@ struct options {
int flags[1<<8];
char *label;
char *watchdir;
char *destdir;
char *watchdirwslash;
char *destdirwslash;
size_t watchdirlen;
size_t destdirlen;
size_t watchdirsize;
size_t destdirsize;
size_t watchdirwslashsize;
size_t destdirwslashsize;
char *actfpath;
char *rulfpath;
char *listoutdir;
... ...
... ... @@ -12,7 +12,7 @@
#define DEFAULT_NOTIFYENGINE NE_INOTIFY
#define DEFAULT_COLLECTDELAY 30
#define DEFAULT_COMMONDELAY (DEFAULT_COLLECTDELAY/2)
#define DEFAULT_COMMONDELAY (0)
#define DEFAULT_BFILETHRESHOLD (128 * 1024 * 1024)
#define DEFAULT_BFILECOLLECTDELAY 1800
#define DEFAULT_LABEL "nolabel"
... ...
#!/bin/bash
make -C .. all
mkdir -p testdir/{from,to,listdir}
cat > rules <<EOF
-d/[Dd]ont[Ss]ync\$
+*.*
EOF
../clsync -RR -d ./testdir/listdir -w 2 -p -t 5 ./testdir/from `which rsync` rules ./testdir/to
... ...
... ... @@ -54,7 +54,7 @@ static struct option long_options[] =
};
int syntax() {
printf("syntax: clsync [flags] <watch dir> <action script> [file with rules regexps]\npossible options:\n");
printf("syntax: clsync [flags] <watch dir> <action script> [file with rules regexps] [destination directory]\npossible options:\n");
int i=0;
while(long_options[i].name != NULL) {
printf("\t--%-24s-%c\n", long_options[i].name, long_options[i].val);
... ... @@ -68,7 +68,7 @@ int version() {
exit(0);
}
int parse_arguments(int argc, char *argv[], struct options *options) {
int parse_arguments(int argc, char *argv[], struct options *options_p) {
int c;
int option_index = 0;
while(1) {
... ... @@ -85,49 +85,59 @@ int parse_arguments(int argc, char *argv[], struct options *options) {
syntax();
break;
case 'd':
options->listoutdir = optarg;
options_p->listoutdir = optarg;
break;
case 'l':
options->label = optarg;
options_p->label = optarg;
break;
case 'w':
options->commondelay = (unsigned int)atol(optarg);
options_p->commondelay = (unsigned int)atol(optarg);
case 't':
options->_queues[QUEUE_NORMAL].collectdelay = (unsigned int)atol(optarg);
options_p->_queues[QUEUE_NORMAL].collectdelay = (unsigned int)atol(optarg);
break;
case 'T':
options->_queues[QUEUE_BIGFILE].collectdelay = (unsigned int)atol(optarg);
options_p->_queues[QUEUE_BIGFILE].collectdelay = (unsigned int)atol(optarg);
break;
case 'B':
options->bfilethreshold = (unsigned long)atol(optarg);
options_p->bfilethreshold = (unsigned long)atol(optarg);
break;
#ifdef FANOTIFY_SUPPORT
case 'f':
options->notifyengine = NE_FANOTIFY;
options_p->notifyengine = NE_FANOTIFY;
break;
#endif
case 'i':
options->notifyengine = NE_INOTIFY;
options_p->notifyengine = NE_INOTIFY;
break;
case 'L':
options->rsyncinclimit = (unsigned int)atol(optarg);
options_p->rsyncinclimit = (unsigned int)atol(optarg);
break;
case 'V':
version();
break;
default:
options->flags[c]++;
options_p->flags[c]++;
break;
}
}
if(optind+1 >= argc)
syntax();
options->actfpath = argv[optind+1];
if(optind+2 < argc)
options->rulfpath = argv[optind+2];
options_p->actfpath = argv[optind+1];
options->watchdir = argv[optind];
if(optind+2 < argc) {
options_p->rulfpath = argv[optind+2];
if(!strcmp(options_p->rulfpath, ""))
options_p->rulfpath = NULL;
}
if(optind+3 < argc) {
options_p->destdir = argv[optind+3];
options_p->destdirlen = strlen(options_p->destdir);
}
options_p->watchdir = argv[optind];
options_p->watchdirlen = strlen(options_p->watchdir);
return 0;
}
... ... @@ -137,6 +147,9 @@ int parse_rules_fromfile(const char *rulfpath, rule_t *rules) {
FILE *f = fopen(rulfpath, "r");
if(f == NULL) {
#ifdef PARANOID
rules->action = RULE_END;
#endif
printf_e("Error: Cannot open \"%s\" for reading: %s (errno: %i).\n", rulfpath, strerror(errno), errno);
return errno;
}
... ... @@ -254,6 +267,8 @@ int main_cleanup(options_t *options_p) {
while((i < MAXRULES) && (options_p->rules[i].action != RULE_END))
regfree(&options_p->rules[i++].expr);
printf_ddd("Debug3: main_cleanup(): %i %i %i %i\n", options_p->watchdirsize, options_p->watchdirwslashsize, options_p->destdirsize, options_p->destdirwslashsize);
return 0;
}
... ... @@ -274,7 +289,7 @@ int main_rehash(options_t *options_p) {
int main(int argc, char *argv[]) {
struct options options;
int ret = 0, nret;
struct stat64 stat64;
struct stat64 stat64={0};
memset(&options, 0, sizeof(options));
options.notifyengine = DEFAULT_NOTIFYENGINE;
options.commondelay = DEFAULT_COMMONDELAY;
... ... @@ -287,9 +302,52 @@ int main(int argc, char *argv[]) {
parse_arguments(argc, argv, &options);
out_init(options.flags);
if((options.flags[RSYNC]>1) && (options.destdir == NULL)) {
printf_e("Error: Option \"-RR\" cannot be used without specifing \"destination directory\".\n");
ret = EINVAL;
}
{
size_t size = options.watchdirlen + 2;
char *newwatchdir = xmalloc(size);
memcpy( newwatchdir, options.watchdir, options.watchdirlen);
if(options.watchdir[options.watchdirlen - 1] == '/') {
options.watchdirsize = size;
options.watchdirwslash = options.watchdir;
options.watchdir = newwatchdir;
options.watchdirlen--;
newwatchdir[options.watchdirlen] = 0x00;
} else {
options.watchdirwslash = newwatchdir;
options.watchdirwslashsize = size;
memcpy(&options.watchdirwslash[options.watchdirlen], "/", 2);
}
}
if(options.destdir != NULL) {
size_t size = options.destdirlen + 2;
char *newdestdir = xmalloc(size);
memcpy( newdestdir, options.destdir, options.destdirlen);
if(options.destdir[options.destdirlen - 1] == '/') {
options.destdirsize = size;
options.destdirwslash = options.destdir;
options.destdir = newdestdir;
options.destdirlen--;
newdestdir[options.destdirlen] = 0x00;
} else {
options.destdirwslash = newdestdir;
options.destdirwslashsize = size;
memcpy(&options.destdirwslash[options.destdirlen], "/", 2);
}
}
printf_ddd("Debug3: %s [%s] (%p) -> %s [%s]\n", options.watchdir, options.watchdirwslash, options.watchdirwslash, options.destdir?options.destdir:"", options.destdirwslash?options.destdirwslash:"");
if(options.flags[RSYNC] && (options.listoutdir == NULL)) {
printf_e("Error: Option \"--rsync\" cannot be used without \"--outlistsdir\".\n");
return EINVAL;
ret = EINVAL;
}
if(options.flags[RSYNC_PREFERINCLUDE] && (!options.flags[RSYNC]))
printf_e("Warning: Option \"--rsyncpreferinclude\" is useless without \"--rsync\".\n");
... ... @@ -330,12 +388,23 @@ int main(int argc, char *argv[]) {
ret = nret;
}
options.watchdirlen = strlen(options.watchdir);
if(ret == 0)
ret = sync_run(&options);
main_cleanup(&options);
if(options.watchdirsize)
free(options.watchdir);
if(options.watchdirwslashsize)
free(options.watchdirwslash);
if(options.destdirsize)
free(options.destdir);
if(options.destdirwslashsize)
free(options.destdirwslash);
out_flush();
printf_d("Debug: finished, exitcode: %i.\n", ret);
out_flush();
... ...
... ... @@ -32,6 +32,9 @@ clsync \- live sync tool, written in GNU C
.B [
.I rules\-file
.B ]
.B [
.I dest-dir
.B ]
.SH DESCRIPTION
.B clsync
executes
... ... @@ -84,7 +87,7 @@ Default value is "30".
Sets the additional delay (in seconds) to collect events about any file
or directory.
Default value is "15".
Default value is "0".
.RE
.PP
... ... @@ -125,6 +128,13 @@ option.
Recommended option.
You can use "-RR" to run "rsync" directly, bypassing intermediate script
(see
.B ACTION SCRIPT
case
.B e
)
Is not set by default.
.RE
... ... @@ -289,7 +299,7 @@ is only a convenient way to run a syncing script.
.B clsync
can run
.I action\-script
in four ways:
in five ways:
case
.B a
... ... @@ -372,7 +382,26 @@ if option
is disabled.
.RE
As can be noticed, in all four cases clsync's
case
.B e
.RS
.I action\-script
\-avH \-\-delete\-before [\-\-exclude\-from
.I rsync\-exclude\-listpath
]
\-\-include\-from
.I rsync\-listpath
\-\-exclude '*'
.I watch-dir/ dest-dir/
In this case,
.I action\-script
is supposed to be a path to
.B rsync
binary.
.RE
As can be noticed, in the first four cases clsync's
.I label
is passed (see
.I \-\-label
... ...
... ... @@ -620,6 +620,7 @@ int sync_initialsync(const char *path, options_t *options_p, indexes_t *indexes_
}
int sync_notify_mark(int notify_d, options_t *options_p, const char *accpath, const char *path, size_t pathlen, indexes_t *indexes_p) {
printf_ddd("Debug3: sync_notify_mark(..., \"%s\", %i,...)\n", path, pathlen);
int wd = indexes_fpath2wd(indexes_p, path);
if(wd != -1) {
printf_d("Debug: \"%s\" is already marked (wd: %i). Skipping.\n", path, wd);
... ... @@ -870,6 +871,30 @@ int sync_idle_dosync_collectedevents_cleanup(options_t *options_p, char **argv)
if(options_p->flags[DONTUNLINK])
return 0;
if(options_p->flags[RSYNC] >= 2) {
int ret0, ret1;
if(argv[4] == NULL) {
printf_e("Error: Unexpected *argv[] end.");
return EINVAL;
}
printf_ddd("Debug3: unlink()-ing \"%s\"\n", argv[4]);
ret0 = unlink(argv[4]);
if(options_p->flags[RSYNC_PREFERINCLUDE])
return ret0;
if(argv[6] == NULL) {
printf_e("Error: Unexpected *argv[] end.");
return EINVAL;
}
printf_ddd("Debug3: unlink()-ing \"%s\"\n", argv[6]);
ret1 = unlink(argv[6]);
return ret0 == 0 ? ret1 : ret0;
}
if(argv[3] == NULL) {
printf_e("Error: Unexpected *argv[] end.");
return EINVAL;
... ... @@ -1012,8 +1037,8 @@ gboolean sync_idle_dosync_collectedevents_rsync_exclistpush(gpointer fpath_gp, g
}
int sync_idle_dosync_collectedevents_commitpart(struct dosync_arg *dosync_arg_p) {
printf_ddd("Debug3: Committing the file\n");
options_t *options_p = dosync_arg_p->options_p;
printf_ddd("Debug3: Committing the file (flags[RSYNC] == %i)\n", options_p->flags[RSYNC]);
fclose(dosync_arg_p->outf);
dosync_arg_p->outf = NULL;
... ... @@ -1037,6 +1062,24 @@ int sync_idle_dosync_collectedevents_commitpart(struct dosync_arg *dosync_arg_p)
dosync_arg_p->outf_path,
*(dosync_arg_p->outf_path)?dosync_arg_p->outf_path:NULL,
NULL);*/
printf_ddd("Debug3: %s [%s] (%p) -> %s [%s]\n", options_p->watchdir, options_p->watchdirwslash, options_p->watchdirwslash,
options_p->destdir?options_p->destdir:"", options_p->destdirwslash?options_p->destdirwslash:"");
if(options_p->flags[RSYNC] >= 2)
return SYNC_EXEC(options_p,
sync_idle_dosync_collectedevents_cleanup,
options_p->actfpath,
"-avH",
"--delete-before",
*(dosync_arg_p->excf_path) ? "--exclude-from" : "--include-from",
*(dosync_arg_p->excf_path) ? dosync_arg_p->excf_path : dosync_arg_p->outf_path,
*(dosync_arg_p->excf_path) ? "--include-from" : "--exclude=*",
*(dosync_arg_p->excf_path) ? dosync_arg_p->outf_path : options_p->watchdirwslash,
*(dosync_arg_p->excf_path) ? "--exclude=*" : options_p->destdirwslash,
*(dosync_arg_p->excf_path) ? options_p->watchdirwslash : NULL,
*(dosync_arg_p->excf_path) ? options_p->destdirwslash : NULL,
NULL);
return SYNC_EXEC(options_p,
sync_idle_dosync_collectedevents_cleanup,
options_p->actfpath,
... ... @@ -1105,11 +1148,21 @@ gboolean sync_idle_dosync_collectedevents_listpush(gpointer fpath_gp, gpointer e
outf = dosync_arg_p->outf;
}
size_t fpathlen = strlen(fpath);
char *fpath_rel_p = xmalloc(fpathlen+1);
size_t fpath_len = strlen(fpath);
size_t fpath_rel_len = fpath_len - options_p->watchdirlen;
char *fpath_rel_p = xmalloc(fpath_len+1);
char *fpath_rel = fpath_rel_p;
memcpy(fpath_rel, &fpath[options_p->watchdirlen], fpathlen+1 - options_p->watchdirlen);
memcpy(fpath_rel, &fpath[options_p->watchdirlen], fpath_rel_len+1);
#ifdef VERYPARANOID
// Removing "/" on the end
printf_ddd("Debug3: \"%s\" (len: %i) --%i--> ", fpath_rel, fpath_rel_len, fpath_rel[fpath_rel_len - 1] == '/');
if(fpath_rel[fpath_rel_len - 1] == '/')
fpath_rel[--fpath_rel_len] = 0x00;
printf_ddd("\"%s\" (len: %i)\n", fpath_rel, fpath_rel_len);
#endif
char *end=fpath_rel;
... ... @@ -1446,7 +1499,10 @@ int sync_inotify_loop(int inotify_d, options_t *options_p, indexes_t *indexes_p)
int events = sync_inotify_wait(inotify_d, options_p, indexes_p);
switch(state) {
case STATE_PTHREAD_GC:
thread_gc(options_p);
if(thread_gc(options_p)) {
state=STATE_EXIT;
break;
}
state = STATE_RUNNING;
continue;
case STATE_RUNNING:
... ...