UT / clsync

Go to a project
Toggle navigation Toggle navigation pinning
redmine
95b61744 1 parent 31b9e40f

Tiny refactoring of "rules" code

Expand all
Inline Side-by-side
Showing 9 changed files with 80 additions and 403 deletions
... ... @@ -5,9 +5,10 @@ if CLSYNC
bin_PROGRAMS = clsync
clsync_SOURCES = calc.c cluster.c error.c fileutils.c glibex.c \
indexes.c main.c malloc.c stringex.c sync.c privileged.c calc.h \
cluster.h fileutils.h glibex.h main.h port-hacks.h stringex.h \
sync.h common.h control.h privileged.h syscalls.h
indexes.c main.c malloc.c rules.c stringex.c sync.c privileged.c\
calc.h cluster.h fileutils.h glibex.h main.h port-hacks.h \
stringex.h sync.h common.h control.h privileged.h rules.h \
syscalls.h
clsync_CFLAGS = $(AM_CFLAGS)
... ... @@ -52,6 +53,9 @@ if HAVE_PIVOTROOT
clsync_CFLAGS += -DPIVOTROOT_OPT_SUPPORT
endif
endif
if HAVE_TRE
clsync_CFLAGS += -DTRE_SUPPORT
endif
if HLLOCKS
clsync_CFLAGS += -DHL_LOCKS
... ...
... ... @@ -55,7 +55,6 @@ AS_HELP_STRING(--enable-cluster,
[enable clustering support (not yet implemented), default: no]))
AS_IF([test "x$enable_cluster" = "xyes"], [CPPFLAGS="${CPPFLAGS} -DCLUSTER_SUPPORT"
dnl mhash check
AC_ARG_WITH(mhash,
AS_HELP_STRING(--with-mhash,
... ... @@ -68,6 +67,7 @@ AS_IF([test "x$enable_cluster" = "xyes"], [CPPFLAGS="${CPPFLAGS} -DCLUSTER_SUPPO
])
])
dnl --enable-socket
AC_ARG_ENABLE(socket,
AS_HELP_STRING(--enable-socket,
... ... @@ -179,6 +179,35 @@ case "$with_capabilities" in
;;
esac
dnl tre check
AC_ARG_WITH(tre,
AS_HELP_STRING(--with-tre,
[Enable tre support be able to predict which directories should be scanned for excludes; values: no, check, yes; default: check])
[],
[with_capabilities=check]
)
case "$with_tre" in
yes)
AC_CHECK_FUNC([regaexec],
[
AC_CHECK_HEADER(tre/tre.h, [HAVE_TRE=2], [AC_MSG_FAILURE([Cannot find tre/tre.h])])
],
[
AC_MSG_FAILURE([Cannot call function regaexec from libtre])
]
)
;;
check)
AC_CHECK_FUNC([regaexec],
[
AC_CHECK_HEADER(tre/tre.h, [HAVE_TRE=2])
]
)
;;
esac
dnl kqueue/inotify/bsm
AC_ARG_WITH(kqueue,
... ... @@ -341,6 +370,7 @@ AM_CONDITIONAL([HAVE_GETMNTENT], [test "x$HAVE_GETMNTENT" != "x"])
AM_CONDITIONAL([HAVE_PIVOTROOT], [test "x$HAVE_PIVOTROOT" != "x"])
AM_CONDITIONAL([HAVE_UNSHARE], [test "x$HAVE_UNSHARE" != "x"])
AM_CONDITIONAL([HAVE_SECCOMP], [test "x$HAVE_SECCOMP" != "x"])
AM_CONDITIONAL([HAVE_TRE], [test "x$HAVE_TRE" != "x"])
AS_IF([test "$HAVE_KQUEUE" = '' -a "$HAVE_INOTIFY" = '' -a "$HAVE_FANOTIFY" = '' -a "$HAVE_BSM" = '' ], [AC_MSG_FAILURE([kqueue, inotify and bsm are not supported on this system])])
... ...
... ... @@ -172,7 +172,6 @@ enum ruleaction_enum {
RA_MONITOR = 0x01,
RA_WALK = 0x02,
RA_ALL = 0x0f,
RA_INITSYNC_STARTDENY = 0x10,
};
typedef enum ruleaction_enum ruleaction_t;
... ... @@ -187,7 +186,6 @@ enum sigusr_enum {
struct rule {
int num;
regex_t expr;
regex_t expr_start; /* To optimize FTS scanning */
mode_t objtype;
ruleaction_t perm;
ruleaction_t mask;
... ... @@ -270,6 +268,7 @@ struct ctx {
int children; // Used only for non-pthread mode
uint32_t iteration_num;
rule_t rules[MAXRULES];
size_t rules_count;
dev_t st_dev;
#endif
char *flags_values_raw[OPTION_FLAGS];
... ...
... ... @@ -47,6 +47,7 @@
#include "fileutils.h"
#include "socket.h"
#include "syscalls.h"
#include "rules.h"
//#include "revision.h"
... ... @@ -1809,269 +1810,6 @@ void ctx_cleanup(ctx_t *ctx_p) {
return;
}
int rule_complete(rule_t *rule_p, char *expr) {
char *end_of_start;
debug(3, "<%s>.", expr);
#ifdef VERYPARANOID
if (rule_p->mask == RA_NONE) {
error("Received a rule with rule_p->mask == 0x00. Exit.");
return EINVAL;
}
#endif
char buf[BUFSIZ];
int ret = 0;
if (rule_p->num >= MAXRULES) {
error("Too many rules (%i >= %i).", rule_p->num, MAXRULES);
return ENOMEM;
}
if ((ret = regcomp(&rule_p->expr, expr, REG_EXTENDED | REG_NOSUB))) {
regerror(ret, &rule_p->expr, buf, BUFSIZ);
error("Invalid regexp pattern <%s>: %s (regex-errno: %i).", expr, buf, ret);
return ret;
}
end_of_start = strstr(expr, ".*");
if (end_of_start != NULL)
*end_of_start = 0;
if ((ret = regcomp(&rule_p->expr_start, expr, REG_EXTENDED | REG_NOSUB))) {
regerror(ret, &rule_p->expr, buf, BUFSIZ);
error("Invalid regexp pattern <%s> (shorted): %s (regex-errno: %i).", expr, buf, ret);
return ret;
}
if (end_of_start != NULL)
*end_of_start = '.';
return ret;
}
int parse_rules_fromfile(ctx_t *ctx_p) {
int ret = 0;
char *rulfpath = ctx_p->rulfpath;
rule_t *rules = ctx_p->rules;
char *line_buf=NULL;
FILE *f = fopen(rulfpath, "r");
if(f == NULL) {
rules->mask = RA_NONE; // Terminator. End of rules' chain.
rules->perm = DEFAULT_RULES_PERM;
error("Cannot open \"%s\" for reading.", rulfpath);
return errno;
}
GHashTable *autowrules_ht = g_hash_table_new_full(g_str_hash, g_str_equal, free, 0);
int i=0;
size_t linelen, size=0;
while((linelen = getline(&line_buf, &size, f)) != -1) {
if(linelen>1) {
uint8_t sign = 0;
char *line = line_buf;
rule_t *rule;
rule = &rules[i];
#ifdef VERYPARANOID
memset(rule, 0, sizeof(*rule));
#endif
rule->num = i++;
line[--linelen] = 0;
// Parsing the first character of the line
switch(*line) {
case '+':
sign = RS_PERMIT;
break;
case '-':
sign = RS_REJECT;
break;
case '#': // Comment?
i--; // Canceling new rule
continue;
default:
error("Wrong rule action <%c>.", *line);
return EINVAL;
}
line++;
linelen--;
// Parsing the second character of the line
*line |= 0x20; // lower-casing
// Default rule->mask and rule->perm
// rule->mask - sets bitmask of operations that are affected by the rule
// rule->perm - sets bitmask of permit/reject for every operation. Effect have only bits specified by the rule->mask.
rule->mask = RA_ALL;
switch(sign) {
case RS_REJECT:
rule->perm = RA_NONE;
break;
case RS_PERMIT:
rule->perm = RA_ALL;
break;
}
switch(*line) {
case '*':
rule->objtype = 0; // "0" - means "of any type"
break;
#ifdef DETAILED_FTYPE
case 's':
rule->objtype = S_IFSOCK;
break;
case 'l':
rule->objtype = S_IFLNK;
break;
case 'b':
rule->objtype = S_IFBLK;
break;
case 'c':
rule->objtype = S_IFCHR;
break;
case 'p':
rule->objtype = S_IFIFO;
break;
#endif
case 'f':
rule->objtype = S_IFREG;
break;
case 'd':
rule->objtype = S_IFDIR;
break;
case 'w': // accept or reject walking to directory
if(
(ctx_p->flags[MODE] == MODE_RSYNCDIRECT) ||
(ctx_p->flags[MODE] == MODE_RSYNCSHELL) ||
(ctx_p->flags[MODE] == MODE_RSYNCSO)
) {
error("Warning: Used \"w\" rule in \"--rsync\" case."
" This may cause unexpected problems.");
}
rule->objtype = S_IFDIR;
rule->mask = RA_WALK;
break;
default:
error("Warning: Cannot parse the rule <%s>", &line[-1]);
i--; // Canceling new rule
continue;
}
line++;
linelen--;
// Parsing the rest part of the line
debug(1, "Rule #%i <%c>[0x%02x 0x%02x] <%c>[0x%04x] pattern <%s> (length: %i).", rule->num, line[-2], rule->perm, rule->mask, line[-1], rule->objtype, line, linelen);
if((ret=rule_complete(rule, line)))
goto l_parse_rules_fromfile_end;
// Post-processing:
line--;
linelen++;
if(*line != 'w') {
// processing --auto-add-rules-w
if(ctx_p->flags[AUTORULESW] && (sign == RS_PERMIT)) {
// Preparing to add appropriate w-rules
char skip = 0;
char *expr = alloca(linelen+2);
memcpy(expr, line, linelen+1);
size_t exprlen = linelen;
// Making expr to be starting with '^'
if(line[1] == '^') {
expr++;
exprlen--;
} else
*expr = '^';
char *end;
if(*line == 'd' || *line == '*') {
// "d" rule already doing what we need, so we can skip the last level
end = &expr[exprlen];
if(end[-1] != '$')
*(end++) = '$';
*end = 0;
// debug(3, "Don't adding w-rule for \"%s\" due to [*d]-rule for \"%s\"",
// expr, &line[1]);
g_hash_table_insert(autowrules_ht, strdup(expr), GINT_TO_POINTER(1));
}
if(!skip) {
do {
// Decreasing directory level and make the '$' ending
end = strrchr(expr, '/');
if(end != NULL) {
if(end[-1] != '$')
*(end++) = '$';
*end = 0;
exprlen = (size_t)(end - expr);
} else {
expr[1] = '$';
expr[2] = 0;
exprlen = 2;
}
// Checking if it not already set
if(!g_hash_table_lookup(autowrules_ht, expr)) {
// Switching to next rule:
rule = &rules[i];
rule->num = i++;
// Adding the rule
rule->objtype = S_IFDIR;
rule->mask = RA_WALK;
rule->perm = RA_WALK;
debug(1, "Rule #%i <+> <w> pattern <%s> (length: %i) [auto].",
rule->num, expr, exprlen);
if((ret=rule_complete(rule, expr)))
goto l_parse_rules_fromfile_end;
g_hash_table_insert(autowrules_ht, strdup(expr), GINT_TO_POINTER(1));
}
} while(end != NULL);
}
}
}
}
}
l_parse_rules_fromfile_end:
if(size)
free(line_buf);
fclose(f);
debug(3, "Adding tail-rule #%u (effective #%u).", -1, i);
rules[i].mask = RA_NONE; // Terminator. End of rules' chain.
rules[i].perm = DEFAULT_RULES_PERM;
g_hash_table_destroy(autowrules_ht);
#ifdef _DEBUG_FORCE
debug(3, "Total (p == %p):", rules);
i=0;
do {
debug(4, "\t%i\t%i\t%p/%p", i, rules[i].objtype, (void *)(long)rules[i].perm, (void *)(long)rules[i].mask);
i++;
} while(rules[i].mask != RA_NONE);
#endif
return ret;
}
int becomedaemon() {
int pid;
signal(SIGPIPE, SIG_IGN);
... ...
... ... @@ -23,6 +23,7 @@ extern int main_rehash(ctx_t *ctx_p);
extern int main_status_update(ctx_t *ctx_p);
extern int ctx_set(ctx_t *ctx_p, const char *const parameter_name, const char *const parameter_value);
extern int config_block_parse(ctx_t *ctx_p, const char *const config_block_name);
extern int rules_count(ctx_t *ctx_p);
extern char *parameter_expand(
ctx_t *ctx_p,
char *arg,
... ...
... ... @@ -796,9 +796,9 @@ Is not set by default.
.RS
Enable experimental features to optimize file tree scanning while using
.BR fts "(3)."
The features will be enabled by default after appropriate testing.
At the moment it will try force to skip scanning for excludes of
directories if their content doesn't fit to any MONITOR deny rule.
At the moment the option doesn't do anything but can be used in future.
Is not set by default.
.RE
... ...
This diff is collapsed. Click to expand it.
/*
clsync - file tree sync utility based on inotify/kqueue
Copyright (C) 2013-2014 Dmitry Yu Okunev <dyokunev@ut.mephi.ru> 0x8E30679C
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
extern int parse_rules_fromfile(struct ctx *ctx_p);
extern ruleaction_t rules_search_getperm(const char *fpath, mode_t st_mode, rule_t *rules_p, const ruleaction_t ruleaction, rule_t **rule_pp);
extern ruleaction_t rules_getperm(const char *fpath, mode_t st_mode, struct rule *rules_p, ruleaction_t ruleactions);
... ...
... ... @@ -45,6 +45,7 @@
#include "control.h"
#include "indexes.h"
#include "privileged.h"
#include "rules.h"
#include <stdio.h>
#include <dlfcn.h>
... ... @@ -141,122 +142,6 @@ int exitcode_process(ctx_t *ctx_p, int exitcode) {
return err;
}
/**
* @brief Checks file path by rules' expressions (parsed from file)
*
* @param[in] fpath Path to file of directory
* @param[in] st_mode st_mode received via *stat() functions
* @param[in] rules_p Pointer to start of rules array
* @param[in] ruleaction Operaton ID (see ruleaction_t)
* @param[i/o] rule_pp Pointer to pointer to rule, where the last search ended. Next search will be started from the specified rule. Can be "NULL" to disable this feature.
*
* @retval perm Permission bitmask
*
*/
// Checks file path by rules' expressions (parsed from file)
// Return: RS_PERMIT or RS_REJECT for the "file path" and specified ruleaction
ruleaction_t rules_search_getperm(const char *fpath, mode_t st_mode, rule_t *rules_p, const ruleaction_t ruleaction, rule_t **rule_pp) {
debug(3, "rules_search_getperm(\"%s\", %p, %p, %p, %p)",
fpath, (void *)(unsigned long)st_mode, rules_p,
(void *)(long)ruleaction, (void *)(long)rule_pp
);
int i;
i = 0;
rule_t *rule_p = rules_p;
mode_t ftype = st_mode & S_IFMT;
ruleaction_t ruleaction_eff;
#ifdef _DEBUG_FORCE
debug(3, "Rules (p == %p):", rules_p);
i=0;
do {
debug(3, "\t%i\t%i\t%p/%p", i, rules_p[i].objtype, (void *)(long)rules_p[i].perm, (void *)(long)rules_p[i].mask);
i++;
} while(rules_p[i].mask != RA_NONE);
#endif
ruleaction_eff = (ruleaction & ~RA_ALL ? RA_MONITOR : ruleaction);
i=0;
if (rule_pp != NULL)
if (*rule_pp != NULL) {
debug(3, "Previous position is set.");
if (rule_p->mask == RA_NONE)
return rule_p->perm;
rule_p = ++(*rule_pp);
i = rule_p->num;
}
debug(3, "Starting from position %i", i);
while (rule_p->mask != RA_NONE) {
debug(3, "%i -> %p/%p: type compare: %p, %p -> %i",
i,
(void *)(long)rule_p->perm, (void *)(long)rule_p->mask,
(void *)(unsigned long)ftype, (void *)(unsigned long)rule_p->objtype,
(unsigned char)!(rule_p->objtype && (rule_p->objtype != ftype))
);
if (!(rule_p->mask & ruleaction_eff)) { // Checking wrong operation type
debug(3, "action-mask mismatch. Skipping.");
rule_p++;i++;// = &rules_p[++i];
continue;
}
if (rule_p->objtype && (rule_p->objtype != ftype)) {
debug(3, "objtype mismatch. Skipping.");
rule_p++;i++;// = &rules_p[++i];
continue;
}
if (!regexec((ruleaction & ~RA_ALL ? &rule_p->expr_start : &rule_p->expr), fpath, 0, NULL, 0)) {
if (!(ruleaction & ~RA_ALL))
break;
if (!(rule_p->perm & ruleaction_eff))
break;
}
debug(3, "doesn't match regex. Skipping.");
rule_p++;i++;// = &rules_p[++i];
}
debug(2, "matched to rule #%u for \"%s\":\t%p/%p (queried: %p).", rule_p->mask==RA_NONE?-1:i, fpath,
(void *)(long)rule_p->perm, (void *)(long)rule_p->mask,
(void *)(long)ruleaction
);
if (rule_pp != NULL)
*rule_pp = rule_p;
return rule_p->perm;
}
static inline ruleaction_t rules_getperm(const char *fpath, mode_t st_mode, rule_t *rules_p, ruleaction_t ruleactions) {
rule_t *rule_p = NULL;
ruleaction_t gotpermto = 0;
ruleaction_t resultperm = 0;
debug(3, "rules_getperm(\"%s\", %p, %p (#%u), %p)",
fpath, (void *)(long)st_mode, rules_p, rules_p->num, (void *)(long)ruleactions);
while((gotpermto&ruleactions) != ruleactions) {
rules_search_getperm(fpath, st_mode, rules_p, ruleactions, &rule_p);
if(rule_p->mask == RA_NONE) { // End of rules' list
resultperm |= rule_p->perm & (gotpermto^RA_ALL);
break;
}
resultperm |= rule_p->perm & ((gotpermto^rule_p->mask)&rule_p->mask); // Adding perm bitmask of operations that was unknown before
gotpermto |= rule_p->mask; // Adding the mask
}
debug(3, "rules_getperm(\"%s\", %p, rules_p, %p): result perm is %p",
fpath, (void *)(long)st_mode, (void *)(long)ruleactions, (void *)(long)resultperm);
return resultperm;
}
threadsinfo_t *thread_info() { // TODO: optimize this
static threadsinfo_t threadsinfo={{{{0}}},{{{0}}},0};
... ... @@ -1342,6 +1227,8 @@ int sync_initialsync_walk(ctx_t *ctx_p, const char *dirpath, indexes_t *indexes_
if ((!ctx_p->flags[RSYNCPREFERINCLUDE]) && skip_rules)
return 0;
skip_rules |= (ctx_p->rules_count == 0);
char fts_no_stat = (initsync==INITSYNC_FULL) && !(ctx_p->flags[EXCLUDEMOUNTPOINTS]);
int fts_opts = FTS_NOCHDIR | FTS_PHYSICAL |
... ... @@ -1427,22 +1314,6 @@ int sync_initialsync_walk(ctx_t *ctx_p, const char *dirpath, indexes_t *indexes_
debug(3, "Rejecting to walk into \"%s\".", path_rel);
fts_set(tree, node, FTS_SKIP);
} else
/* "FTS optimization" */
if (
ctx_p->flags[FTS_EXPERIMENTAL_OPTIMIZATION] &&
node->fts_info == FTS_D &&
!ctx_p->flags[EXCLUDEMOUNTPOINTS]
) {
debug(4, "\"FTS optimizator\"");
ruleaction_t perm = rules_getperm(path_rel, st_mode, rules_p, RA_INITSYNC_STARTDENY);
int result = ((perm&RA_MONITOR) != 0) ^ (ctx_p->flags[RSYNCPREFERINCLUDE] != 0);
if (result) {
debug(3, "\"FTS optimizator\": Rejecting to walk into \"%s\".", path_rel);
fts_set(tree, node, FTS_SKIP);
}
}
if (!(perm&RA_MONITOR)) {
debug(3, "Excluding \"%s\".", path_rel);
... ... @@ -1474,6 +1345,17 @@ int sync_initialsync_walk(ctx_t *ctx_p, const char *dirpath, indexes_t *indexes_
ret = errno;
goto l_sync_initialsync_walk_end;
}
continue;
}
/* "FTS optimization" */
if (
skip_rules &&
node->fts_info == FTS_D &&
!ctx_p->flags[EXCLUDEMOUNTPOINTS]
) {
debug(4, "\"FTS optimizator\"");
fts_set(tree, node, FTS_SKIP);
}
}
if (errno) {
... ...