Merge tag 'upstream/0.4'

Conflicts: (mainly take debian branch)
	debian/changelog
	debian/clsync-dev.install
	debian/clsync.install
	debian/control
	debian/libclsync-dev.install
	debian/rules

@@ -54,7 +54,7 @@ PROJECT_LOGO           =
 # If a relative path is entered, it will be relative to the location
 # where doxygen was started. If left blank the current directory will be used.
 
-OUTPUT_DIRECTORY       = doc
+OUTPUT_DIRECTORY       = doc/doxygen
 
 # If the CREATE_SUBDIRS tag is set to YES, then doxygen will create
 # 4096 sub-directories (in 2 levels) under the output directory of each output

@@ -54,6 +54,7 @@ tmp
 /config.log
 /config.status
 /config.sub
+/config.cache
 /configure
 /depcomp
 /examples/Makefile

@@ -22,7 +22,8 @@ build_test() {
 	$MAKE clean
 	echo ">>> Testing with \"$@\""
 	# make sure we test paralled build as they tend to fail when single works
-	./configure $@ && $MAKE -j5 || {
+	#./configure -C $@ >/dev/null || rm -f config.cache && ./configure -C $@ >/dev/null && $MAKE -j5 >/dev/null || {
+	./configure $@ >/dev/null && $MAKE -j5 >/dev/null || {
 		echo "!!! test with \"$@\" configure options failed"
 		exit 1
 	}
@@ -40,18 +41,22 @@ run_example_cleanup_failure() {
 }
 
 # Run example script
+run_example_counter=0
 run_example() {
-	MODE="$1"
+	MODE="$1"; shift;
 
-	export CLSYNC_PIDFILE="/tmp/clsync-example-$MODE.pid"
+	export CLSYNC_PIDFILE="/tmp/clsync-example-$MODE.$$.${run_example_counter}.pid"
+
+	run_example_counter=$(( $run_example_counter + 1 ))
 
 	rm -rf "examples/testdir"/*/*
 	mkdir -p "examples/testdir/to" "examples/testdir/from"
 
 	trap run_example_cleanup_failure INT TERM
-	cd examples
-	bash -x clsync-start-"$MODE".sh --background --pid-file "$CLSYNC_PIDFILE" --config-file '/NULL/' -w1 -t1 -d1
-	cd -
+	(
+		cd examples
+		bash -x clsync-start-"$MODE".sh --background --pid-file "$CLSYNC_PIDFILE" --config-file '/NULL/' -w1 -t1 -d0 $@
+	)
 
 	sleep 1
 	mkdir -p examples/testdir/from/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z/DIR
@@ -103,13 +108,16 @@ if true; then
 
 	# clsync enabled
 	a0="--enable-clsync"
-	for a1 in "--enable-cluster --with-mhash" "--enable-cluster --without-mhash" "--disable-cluster"; do
-	for a2 in "--enable-debug" "--disable-debug"; do
+	for a1 in "--enable-cluster" "--disable-cluster"; do
+#	for a2 in "--enable-debug" "--disable-debug"; do
 	for a3 in "--enable-paranoid=0" "--enable-paranoid=1" "--enable-paranoid=2" ; do
 	for a4 in "--with-capabilities" "--without-capabilities"; do
 	for a5 in "--enable-socket" "--disable-socket"; do
-	for a6 in "--enable-libclsync" "--disable-libclsync"; do
-		arg="$a0 $a1 $a2 $a3 $a4 $a5 $a6"
+	for a6 in "--enable-socket-library" "--disable-socket-library"; do
+	for a7 in "--enable-highload-locks" ""; do
+#	for a8 in "--with-libcgroup" "--without-libcgroup"; do
+#	for a9 in "--with-libseccomp" "--without-libseccomp"; do
+		arg="$a0 $a1 $a2 $a3 $a4 $a5 $a6 $a7 $a8 $a9"
 		build_test "$arg"
 	done
 	done
@@ -117,18 +125,21 @@ if true; then
 	done
 	done
 	done
+#	done
+#	done
+#	done
 
 	# clsync disabled, libclsync enabled
-	a0="--disable-clsync --enable-libclsync"
-	for a2 in "--enable-debug" "--disable-debug"; do
+	a0="--disable-clsync --enable-socket-library"
+#	for a2 in "--enable-debug" "--disable-debug"; do
 	for a3 in "--enable-paranoid=0" "--enable-paranoid=1" "--enable-paranoid=2" ; do
 		arg="$a0 $a1 $a2"
 		build_test "$arg"
-	done
+#	done
 	done
 
 	# clsync disabled, libclsync disabled
-	build_test "--disable-clsync --disable-libclsync"
+	build_test "--disable-clsync --disable-socket-library"
 
 fi
 
@@ -140,7 +151,10 @@ if true; then
 	export PATH="$(pwd):$PATH"
 	build_test --enable-cluster --enable-debug --enable-paranoid=2 --with-capabilities --without-mhash
 	run_example rsyncdirect
-	run_example rsyncshell
+	run_example rsyncdirect --splitting=thread --threading=off
+	run_example rsyncdirect --splitting=process --threading=off
+	run_example rsyncdirect --threading=safe
+#	run_example rsyncshell
 #	run_example rsyncso
 	#run_example so
 	#run_example cluster

 language: c
 before_install:
- - sudo apt-get install libcap-dev libglib2.0-dev libmhash-dev libkqueue.*
+ - sudo apt-get update
+ - sudo apt-get install libcap-dev libglib2.0-dev libmhash-dev libkqueue.* libcgroup-dev libseccomp-dev linux-libc-dev libkqueue-dev ccache
  - sudo pip install cpp-coveralls --use-mirrors
+ - sudo ln -s /usr/bin/ccache /usr/local/bin/gcc
+ - sudo ln -s /usr/bin/ccache /usr/local/bin/cc
 script:
  - ./.travis.sh
 after_success:

@@ -2,43 +2,88 @@ ACLOCAL_AMFLAGS = -I m4
 SUBDIRS = examples
 
 if CLSYNC
-bin_PROGRAMS = clsync
+bin_PROGRAMS = clsync gencompilerflags
+
+gencompilerflags_SOURCES = gencompilerflags.c
 
 clsync_SOURCES = calc.c cluster.c error.c fileutils.c glibex.c		\
-	indexes.c main.c malloc.c sync.c calc.h cluster.h fileutils.h	\
-	glibex.h main.h port-hacks.h sync.h common.h control.h
+	indexes.c main.c malloc.c rules.c stringex.c sync.c		\
+	posix-hacks.c privileged.c pthreadex.c calc.h cluster.h		\
+	fileutils.h glibex.h main.h port-hacks.h posix-hacks.h		\
+	pthreadex.h stringex.h sync.h common.h control.h privileged.h	\
+	rules.h syscalls.h
 
-clsync_CFLAGS = $(AM_CFLAGS)
+clsync_CFLAGS  = $(AM_CFLAGS)
+clsync_LDFLAGS = $(AM_LDFLAGS)
 
 if HAVE_KQUEUE
-clsync_CFLAGS += -DKQUEUE_SUPPORT
+clsync_CFLAGS  += -DKQUEUE_SUPPORT
 clsync_SOURCES += mon_kqueue.c mon_kqueue.h
 endif
 if HAVE_INOTIFY
-clsync_CFLAGS += -DINOTIFY_SUPPORT
+clsync_CFLAGS  += -DINOTIFY_SUPPORT
 clsync_SOURCES += mon_inotify.c mon_inotify.h
 if INOTIFY_OLD
-clsync_CFLAGS += -DINOTIFY_OLD
+clsync_CFLAGS  += -DINOTIFY_OLD
 endif
 endif
 if HAVE_FANOTIFY
-clsync_CFLAGS += -DFANOTIFY_SUPPORT
+clsync_CFLAGS  += -DFANOTIFY_SUPPORT
 clsync_SOURCES += mon_fanotify.c mon_fanotify.h
 endif
 if HAVE_BSM
-clsync_CFLAGS += -DBSM_SUPPORT
+clsync_CFLAGS  += -DBSM_SUPPORT
 clsync_SOURCES += mon_bsm.c mon_bsm.h
 endif
+if HAVE_GIO
+clsync_CFLAGS  += -DGIO_SUPPORT $(GIO_CFLAGS)
+clsync_LDFLAGS += $(GIO_LIBS)
+clsync_SOURCES += mon_gio.c mon_gio.h
+endif
 if HAVE_DTRACEPIPE
-clsync_CFLAGS += -DDTRACEPIPE_SUPPORT
+clsync_CFLAGS  += -DDTRACEPIPE_SUPPORT
 clsync_SOURCES += mon_dtracepipe.c mon_dtracepipe.h
 endif
+if HAVE_BACKTRACE
+clsync_CFLAGS  += -DBACKTRACE_SUPPORT
+endif
+if HAVE_CAPABILITIES
+clsync_CFLAGS  += -DCAPABILITIES_SUPPORT
+if HAVE_SECCOMP
+clsync_CFLAGS  += -DSECCOMP_SUPPORT
+endif
+endif
+if HAVE_GETMNTENT
+clsync_CFLAGS  += -DGETMNTENT_SUPPORT
+endif
+if HAVE_UNSHARE
+clsync_CFLAGS  += -DUNSHARE_SUPPORT
+if HAVE_PIVOTROOT
+clsync_CFLAGS  += -DPIVOTROOT_OPT_SUPPORT
+endif
+endif
+#if HAVE_TRE
+#clsync_CFLAGS  += -DTRE_SUPPORT
+#endif
+if HAVE_LIBCGROUP
+clsync_CFLAGS  += -DCGROUP_SUPPORT
+clsync_SOURCES += cgroup.c cgroup.h
+endif
+
+if HLLOCKS
+clsync_CFLAGS  += -DHL_LOCKS
+endif
 
 if SOCKET
 clsync_SOURCES += socket.c control.c program.h
 endif
 
-main.o: revision.h
+gencompilerflags_CFLAGS = $(clsync_CFLAGS)
+
+main.o: compilerflags.h
+
+compilerflags.h: gencompilerflags
+	./gencompilerflags > compilerflags.h
 
 dist_man_MANS = man/man1/clsync.1
 endif
@@ -49,7 +94,7 @@ EXTRA_DIST = .doxygen .travis.sh .travis.yml NOTES SHORTHANDS freebsd debian gen
 
 if LIBCLSYNC
 lib_LTLIBRARIES = libclsync.la
-libclsync_la_SOURCES = malloc.c libclsync.c socket.c
+libclsync_la_SOURCES = malloc.c libclsync.c socket.c error.c
 libclsync_la_LDFLAGS = -version-info 0:0:0
 endif
 
@@ -58,7 +103,7 @@ REVISION=$(shell [ -d .git ] &&\
 		'\".'$$(($$(git log 2>/dev/null \
 			| grep -c ^commit \
 			| tr -d "\n" \
-			)-612 \
+			)-994 \
 		))'\"' \
 	) || echo -n '\"-release\"' )
 
@@ -69,16 +114,18 @@ libclsync_includedir = $(includedir)/libclsync
 
 if CLSYNC
 clsync_include_HEADERS = \
-	error.h \
-	malloc.h
-clsync_include_HEADERS += \
 	clsync.h \
+	port-hacks.h \
 	configuration.h \
-	indexes.h
+	ctx.h \
+	error.h \
+	indexes.h \
+	malloc.h \
+	compilerflags.h
+
 if SOCKET
 clsync_include_HEADERS += \
-	socket.h \
-	ctx.h
+	socket.h
 endif
 endif
 if LIBCLSYNC
@@ -93,13 +140,13 @@ endif
 doc:
 	doxygen .doxygen
 
-if SOCKET
+if LIBCLSYNC
 pkgconfig_DATA = pkgconfig/libclsync.pc
 endif
 
-CLEANFILES = revision.h
+CLEANFILES = compilerflags.h gencompilerflags
 if CLSYNC
 CLEANFILES += examples/rules
 clean-local:
-	-rm -rf examples/testdir examples/*.o examples/*.so examples/*.xz doc
+	-rm -rf examples/testdir examples/*.o examples/*.so examples/*.xz doc/doxygen
 endif

@@ -5,19 +5,19 @@ broadcast:	dst_node_id == NOID
 
 session example of two nodes (A and B):
 
-A appears (getting node_id):
+"A" appears (getting node_id):
 
-A -> getmyid	(serial: 0;	src: NOID	; dst: NOID;	name: A)		| cluster_init()
+A -> hello	(serial: 0;	src: NOID	; dst: NOID;	name: A)		| cluster_init()
 A -> register	(serial: 1;	src: 0		; dst: NOID;	name: A)		|
 
 Trying to sync with somebody:
 
 A -> updtree	(serial: 2;	src: 0		; dst: NOID;	[A modtree])
 
-B appears (getting node_id):
+"B" appears (getting node_id):
 
-B -> getmyid	(serial: 0;	src: NOID	; dst: NOID;	name: B)		| cluster_init()
-A -> setid	(serial: 3;	src: 0		; dst: NOID;	name: A; updatets: 100)	
+B -> hello	(serial: 0;	src: NOID	; dst: NOID;	name: B)		| cluster_init()
+A -> welcome	(serial: 3;	src: 0		; dst: NOID;	my_name: A; to_name: B)
 B -> register	(serial: 1;	src: 1		: dst: NOID;	name: B)		|
 A -> ack	(serial: 4;	src: 0		; dst: 1;	ack_serial: 1)
 
@@ -41,11 +41,11 @@ B -> ack	(serial: 6;	src: 1		; dst: 0;	ack_serial: 10)
 A -> unlockall	(serial: 11:	src: 0		; dst: NOID)
 B -> ack	(serial: 7;	src: 1		; dst: 0;	ack_serial: 11)
 
-A disappers (shutdown)
-A appears (registering with old node_id)
+"A" disappers (shutdown)
+"A" appears (registering with old node_id)
 
-A -> getmyid	(serial: 0;	src: NOID	; dst: NOID;	name: A)
-B -> setid	(serial: 4;	src: 1		; dst: 0;	name: A; updatets: 200)
+A -> hello	(serial: 0;	src: NOID	; dst: NOID;	name: A)
+B -> welcome	(serial: 4;	src: 1		; dst: 0;	my_name: B; to_name: A)
 A -> register	(serial: 1;	src: 0		; dst: NOID;	name: A)
 B -> ack	(serial: 5;	src: 1		; dst: 0;	ack_serial: 1)
 

@@ -17,7 +17,10 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "common.h"
+
 #include "calc.h"
+#include "error.h"
 
 #ifdef HAVE_MHASH
 #include <mhash.h>
@@ -36,7 +39,13 @@
 
 // Copied from http://en.wikipedia.org/wiki/Adler-32
 uint32_t adler32_calc(const unsigned char *const data, uint32_t len) { // where data is the location of the data in physical
-                                                           // memory and len is the length of the data in bytes
+                                                                       // memory and len is the length of the data in bytes
+
+	if (len&3)
+		warning("len [%i] & 3 == %i != 0. Wrong length (not a multiple of 4).", len, len&3);
+
+	debug(70, "%p, %i", data, len);
+
 	const int MOD_ADLER = 65521;
 	uint32_t a = 1, b = 0;
 	int32_t index;
@@ -44,6 +53,7 @@ uint32_t adler32_calc(const unsigned char *const data, uint32_t len) { // where
 	// Process each byte of the data in order
 	for (index = 0; index < len; ++index)
 	{
+		debug(80, "%5i: %02x %02x %02x", index, data[index], a, b);
 		a = (a + data[index]) % MOD_ADLER;
 		b = (b + a) % MOD_ADLER;
 	}

@@ -22,8 +22,11 @@
 #ifdef HAVE_MHASH
 #include <mhash.h>
 
+#include "error.h"
+
 static inline uint32_t adler32_calc(const unsigned char *const data, uint32_t len) {
 	uint32_t adler32;
+	debug(70, "%p, %i -> mhash", data, len);
 
 	MHASH td = mhash_init(MHASH_ADLER32);
 	mhash(td, data, len);

+/*
+    clsync - file tree sync utility based on inotify/kqueue/bsm
+
+    Copyright (C) 2014  Dmitry Yu Okunev <dyokunev@ut.mephi.ru> 0x8E30679C
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "common.h"
+#include "error.h"
+#include <libcgroup.h>
+
+static struct cgroup *cgroup = NULL;
+
+int clsync_cgroup_init(ctx_t *ctx_p) {
+	debug(2, "cgroup_name == \"%s\"", ctx_p->cg_groupname);
+
+	SAFE( cgroup_init(),							return -1; );
+	SAFE( (cgroup = cgroup_new_cgroup(ctx_p->cg_groupname)) == NULL,	return -1; );
+
+	return 0;
+}
+
+int clsync_cgroup_forbid_extra_devices() {
+	int rc;
+	char *allowed_devices[] = CG_ALLOWED_DEVICES, **allowed_device_i;
+
+	/*
+	 * Unfortunately, libcgroup doesn't allow multiple values for one key, and cgroups doesn't allow multiple devices for one set. So I was been have to write this hack. It adds character '/' to start of "devices.allow" for every new entry. So libclsync thinks that it's different keys, "/sys/fs/cgroup/devices/clsync/123/devices.allow" == "/sys/fs/cgroup/devices/clsync/123//devices.allow".
+	 */
+
+	char control_name_buf[BUFSIZ+BUFSIZ]={[0 ... BUFSIZ-1] = '/', 'd', 'e', 'v', 'i', 'c', 'e', 's', '.', 'a', 'l', 'l', 'o', 'w'}, *control_name = &control_name_buf[BUFSIZ];
+	debug(2, "");
+
+	struct cgroup_controller *cgc;
+
+	SAFE( (cgc = cgroup_add_controller(cgroup, "devices")) == NULL,	return -1; );
+
+	debug(8, "Deny device: \"a\"");
+	SAFE( cgroup_add_value_string(cgc, "devices.deny", "a"),	return -1; );
+	allowed_device_i = allowed_devices;
+	while (*allowed_device_i != NULL) {
+
+		critical_on (control_name < control_name_buf);
+
+		debug(8, "Allow device: \"%s\" (\"%s\" = \"%s\")", *allowed_device_i, control_name, *allowed_device_i);
+		SAFE( cgroup_add_value_string(cgc, control_name, *allowed_device_i),return -1; );
+		control_name--;
+		allowed_device_i++;
+	}
+
+	if ((rc=cgroup_create_cgroup(cgroup, 1))) {
+		error("Got error while cgroup_create_cgroup(): %s", cgroup_strerror(rc));
+		return -1;
+	}
+
+	return 0;
+}
+
+int clsync_cgroup_attach(ctx_t *ctx_p) {
+	int rc;
+	debug(2, "");
+
+	if ((rc=cgroup_attach_task_pid(cgroup, ctx_p->pid))) {
+		error("Got error while cgroup_attach_task_pid(): %s", cgroup_strerror(rc));
+		return -1;
+	}
+
+	return 0;
+}
+
+int clsync_cgroup_deinit(ctx_t *ctx_p) {
+	debug(2, "");
+
+	error_on(cgroup_delete_cgroup_ext(cgroup, CGFLAG_DELETE_IGNORE_MIGRATION | CGFLAG_DELETE_RECURSIVE));
+	cgroup_free(&cgroup);
+
+	debug(15, "end");
+	return 0;
+}
+

+/*
+    clsync - file tree sync utility based on inotify/kqueue/bsm
+
+    Copyright (C) 2014  Dmitry Yu Okunev <dyokunev@ut.mephi.ru> 0x8E30679C
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+extern int clsync_cgroup_init(struct ctx *ctx_p);
+extern int clsync_cgroup_forbid_extra_devices();
+extern int clsync_cgroup_attach(struct ctx *ctx_p);
+extern int clsync_cgroup_deinit(struct ctx *ctx_p);
+

@@ -24,6 +24,10 @@
 #include <stdint.h>
 #include <sys/types.h>
 
+#ifndef CLSYNC_ITSELF
+#	include <clsync/port-hacks.h>
+#endif
+
 #define CLSYNC_API_VERSION 2
 
 enum eventobjtype {

@@ -68,16 +68,15 @@
 // Types
 
 enum adler32_calc {
-	ADLER32_CALC_NONE		= 0x00,
+	ADLER32_CALC_NONE	= 0x00,
 	ADLER32_CALC_HEADER	= 0x01,
-	ADLER32_CALC_DATA		= 0x02,
-	ADLER32_CALC_ALL		= 0x03,
+	ADLER32_CALC_DATA	= 0x02,
+	ADLER32_CALC_ALL	= 0x03,
 };
 typedef enum adler32_calc adler32_calc_t;
 
 enum cluster_read_flags {
 	CLREAD_NONE		= 0x00,
-	CLREAD_CONTINUE		= 0x01,
 	CLREAD_ALL		= 0xff
 };
 typedef enum cluster_read_flags cluster_read_flags_t;
@@ -112,33 +111,38 @@ struct nodeinfo {
 	packets_stats_t	 packets_in;
 	packets_stats_t	 packets_out;
 	uint32_t	 last_serial;
+	char		 *node_name;
 };
 typedef struct nodeinfo nodeinfo_t;
 
 enum clustercmd_id {
-	CLUSTERCMDID_PING 	= 0,
-	CLUSTERCMDID_ACK 	= 1,
-	CLUSTERCMDID_REG 	= 2,
-	CLUSTERCMDID_GETMYID	= 3,
-	CLUSTERCMDID_SETID	= 4,
-	CLUSTERCMDID_HT_EXCH	= 5,
+	CLUSTERCMDID_PING	= 0,
+	CLUSTERCMDID_ACK	= 1,
+	CLUSTERCMDID_REG	= 2,
+	CLUSTERCMDID_HELLO	= 3,
+	CLUSTERCMDID_WELCOME	= 4,
+	CLUSTERCMDID_DIE	= 5,
+	CLUSTERCMDID_HT_EXCH	= 6,
 	COUNT_CLUSTERCMDID
 };
 typedef enum clustercmd_id clustercmd_id_t;
 
-struct clustercmd_getmyid {
-	char      node_name[1];
+struct clustercmd_hello {
+	char      node_name[0];
 };
-typedef struct clustercmd_getmyid clustercmd_getmyid_t;
-
-struct clustercmd_setiddata {
-	uint32_t  updatets;
-	char      node_name[1];
+typedef struct clustercmd_hello clustercmd_hello_t;
+
+#define    welcome_to_node_name_len(cmd_p) ((cmd_p)->h.data_len-(((clustercmd_welcome_t *)&(cmd_p)->data)->from_node_name_len)-sizeof(clustercmd_welcome_t))
+#define    welcome_to_node_name(cmddata_p) (&cmddata_p->from_node_name[cmddata_p->from_node_name_len])
+struct clustercmd_welcome {
+	size_t    from_node_name_len;
+	char      from_node_name[0];
+//                to_node_name  ==  my_node_name+my_node_name_len
 };
-typedef struct clustercmd_setiddata clustercmd_setiddata_t;
+typedef struct clustercmd_welcome clustercmd_welcome_t;
 
 struct clustercmd_reg {
-	char      node_name[1];
+	char      node_name[0];
 };
 typedef struct clustercmd_reg clustercmd_reg_t;
 
@@ -149,7 +153,7 @@ typedef struct clustercmd_ack clustercmd_ack_t;
 
 enum reject_reason {
 	REJ_UNKNOWN		= 0,
-	REJ_adler32MISMATCH,
+	REJ_ADLER32MISMATCH,
 };
 typedef enum reject_reason reject_reason_t;
 
@@ -162,13 +166,13 @@ typedef struct clustercmd_rej clustercmd_rej_t;
 struct clustercmd_ht_exch {
 	time_t	 ctime;
 	size_t	 path_length;
-	char	 path[1];
+	char	 path[0];
 };
 typedef struct clustercmd_ht_exch clustercmd_ht_exch_t;
 
 struct clustercmdadler32 {
-	uint32_t hdr;
-	uint32_t dat;
+	uint32_t hdr;					// 32
+	uint32_t dat;					// 64
 };
 typedef struct clustercmdadler32 clustercmdadler32_t;
 
@@ -177,23 +181,26 @@ struct clustercmdhdr {					// bits
 	uint8_t			src_node_id;		// 16
 	uint8_t			flags;			// 24	(for future compatibility)
 	uint8_t			cmd_id;			// 32
-	clustercmdadler32_t	adler32;		// 64
-	uint32_t		data_len;		// 96
-	uint32_t		ts;			// 128
-	uint32_t		serial;			// 160
+	clustercmdadler32_t	adler32;		// 96
+	uint32_t		data_len;		// 128
+	uint32_t		ts;			// 160
+	uint32_t		serial;			// 192
 };
 typedef struct clustercmdhdr clustercmdhdr_t;
 
+typedef char clustercmd_die_t;
+
 struct clustercmd {
 	clustercmdhdr_t h;
 	union data {
-		char 			p[1];
-		clustercmd_setiddata_t	setid;
+		char 			p[0];
+		clustercmd_welcome_t	welcome;
 		clustercmd_reg_t	reg;
 		clustercmd_ack_t	ack;
 		clustercmd_rej_t	rej;
-		clustercmd_getmyid_t	getmyid;
+		clustercmd_hello_t	hello;
 		clustercmd_ht_exch_t	ht_exch;
+		clustercmd_die_t	die;
 	} data;
 };
 typedef struct clustercmd clustercmd_t;

@@ -67,12 +67,8 @@
 #include <netinet/in.h>
 #include <libgen.h>
 #include <pthread.h>
-#include <glib.h>
 
-#ifdef HAVE_CAPABILITIES
-#	include <sys/capability.h>	// for capset()/capget() for --preserve-file-access
-#	include <sys/prctl.h>		// for prctl() for --preserve-fil-access
-#endif
+#define CLSYNC_ITSELF
 
 #include "configuration.h"
 #ifdef HAVE_CONFIG_H
@@ -80,6 +76,8 @@
 #endif
 
 #include "clsync.h"
+#include "port-hacks.h"
+#include "posix-hacks.h"
 #include "ctx.h"
 #include "program.h"
 
@@ -117,20 +115,46 @@
 #	endif
 #endif
 
+#ifndef offsetof
+#	define offsetof(a, b) __builtin_offsetof(a, b)
+#endif
+
+// clang defines "__GNUC__", but not compatible with gnuc. Fixing.
+#ifdef __clang__
+#	ifdef __GNUC__
+#		undef __GNUC__
+#	endif
+#endif
 
 #define TOSTR(a) # a
 #define XTOSTR(a) TOSTR(a)
 
 #define COLLECTDELAY_INSTANT ((unsigned int)~0)
 
+
+#define MSG_SECURITY_PROBLEM(a) "Security problem: "a". Don't use this application until the bug will be fixed. Report about the problem to: "AUTHOR
+
 #define require_strlen_le(str, limit) \
 	if (strlen(str) >= limit)\
 		critical("length of "TOSTR(str)" (\"%s\") >= "TOSTR(limit));\
 
+#define SAFE(code, onfail) ({\
+	long _SAFE_rc;\
+	if ((_SAFE_rc = code)) {\
+		error("Got error while "TOSTR(code));\
+		onfail;\
+	} \
+	_SAFE_rc;\
+})
+
 enum paramsource_enum {
 	PS_UNKNOWN	 = 0,
 	PS_ARGUMENT,
-	PS_CONFIG
+	PS_CONFIG,
+	PS_CONTROL,
+	PS_DEFAULTS,
+//	PS_REHASH,
+	PS_CORRECTION,
 };
 typedef enum paramsource_enum paramsource_t;
 
@@ -141,23 +165,12 @@ enum notifyengine_enum {
 	NE_INOTIFY,
 	NE_KQUEUE,
 	NE_BSM,
+	NE_BSM_PREFETCH,
 	NE_DTRACEPIPE,
+	NE_GIO,
 };
 typedef enum notifyengine_enum notifyengine_t;
 
-#define STATE_STARTING(state_p) (state_p == NULL)
-enum state_enum {
-	STATE_EXIT 	= 0,
-	STATE_STARTING,
-	STATE_RUNNING,
-	STATE_REHASH,
-	STATE_TERM,
-	STATE_THREAD_GC,
-	STATE_INITSYNC,
-	STATE_UNKNOWN
-};
-typedef enum state_enum state_t;
-
 enum threadingmode {
 	PM_OFF	= 0,
 	PM_SAFE,
@@ -165,6 +178,13 @@ enum threadingmode {
 };
 typedef enum threadingmode threadingmode_t;
 
+enum splittingmode_enum {
+	SM_OFF		= 0,
+	SM_THREAD,
+	SM_PROCESS,
+};
+typedef enum splittingmode_enum splittingmode_t;
+
 /*
 struct excludeinfo {
 	unsigned int	seqid_min;
@@ -187,32 +207,6 @@ struct eventinfo {
 };
 typedef struct eventinfo eventinfo_t;
 
-
-typedef int (*thread_callbackfunct_t)(ctx_t *ctx_p, char **argv);
-struct threadinfo {
-	int			  thread_num;
-	uint32_t		  iteration;
-	thread_callbackfunct_t 	  callback;
-	char 			**argv;
-	pthread_t		  pthread;
-	int			  exitcode;
-	int			  errcode;
-	state_t			  state;
-	ctx_t			 *ctx_p;
-	time_t			  starttime;
-	time_t			  expiretime;
-	int			  child_pid;
-
-	GHashTable		 *fpath2ei_ht;		// file path -> event information
-
-	int			  try_n;
-
-	// for so-synchandler
-	int			  n;
-	api_eventinfo_t		 *ei;
-};
-typedef struct threadinfo threadinfo_t;
-
 enum pthread_mutex_id {
 	PTHREAD_MUTEX_STATE,
 	PTHREAD_MUTEX_SELECT,
@@ -221,18 +215,6 @@ enum pthread_mutex_id {
 };