/* * Routines used by the file-transfer code. * * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2003-2022 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License along * with this program; if not, visit the http://fsf.org website. */ #include "rsync.h"#include "itypes.h"#include <zlib.h>#ifdef SUPPORT_ZSTD#include <zstd.h>#endif#ifdef SUPPORT_LZ4#include <lz4.h>#endif extern int do_compression;extern int protocol_version;extern int module_id;extern int do_compression_level;extern int do_compression_threads;extern char *skip_compress; #ifndef Z_INSERT_ONLY#define Z_INSERT_ONLY Z_SYNC_FLUSH#endif static int skip_compression_level; /* The least possible compressing for handling skip-compress files. */static int per_file_default_level; /* The default level that each new file gets prior to checking its suffix. */ struct suffix_tree {	struct suffix_tree *sibling;	struct suffix_tree *child;	char letter, word_end;}; static char *match_list;static struct suffix_tree *suftree; void init_compression_level(void){	int min_level, max_level, def_level, off_level; 	switch (do_compression) {	case CPRES_NONE:		return;	case CPRES_ZLIB:	case CPRES_ZLIBX:		min_level = 1;		max_level = Z_BEST_COMPRESSION;		def_level = 6; /* Z_DEFAULT_COMPRESSION is -1, so set it to the real default */		off_level = skip_compression_level = Z_NO_COMPRESSION;		if (do_compression_level == Z_DEFAULT_COMPRESSION)			do_compression_level = def_level;		break;#ifdef SUPPORT_ZSTD	case CPRES_ZSTD:		min_level = skip_compression_level = ZSTD_minCLevel();		max_level = ZSTD_maxCLevel();		def_level = ZSTD_CLEVEL_DEFAULT;		off_level = CLVL_NOT_SPECIFIED;		if (do_compression_level == 0)			do_compression_level = def_level;		break;#endif#ifdef SUPPORT_LZ4	case CPRES_LZ4:		min_level = skip_compression_level = 0;		max_level = 0;		def_level = 0;		off_level = CLVL_NOT_SPECIFIED;		break;#endif	default: /* paranoia to prevent missing case values */		NOISY_DEATH("Unknown do_compression value");	} 	if (do_compression_level == CLVL_NOT_SPECIFIED)		do_compression_level = def_level;	else if (do_compression_level == off_level) {		do_compression = CPRES_NONE;		return;	} 	/* We don't bother with any errors or warnings -- just make sure that the values are valid. */	if (do_compression_level < min_level)		do_compression_level = min_level;	else if (do_compression_level > max_level)		do_compression_level = max_level;} static void add_suffix(struct suffix_tree **prior, char ltr, const char *str){	struct suffix_tree *node, *newnode; 	if (ltr == '[') {		const char *after = strchr(str, ']');		/* Treat "[foo" and "[]" as having a literal '['. */		if (after && after++ != str+1) {			while ((ltr = *str++) != ']')				add_suffix(prior, ltr, after);			return;		}	} 	for (node = *prior; node; prior = &node->sibling, node = node->sibling) {		if (node->letter == ltr) {			if (*str)				add_suffix(&node->child, *str, str+1);			else				node->word_end = 1;			return;		}		if (node->letter > ltr)			break;	}	newnode = new(struct suffix_tree);	newnode->sibling = node;	newnode->child = NULL;	newnode->letter = ltr;	*prior = newnode;	if (*str) {		add_suffix(&newnode->child, *str, str+1);		newnode->word_end = 0;	} else		newnode->word_end = 1;} static void add_nocompress_suffixes(const char *str){	char *buf, *t;	const char *f = str; 	buf = new_array(char, strlen(f) + 1); 	while (*f) {		if (*f == '/') {			f++;			continue;		} 		t = buf;		do {			if (isUpper(f))				*t++ = toLower(f);			else				*t++ = *f;		} while (*++f != '/' && *f);		*t++ = '\0'; 		/* add_suffix() recurses once per character; an attacker-supplied		 * --skip-compress (from our own argv OR propagated to a daemon		 * receiver via server_options()) with one very long token		 * exhausts the stack.  Real file extensions are short -- cap		 * the recursion depth by truncating absurd suffixes. */		if (t - buf > 32 + 1)			buf[32] = '\0';		add_suffix(&suftree, *buf, buf+1);	} 	free(buf);} static void init_set_compression(void){	const char *f;	char *t, *start; 	if (skip_compress)		add_nocompress_suffixes(skip_compress); 	/* A non-daemon transfer skips the default suffix list if the	 * user specified --skip-compress. */	if (skip_compress && module_id < 0)		f = "";	else		f = lp_dont_compress(module_id); 	match_list = t = new_array(char, strlen(f) + 2); 	per_file_default_level = do_compression_level; 	while (*f) {		if (*f == ' ') {			f++;			continue;		} 		start = t;		do {			if (isUpper(f))				*t++ = toLower(f);			else				*t++ = *f;		} while (*++f != ' ' && *f);		*t++ = '\0'; 		if (t - start == 1+1 && *start == '*') {			/* Optimize a match-string of "*". */			*match_list = '\0';			suftree = NULL;			per_file_default_level = skip_compression_level;			break;		} 		/* Move *.foo items into the stuffix tree. */		if (*start == '*' && start[1] == '.' && start[2]		 && !strpbrk(start+2, ".?*")) {			add_suffix(&suftree, start[2], start+3);			t = start;		}	}	*t++ = '\0';} /* determine the compression level based on a wildcard filename list */void set_compression(const char *fname){#if 0 /* No compression algorithms currently allow mid-stream changing of the level. */	const struct suffix_tree *node;	const char *s;	char ltr;#endif 	if (!do_compression)		return; 	if (!match_list)		init_set_compression(); #if 0	compression_level = per_file_default_level; 	if (!*match_list && !suftree)		return; 	if ((s = strrchr(fname, '/')) != NULL)		fname = s + 1; 	for (s = match_list; *s; s += strlen(s) + 1) {		if (iwildmatch(s, fname)) {			compression_level = skip_compression_level;			return;		}	} 	if (!(node = suftree) || !(s = strrchr(fname, '.'))	 || s == fname || !(ltr = *++s))		return; 	while (1) {		if (isUpper(&ltr))			ltr = toLower(&ltr);		while (node->letter != ltr) {			if (node->letter > ltr)				return;			if (!(node = node->sibling))				return;		}		if ((ltr = *++s) == '\0') {			if (node->word_end)				compression_level = skip_compression_level;			return;		}		if (!(node = node->child))			return;	}#else	(void)fname;#endif} /* non-compressing recv token */static int32 simple_recv_token(int f, char **data){	static int32 residue;	static char *buf;	int32 n; 	if (!buf)		buf = new_array(char, CHUNK_SIZE); 	if (residue == 0) {		int32 i = read_int(f);		if (i <= 0)			return i;		/* A literal run may exceed CHUNK_SIZE: some peers (e.g. the		 * acrosync library) use a 64k block size.  The loop below reads		 * the run CHUNK_SIZE bytes at a time, so read_buf never writes		 * past the static CHUNK_SIZE buffer regardless of i. */		residue = i;	} 	*data = buf;	n = MIN(CHUNK_SIZE,residue);	residue -= n;	read_buf(f,buf,n);	return n;} /* non-compressing send token */static void simple_send_token(int f, int32 token, struct map_struct *buf, OFF_T offset, int32 n){	if (n > 0) {		int32 len = 0;		while (len < n) {			int32 n1 = MIN(CHUNK_SIZE, n-len);			write_int(f, n1);			write_buf(f, map_ptr(buf, offset+len, n1), n1);			len += n1;		}	}	/* a -2 token means to send data only and no token */	if (token != -2)		write_int(f, -(token+1));} /* Flag bytes in compressed stream are encoded as follows: */#define END_FLAG	0	/* that's all folks */#define TOKEN_LONG	0x20	/* followed by 32-bit token number */#define TOKENRUN_LONG	0x21	/* ditto with 16-bit run count */#define DEFLATED_DATA	0x40	/* + 6-bit high len, then low len byte */#define TOKEN_REL	0x80	/* + 6-bit relative token number */#define TOKENRUN_REL	0xc0	/* ditto with 16-bit run count */ #define MAX_DATA_COUNT	16383	/* fit 14 bit count into 2 bytes with flags */ /* zlib.h says that if we want to be able to compress something in a single * call, avail_out must be at least 0.1% larger than avail_in plus 12 bytes. * We'll add in 0.1%+16, just to be safe (and we'll avoid floating point, * to ensure that this is a compile-time value). */#define AVAIL_OUT_SIZE(avail_in_size) ((avail_in_size)*1001/1000+16) /* For coding runs of tokens */static int32 last_token = -1;static int32 run_start;static int32 last_run_end; /* Deflation state */static z_stream tx_strm; /* Output buffer */static char *obuf; /* We want obuf to be able to hold both MAX_DATA_COUNT+2 bytes as well as * AVAIL_OUT_SIZE(CHUNK_SIZE) bytes, so make sure that it's large enough. */#if MAX_DATA_COUNT+2 > AVAIL_OUT_SIZE(CHUNK_SIZE)#define OBUF_SIZE	(MAX_DATA_COUNT+2)#else#define OBUF_SIZE	AVAIL_OUT_SIZE(CHUNK_SIZE)#endif /* Send a deflated token */static voidsend_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, int32 nb, int32 toklen){	static int init_done, flush_pending;	int32 n, r; 	if (last_token == -1) {		/* initialization */		if (!init_done) {			tx_strm.next_in = NULL;			tx_strm.zalloc = NULL;			tx_strm.zfree = NULL;			if (deflateInit2(&tx_strm, per_file_default_level,					 Z_DEFLATED, -15, 8,					 Z_DEFAULT_STRATEGY) != Z_OK) {				rprintf(FERROR, "compression init failed\n");				exit_cleanup(RERR_PROTOCOL);			}			obuf = new_array(char, OBUF_SIZE);			init_done = 1;		} else			deflateReset(&tx_strm);		last_run_end = 0;		run_start = token;		flush_pending = 0;	} else if (last_token == -2) {		run_start = token;	} else if (nb != 0 || token != last_token + 1 || token >= run_start + 65536) {		/* output previous run */		r = run_start - last_run_end;		n = last_token - run_start;		if (r >= 0 && r <= 63) {			write_byte(f, (n==0? TOKEN_REL: TOKENRUN_REL) + r);		} else {			write_byte(f, (n==0? TOKEN_LONG: TOKENRUN_LONG));			write_int(f, run_start);		}		if (n != 0) {			write_byte(f, n);			write_byte(f, n >> 8);		}		last_run_end = last_token;		run_start = token;	} 	last_token = token; 	if (nb != 0 || flush_pending) {		/* deflate the data starting at offset */		int flush = Z_NO_FLUSH;		tx_strm.avail_in = 0;		tx_strm.avail_out = 0;		do {			if (tx_strm.avail_in == 0 && nb != 0) {				/* give it some more input */				n = MIN(nb, CHUNK_SIZE);				tx_strm.next_in = (Bytef *)					map_ptr(buf, offset, n);				tx_strm.avail_in = n;				nb -= n;				offset += n;			}			if (tx_strm.avail_out == 0) {				tx_strm.next_out = (Bytef *)(obuf + 2);				tx_strm.avail_out = MAX_DATA_COUNT;				if (flush != Z_NO_FLUSH) {					/*					 * We left the last 4 bytes in the					 * buffer, in case they are the					 * last 4.  Move them to the front.					 */					memcpy(tx_strm.next_out, obuf+MAX_DATA_COUNT-2, 4);					tx_strm.next_out += 4;					tx_strm.avail_out -= 4;				}			}			if (nb == 0 && token != -2)				flush = Z_SYNC_FLUSH;			r = deflate(&tx_strm, flush);			if (r != Z_OK) {				rprintf(FERROR, "deflate returned %d\n", r);				exit_cleanup(RERR_STREAMIO);			}			if (nb == 0 || tx_strm.avail_out == 0) {				n = MAX_DATA_COUNT - tx_strm.avail_out;				if (flush != Z_NO_FLUSH) {					/*					 * We have to trim off the last 4					 * bytes of output when flushing					 * (they are just 0, 0, ff, ff).					 */					n -= 4;				}				if (n > 0) {					obuf[0] = DEFLATED_DATA + (n >> 8);					obuf[1] = n;					write_buf(f, obuf, n+2);				}			}		} while (nb != 0 || tx_strm.avail_out == 0);		flush_pending = token == -2;	} 	if (token == -1) {		/* end of file - clean up */		write_byte(f, END_FLAG);	} else if (token != -2 && do_compression == CPRES_ZLIB) {		/* Add the data in the current block to the compressor's		 * history and hash table. */		do {			/* Break up long sections in the same way that			 * see_deflate_token() does. */			int32 n1 = toklen > 0xffff ? 0xffff : toklen;			toklen -= n1;			tx_strm.next_in = (Bytef *)map_ptr(buf, offset, n1);			tx_strm.avail_in = n1;			if (protocol_version >= 31) /* Newer protocols avoid a data-duplicating bug */				offset += n1;			/* With our bundled zlib's Z_INSERT_ONLY this produces no			 * output and consumes the input in one call.  A build			 * against a system zlib lacks Z_INSERT_ONLY and falls back			 * to Z_SYNC_FLUSH (see top of file), which emits a flush			 * block we discard -- and for an incompressible token that			 * block can exceed obuf.  Loop, resetting the output buffer,			 * until all the input is consumed so a large token can't			 * overflow obuf and abort the transfer (#951).  Drain until			 * avail_out != 0 too: a full output buffer can leave pending			 * bytes that would otherwise leak into the next real deflate			 * send and corrupt the stream (same condition as the data loop			 * above).  The discarded output is not sent: the receiver			 * rebuilds the matching history itself in see_deflate_token(). */			do {

Lines 1–500 of 1146. Load more