From 0cdb6aeccd29f82ab5128a9cac2172ffd3d8386b Mon Sep 17 00:00:00 2001 From: "josh.macdonald" Date: Sun, 21 Feb 2010 23:03:48 +0000 Subject: Fix the handling of external source decompression, which changed in 3.0w and was inadequately tested. Mark version 3.0y. --- xdelta3/Makefile | 4 +- xdelta3/Makefile.mingw | 2 +- xdelta3/setup.py | 2 +- xdelta3/xdelta3-main.h | 108 ++++++++++++++++++++++++++++--------------------- xdelta3/xdelta3-test.h | 81 +++++++++++++++++++++++-------------- 5 files changed, 116 insertions(+), 81 deletions(-) (limited to 'xdelta3') diff --git a/xdelta3/Makefile b/xdelta3/Makefile index 62a4adf..398c255 100644 --- a/xdelta3/Makefile +++ b/xdelta3/Makefile @@ -57,7 +57,7 @@ CFLAGS= -Wall -Wshadow -fno-builtin WFLAGS= -Wextra -Wsign-compare -Wconversion -Wextra -Wno-unused-parameter # $Format: "REL=$Xdelta3Version$" $ -REL=3.0x +REL=3.0y RELDIR = xdelta$(REL) @@ -72,7 +72,7 @@ EXTRA = Makefile COPYING linkxd3lib.c badcopy.c xdelta3.swig \ testing/cmp.h testing/delta.h testing/file.h \ testing/modify.h testing/random.h testing/segment.h \ testing/sizes.h testing/test.h testing/Makefile \ - testing/regtest.cc README readme.txt + testing/regtest.cc README readme.txt xdelta3.1 SWIG_FLAGS = -DXD3_DEBUG=1 \ -DEXTERNAL_COMPRESSION=0 \ diff --git a/xdelta3/Makefile.mingw b/xdelta3/Makefile.mingw index e775ec6..22877b3 100644 --- a/xdelta3/Makefile.mingw +++ b/xdelta3/Makefile.mingw @@ -55,7 +55,7 @@ CFLAGS= -Wall -Wshadow -fno-builtin XDWINFLAGS:=-DXD3_STDIO=0 -DXD3_POSIX=0 -DXD3_WIN32=1 -DEXTERNAL_COMPRESSION=0 # $Format: "REL=$Xdelta3Version$" $ -REL=3.0x +REL=3.0y RELDIR = xdelta$(REL) diff --git a/xdelta3/setup.py b/xdelta3/setup.py index 218bb4e..53722bb 100644 --- a/xdelta3/setup.py +++ b/xdelta3/setup.py @@ -50,7 +50,7 @@ xdelta3_ext = Extension('xdelta3main', ]) # $Format: "REL='$Xdelta3Version$'" $ -REL='3.0x' +REL='3.0y' # This provides xdelta3.main(), which calls the xdelta3 command-line main() # from python. diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h index 56c283a..19def5d 100644 --- a/xdelta3/xdelta3-main.h +++ b/xdelta3/xdelta3-main.h @@ -150,9 +150,9 @@ static STARTUPINFO winStartupInfo; * inputs to the Xdelta encoder. */ typedef enum { - RD_FIRST = (1 << 0), - RD_NONEXTERNAL = (1 << 1), - RD_EXTERNAL_V1 = (1 << 2), + RD_FIRST = (1 << 0), + RD_NONEXTERNAL = (1 << 1), + RD_DECOMPSET = (1 << 2), } xd3_read_flags; /* main_file->mode values */ @@ -388,7 +388,7 @@ static int main_version (void) { /* $Format: " DP(RINT \"Xdelta version $Xdelta3Version$, Copyright (C) 2007, 2008, 2009, 2010, Joshua MacDonald\n\");" $ */ - DP(RINT "Xdelta version 3.0x, Copyright (C) 2007, 2008, 2009, 2010, Joshua MacDonald\n"); + DP(RINT "Xdelta version 3.0y, Copyright (C) 2007, 2008, 2009, 2010, Joshua MacDonald\n"); DP(RINT "Xdelta comes with ABSOLUTELY NO WARRANTY.\n"); DP(RINT "This is free software, and you are welcome to redistribute it\n"); DP(RINT "under certain conditions; see \"COPYING\" for details.\n"); @@ -1923,7 +1923,7 @@ main_merge_output (xd3_stream *stream, main_file *ofile) (stream->dec_win_ind & VCD_ADLER32) != 0) { recode_stream->flags |= XD3_ADLER32_RECODE; - recode_stream->recode_adler32 = + recode_stream->recode_adler32 = stream->whole_target.wininfo[window_num].adler32; } @@ -2206,14 +2206,14 @@ main_pipe_copier (uint8_t *pipe_buf, * the decompression command, the second process copies data to the * input of the first. */ static int -main_input_decompress_setup (const main_extcomp *decomp, - main_file *ifile, - uint8_t *input_buf, - usize_t input_bufsize, - uint8_t *pipe_buf, - usize_t pipe_bufsize, - usize_t pipe_avail, - usize_t *nread) +main_input_decompress_setup (const main_extcomp *decomp, + main_file *ifile, + uint8_t *input_buf, + usize_t input_bufsize, + uint8_t *pipe_buf, + usize_t pipe_bufsize, + usize_t pipe_avail, + usize_t *nread) { /* The two pipes: input and output file descriptors. */ int outpipefd[2], inpipefd[2]; @@ -2352,6 +2352,7 @@ main_secondary_decompress_check (main_file *file, usize_t try_read = min (input_size, XD3_ALLOCSIZE); usize_t check_nread; uint8_t check_buf[XD3_ALLOCSIZE]; /* TODO: stack limit */ + const main_extcomp *decompressor = NULL; if ((ret = main_file_read (file, check_buf, try_read, @@ -2360,25 +2361,49 @@ main_secondary_decompress_check (main_file *file, return ret; } - for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) + if (file->flags & RD_DECOMPSET) { - const main_extcomp *decomp = & extcomp_types[i]; - - if ((check_nread > decomp->magic_size) && - /* The following expr skips decompression if we are trying - * to read a VCDIFF input and that is the magic number. */ - !((decomp->flags & RD_NONEXTERNAL) && - (file->flags & RD_NONEXTERNAL)) && - memcmp (check_buf, decomp->magic, decomp->magic_size) == 0) + /* This allows the application header to override the magic + * number, for whatever reason. */ + decompressor = file->compressor; + } + else + { + for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) { - file->size_known = 0; - return main_input_decompress_setup (decomp, file, - input_buf, input_size, - check_buf, XD3_ALLOCSIZE, - check_nread, nread); + const main_extcomp *decomp = & extcomp_types[i]; + + if (check_nread > decomp->magic_size) + { + /* The following expr checks if we are trying to read a + * VCDIFF input, in which case do not treat it as + * "secondary" decompression. */ + int skip_this_type = (decomp->flags & RD_NONEXTERNAL) && + (file->flags & RD_NONEXTERNAL); + + if (skip_this_type) + { + continue; + } + + if (memcmp (check_buf, decomp->magic, decomp->magic_size) == 0) + { + decompressor = decomp; + break; + } + } } } + if (decompressor != NULL) + { + file->size_known = 0; + return main_input_decompress_setup (decompressor, file, + input_buf, input_size, + check_buf, XD3_ALLOCSIZE, + check_nread, nread); + } + /* Now read the rest of the input block. */ (*nread) = 0; @@ -2655,6 +2680,7 @@ main_get_appheader_params (main_file *file, char **parsed, /* Set the compressor, initiate de/recompression later. */ if (file->compressor == NULL && *parsed[1] != 0) { + file->flags |= RD_DECOMPSET; file->compressor = main_get_compressor (parsed[1]); } } @@ -2723,22 +2749,13 @@ static int main_read_primary_input (main_file *file, uint8_t *buf, usize_t size, - usize_t *nread, - int is_source) + usize_t *nread) { #if EXTERNAL_COMPRESSION if (option_decompress_inputs && file->flags & RD_FIRST) { file->flags &= ~RD_FIRST; - - if (is_source && file->compressor == NULL) - { - /* Application header overrides magic number. */ - } - else - { - return main_secondary_decompress_check (file, buf, size, nread); - } + return main_secondary_decompress_check (file, buf, size, nread); } #endif @@ -2879,7 +2896,7 @@ main_set_source (xd3_stream *stream, xd3_cmd cmd, /* We have to read the first block into the cache now, because * size_known can still change (due to secondary - * decompression). Calls main_decompress_input_check() via + * decompression). Calls main_secondary_decompress_check() via * main_read_primary_input(). */ /* TODO(jmacd): This is a huge hack! Fix me. */ lru_size = 1; @@ -3051,7 +3068,7 @@ main_get_winsize (main_file *ifile) { *******************************************************************/ static int -main_getblk_lru (xd3_source *source, xoff_t blkno, +main_getblk_lru (xd3_source *source, xoff_t blkno, main_blklru** blrup, int *is_new) { main_blklru *blru = NULL; @@ -3178,7 +3195,7 @@ main_read_seek_source (xd3_stream *stream, xoff_t skip_blkno; usize_t skip_offset; - xd3_blksize_div (sfile->source_position, source, + xd3_blksize_div (sfile->source_position, source, &skip_blkno, &skip_offset); /* Read past unused data */ @@ -3196,8 +3213,7 @@ main_read_seek_source (xd3_stream *stream, if ((ret = main_read_primary_input (sfile, (uint8_t*) blru->blk, source->blksize, - & nread, - 1 /* source */))) + & nread))) { return ret; } @@ -3295,8 +3311,7 @@ main_getblk_func (xd3_stream *stream, if ((ret = main_read_primary_input (sfile, (uint8_t*) blru->blk, source->blksize, - & nread, - 1 /* source */))) + & nread))) { return ret; } @@ -3564,8 +3579,7 @@ main_input (xd3_cmd cmd, try_read = (usize_t) min ((xoff_t) config.winsize, input_remain); if ((ret = main_read_primary_input (ifile, main_bdata, - try_read, & nread, - 0 /* !source */))) + try_read, & nread))) { return EXIT_FAILURE; } diff --git a/xdelta3/xdelta3-test.h b/xdelta3/xdelta3-test.h index 21d7283..9050554 100644 --- a/xdelta3/xdelta3-test.h +++ b/xdelta3/xdelta3-test.h @@ -43,7 +43,7 @@ void mt_init(mtrand *mt, uint32_t seed) { /* In the previous versions, MSBs of the seed affect */ /* only MSBs of the array mt[]. */ /* 2002/01/09 modified by Makoto Matsumoto */ - mt->mt_buffer_[i] = + mt->mt_buffer_[i] = (1812433253UL * (mt->mt_buffer_[i-1] ^ (mt->mt_buffer_[i-1] >> 30)) + i); } } @@ -70,14 +70,14 @@ uint32_t mt_random (mtrand *mt) { mt->mt_buffer_[MT_LEN - 1] = mt->mt_buffer_[MT_IA - 1] ^ (y >> 1) ^ mag01[y & 0x1UL]; mt->mt_index_ = 0; } - + y = mt->mt_buffer_[mt->mt_index_++]; - + y ^= (y >> 11); y ^= (y << 7) & 0x9d2c5680UL; y ^= (y << 15) & 0xefc60000UL; y ^= (y >> 18); - + return y; } @@ -89,7 +89,7 @@ static uint32_t mt_exp_rand (uint32_t mean, uint32_t max_value) { double mean_d = mean; - double erand = log (1.0 / (mt_random (&static_mtrand) / + double erand = log (1.0 / (mt_random (&static_mtrand) / (double)UINT32_MAX)); uint32_t x = (uint32_t) (mean_d * erand + 0.5); @@ -1961,7 +1961,7 @@ test_recode_command2 (xd3_stream *stream, int has_source, /* Now decode */ sprintf (dcmd, "%s -fd ", program_name); - + if (has_source) { strcat (dcmd, "-s "); @@ -1987,14 +1987,14 @@ test_recode_command2 (xd3_stream *stream, int has_source, } return 0; -} +} static int test_recode_command (xd3_stream *stream, int ignore) { /* Things to test: * - with and without a source file (recode does not change) - * + * * (recode may or may not change -- 8 variations) * - with and without adler32 * - with and without app header @@ -2146,6 +2146,7 @@ test_source_decompression (xd3_stream *stream, int ignore) int ret; char buf[TESTBUFSIZE]; const main_extcomp *ext; + xoff_t dsize; mt_init (& static_mtrand, 0x9f73f7fc); @@ -2153,53 +2154,73 @@ test_source_decompression (xd3_stream *stream, int ignore) if ((ret = test_make_inputs (stream, NULL, NULL))) { return ret; } /* Use gzip. */ - if ((ext = main_get_compressor ("G")) == NULL) { DP(RINT "skipped"); return 0; } + if ((ext = main_get_compressor ("G")) == NULL) + { + DP(RINT "skipped"); + return 0; + } /* Save an uncompressed copy. */ if ((ret = test_save_copy (TEST_TARGET_FILE))) { return ret; } - /* Compress the target. */ - sprintf (buf, "%s %s < %s > %s", ext->recomp_cmdname, - ext->recomp_options, TEST_TARGET_FILE, TEST_SOURCE_FILE); + /* Compress the source. */ + sprintf (buf, "%s -1 %s < %s > %s", ext->recomp_cmdname, + ext->recomp_options, TEST_COPY_FILE, TEST_SOURCE_FILE); if ((ret = do_cmd (stream, buf))) { return ret; } - - /* Copy back to the source. */ - sprintf (buf, "cp -f %s %s", TEST_SOURCE_FILE, TEST_TARGET_FILE); + /* Compress the target. */ + sprintf (buf, "%s -9 %s < %s > %s", ext->recomp_cmdname, + ext->recomp_options, TEST_COPY_FILE, TEST_TARGET_FILE); if ((ret = do_cmd (stream, buf))) { return ret; } /* Now the two identical files are compressed. Delta-encode the target, * with decompression. */ - sprintf (buf, "%s -v -eq -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_TARGET_FILE, TEST_DELTA_FILE); + sprintf (buf, "%s -e -A -vfq -s%s %s %s", program_name, TEST_SOURCE_FILE, + TEST_TARGET_FILE, TEST_DELTA_FILE); if ((ret = do_cmd (stream, buf))) { return ret; } + /* Check that the compressed file is small (b/c inputs are + * identical). */ + if ((ret = test_file_size (TEST_DELTA_FILE, & dsize))) { return ret; } + /* Deltas for identical files should be very small. */ + if (dsize > 200) + { + DP(RINT "external compression did not happen\n"); + stream->msg = "external compression did not happen"; + return XD3_INTERNAL; + } + /* Decode the delta file with recompression disabled, should get an * uncompressed file out. */ - sprintf (buf, "%s -v -dq -R -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE); + sprintf (buf, "%s -v -dq -R -s%s %s %s", program_name, + TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE); if ((ret = do_cmd (stream, buf))) { return ret; } - if ((ret = compare_files (stream, TEST_COPY_FILE, TEST_RECON_FILE))) { return ret; } + if ((ret = compare_files (stream, TEST_COPY_FILE, + TEST_RECON_FILE))) { return ret; } /* Decode the delta file with recompression, should get a compressed file * out. But we can't compare compressed files directly. */ - sprintf (buf, "%s -v -dqf -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE); + sprintf (buf, "%s -v -dqf -s%s %s %s", program_name, + TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE); if ((ret = do_cmd (stream, buf))) { return ret; } - sprintf (buf, "%s %s < %s > %s", ext->decomp_cmdname, ext->decomp_options, TEST_RECON_FILE, TEST_RECON2_FILE); + sprintf (buf, "%s %s < %s > %s", ext->decomp_cmdname, ext->decomp_options, + TEST_RECON_FILE, TEST_RECON2_FILE); if ((ret = do_cmd (stream, buf))) { return ret; } - if ((ret = compare_files (stream, TEST_COPY_FILE, TEST_RECON2_FILE))) { return ret; } + if ((ret = compare_files (stream, TEST_COPY_FILE, + TEST_RECON2_FILE))) { return ret; } /* Encode with decompression disabled */ - sprintf (buf, "%s -v -feqD -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_TARGET_FILE, TEST_DELTA_FILE); + sprintf (buf, "%s -e -D -vfq -s%s %s %s", program_name, + TEST_SOURCE_FILE, TEST_TARGET_FILE, TEST_DELTA_FILE); if ((ret = do_cmd (stream, buf))) { return ret; } - /* Decode the delta file with recompression enabled, it doesn't matter, - * should get the compressed file out. */ - sprintf (buf, "%s -v -fdq -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE); + /* Decode the delta file with decompression disabled, should get the + * identical compressed file out. */ + sprintf (buf, "%s -d -D -vfq -s%s %s %s", program_name, + TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE); if ((ret = do_cmd (stream, buf))) { return ret; } - if ((ret = compare_files (stream, TEST_TARGET_FILE, TEST_RECON_FILE))) { return ret; } + if ((ret = compare_files (stream, TEST_TARGET_FILE, + TEST_RECON_FILE))) { return ret; } - /* Try again with recompression disabled, it doesn't make a difference. */ - sprintf (buf, "%s -v -fqRd -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - if ((ret = compare_files (stream, TEST_TARGET_FILE, TEST_RECON_FILE))) { return ret; } test_cleanup(); return 0; } -- cgit v1.2.3