Fix the handling of external source decompression, which changed in 3.0w

and was inadequately tested.  Mark version 3.0y.

5 files changed, 116 insertions, 81 deletions

diff --git a/xdelta3/Makefile b/xdelta3/Makefile
index 62a4adf..398c255 100644
--- a/xdelta3/Makefile
+++ b/xdelta3/Makefile
@@ -57,7 +57,7 @@ CFLAGS= -Wall -Wshadow -fno-builtin
 WFLAGS= -Wextra -Wsign-compare -Wconversion -Wextra -Wno-unused-parameter
 
 # $Format: "REL=$Xdelta3Version$" $
-REL=3.0x
+REL=3.0y
 
 RELDIR = xdelta$(REL)
 
@@ -72,7 +72,7 @@ EXTRA = Makefile COPYING linkxd3lib.c badcopy.c xdelta3.swig \
         testing/cmp.h testing/delta.h testing/file.h \
         testing/modify.h testing/random.h testing/segment.h \
         testing/sizes.h testing/test.h testing/Makefile \
-        testing/regtest.cc README readme.txt
+        testing/regtest.cc README readme.txt xdelta3.1
 
 SWIG_FLAGS = -DXD3_DEBUG=1 \
               -DEXTERNAL_COMPRESSION=0 \
diff --git a/xdelta3/Makefile.mingw b/xdelta3/Makefile.mingw
index e775ec6..22877b3 100644
--- a/xdelta3/Makefile.mingw
+++ b/xdelta3/Makefile.mingw
@@ -55,7 +55,7 @@ CFLAGS= -Wall -Wshadow -fno-builtin
 XDWINFLAGS:=-DXD3_STDIO=0 -DXD3_POSIX=0 -DXD3_WIN32=1 -DEXTERNAL_COMPRESSION=0
 
 # $Format: "REL=$Xdelta3Version$" $
-REL=3.0x
+REL=3.0y
 
 RELDIR = xdelta$(REL)
 
diff --git a/xdelta3/setup.py b/xdelta3/setup.py
index 218bb4e..53722bb 100644
--- a/xdelta3/setup.py
+++ b/xdelta3/setup.py
@@ -50,7 +50,7 @@ xdelta3_ext = Extension('xdelta3main',
                                                ])
 
 # $Format: "REL='$Xdelta3Version$'" $
-REL='3.0x'
+REL='3.0y'
 
 # This provides xdelta3.main(), which calls the xdelta3 command-line main()
 # from python.
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h
index 56c283a..19def5d 100644
--- a/xdelta3/xdelta3-main.h
+++ b/xdelta3/xdelta3-main.h
@@ -150,9 +150,9 @@ static STARTUPINFO winStartupInfo;
  * inputs to the Xdelta encoder. */
 typedef enum
 {
-  RD_FIRST        = (1 << 0),
-  RD_NONEXTERNAL  = (1 << 1),
-  RD_EXTERNAL_V1  = (1 << 2),
+  RD_FIRST       = (1 << 0),
+  RD_NONEXTERNAL = (1 << 1),
+  RD_DECOMPSET   = (1 << 2),
 } xd3_read_flags;
 
 /* main_file->mode values */
@@ -388,7 +388,7 @@ static int
 main_version (void)
 {
   /* $Format: "  DP(RINT \"Xdelta version $Xdelta3Version$, Copyright (C) 2007, 2008, 2009, 2010, Joshua MacDonald\n\");" $ */
-  DP(RINT "Xdelta version 3.0x, Copyright (C) 2007, 2008, 2009, 2010, Joshua MacDonald\n");
+  DP(RINT "Xdelta version 3.0y, Copyright (C) 2007, 2008, 2009, 2010, Joshua MacDonald\n");
   DP(RINT "Xdelta comes with ABSOLUTELY NO WARRANTY.\n");
   DP(RINT "This is free software, and you are welcome to redistribute it\n");
   DP(RINT "under certain conditions; see \"COPYING\" for details.\n");
@@ -1923,7 +1923,7 @@ main_merge_output (xd3_stream *stream, main_file *ofile)
           (stream->dec_win_ind & VCD_ADLER32) != 0)
         {
           recode_stream->flags |= XD3_ADLER32_RECODE;
-          recode_stream->recode_adler32 = 
+          recode_stream->recode_adler32 =
             stream->whole_target.wininfo[window_num].adler32;
         }
 
@@ -2206,14 +2206,14 @@ main_pipe_copier (uint8_t    *pipe_buf,
  * the decompression command, the second process copies data to the
  * input of the first. */
 static int
-main_input_decompress_setup (const main_extcomp     *decomp,
-                             main_file              *ifile,
-                             uint8_t               *input_buf,
-                             usize_t                 input_bufsize,
-                             uint8_t               *pipe_buf,
-                             usize_t                 pipe_bufsize,
-                             usize_t                 pipe_avail,
-                             usize_t                *nread)
+main_input_decompress_setup (const main_extcomp   *decomp,
+                             main_file            *ifile,
+                             uint8_t              *input_buf,
+                             usize_t               input_bufsize,
+                             uint8_t              *pipe_buf,
+                             usize_t               pipe_bufsize,
+                             usize_t               pipe_avail,
+                             usize_t              *nread)
 {
   /* The two pipes: input and output file descriptors. */
   int outpipefd[2], inpipefd[2];
@@ -2352,6 +2352,7 @@ main_secondary_decompress_check (main_file  *file,
   usize_t try_read = min (input_size, XD3_ALLOCSIZE);
   usize_t check_nread;
   uint8_t check_buf[XD3_ALLOCSIZE];  /* TODO: stack limit */
+  const main_extcomp *decompressor = NULL;
 
   if ((ret = main_file_read (file, check_buf,
                              try_read,
@@ -2360,25 +2361,49 @@ main_secondary_decompress_check (main_file  *file,
       return ret;
     }
 
-  for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)
+  if (file->flags & RD_DECOMPSET)
     {
-      const main_extcomp *decomp = & extcomp_types[i];
-
-      if ((check_nread > decomp->magic_size) &&
-          /* The following expr skips decompression if we are trying
-           * to read a VCDIFF input and that is the magic number. */
-          !((decomp->flags & RD_NONEXTERNAL) &&
-            (file->flags & RD_NONEXTERNAL)) &&
-          memcmp (check_buf, decomp->magic, decomp->magic_size) == 0)
+      /* This allows the application header to override the magic
+       * number, for whatever reason. */
+      decompressor = file->compressor;
+    }
+  else
+    {
+      for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)
         {
-          file->size_known = 0;
-          return main_input_decompress_setup (decomp, file,
-                                              input_buf, input_size,
-                                              check_buf, XD3_ALLOCSIZE,
-                                              check_nread, nread);
+          const main_extcomp *decomp = & extcomp_types[i];
+          
+          if (check_nread > decomp->magic_size)
+            {
+              /* The following expr checks if we are trying to read a
+               * VCDIFF input, in which case do not treat it as
+               * "secondary" decompression. */
+              int skip_this_type = (decomp->flags & RD_NONEXTERNAL) &&
+                                   (file->flags & RD_NONEXTERNAL);
+
+              if (skip_this_type)
+                {
+                  continue;
+                }
+
+              if (memcmp (check_buf, decomp->magic, decomp->magic_size) == 0)
+                {
+                  decompressor = decomp;
+                  break;
+                }
+            }
         }
     }
 
+  if (decompressor != NULL)
+    {
+      file->size_known = 0;
+      return main_input_decompress_setup (decompressor, file,
+                                          input_buf, input_size,
+                                          check_buf, XD3_ALLOCSIZE,
+                                          check_nread, nread);
+    }
+
   /* Now read the rest of the input block. */
   (*nread) = 0;
 
@@ -2655,6 +2680,7 @@ main_get_appheader_params (main_file *file, char **parsed,
   /* Set the compressor, initiate de/recompression later. */
   if (file->compressor == NULL && *parsed[1] != 0)
     {
+      file->flags |= RD_DECOMPSET;
       file->compressor = main_get_compressor (parsed[1]);
     }
 }
@@ -2723,22 +2749,13 @@ static int
 main_read_primary_input (main_file   *file,
                          uint8_t     *buf,
                          usize_t      size,
-                         usize_t     *nread,
-                         int          is_source)
+                         usize_t     *nread)
 {
 #if EXTERNAL_COMPRESSION
   if (option_decompress_inputs && file->flags & RD_FIRST)
     {
       file->flags &= ~RD_FIRST;
-
-      if (is_source && file->compressor == NULL)
-        {
-          /* Application header overrides magic number. */
-        }
-      else
-        {
-          return main_secondary_decompress_check (file, buf, size, nread);
-        }
+      return main_secondary_decompress_check (file, buf, size, nread);
     }
 #endif
 
@@ -2879,7 +2896,7 @@ main_set_source (xd3_stream *stream, xd3_cmd cmd,
 
   /* We have to read the first block into the cache now, because
    * size_known can still change (due to secondary
-   * decompression). Calls main_decompress_input_check() via
+   * decompression). Calls main_secondary_decompress_check() via
    * main_read_primary_input(). */
   /* TODO(jmacd): This is a huge hack!  Fix me. */
   lru_size = 1;
@@ -3051,7 +3068,7 @@ main_get_winsize (main_file *ifile) {
  *******************************************************************/
 
 static int
-main_getblk_lru (xd3_source *source, xoff_t blkno, 
+main_getblk_lru (xd3_source *source, xoff_t blkno,
                  main_blklru** blrup, int *is_new)
 {
   main_blklru *blru = NULL;
@@ -3178,7 +3195,7 @@ main_read_seek_source (xd3_stream *stream,
           xoff_t skip_blkno;
           usize_t skip_offset;
 
-          xd3_blksize_div (sfile->source_position, source, 
+          xd3_blksize_div (sfile->source_position, source,
                            &skip_blkno, &skip_offset);
 
           /* Read past unused data */
@@ -3196,8 +3213,7 @@ main_read_seek_source (xd3_stream *stream,
           if ((ret = main_read_primary_input (sfile,
                                               (uint8_t*) blru->blk,
                                               source->blksize,
-                                              & nread,
-                                              1 /* source */)))
+                                              & nread)))
             {
               return ret;
             }
@@ -3295,8 +3311,7 @@ main_getblk_func (xd3_stream *stream,
   if ((ret = main_read_primary_input (sfile,
                                       (uint8_t*) blru->blk,
                                       source->blksize,
-                                      & nread,
-                                      1 /* source */)))
+                                      & nread)))
     {
       return ret;
     }
@@ -3564,8 +3579,7 @@ main_input (xd3_cmd     cmd,
       try_read = (usize_t) min ((xoff_t) config.winsize, input_remain);
 
       if ((ret = main_read_primary_input (ifile, main_bdata,
-                                          try_read, & nread,
-                                          0 /* !source */)))
+                                          try_read, & nread)))
         {
           return EXIT_FAILURE;
         }
diff --git a/xdelta3/xdelta3-test.h b/xdelta3/xdelta3-test.h
index 21d7283..9050554 100644
--- a/xdelta3/xdelta3-test.h
+++ b/xdelta3/xdelta3-test.h
@@ -43,7 +43,7 @@ void mt_init(mtrand *mt, uint32_t seed) {
     /* In the previous versions, MSBs of the seed affect   */
     /* only MSBs of the array mt[].                        */
     /* 2002/01/09 modified by Makoto Matsumoto             */
-    mt->mt_buffer_[i] = 
+    mt->mt_buffer_[i] =
         (1812433253UL * (mt->mt_buffer_[i-1] ^ (mt->mt_buffer_[i-1] >> 30)) + i);
   }
 }
@@ -70,14 +70,14 @@ uint32_t mt_random (mtrand *mt) {
     mt->mt_buffer_[MT_LEN - 1] = mt->mt_buffer_[MT_IA - 1] ^ (y >> 1) ^ mag01[y & 0x1UL];
     mt->mt_index_ = 0;
   }
-  
+
   y = mt->mt_buffer_[mt->mt_index_++];
-  
+
   y ^= (y >> 11);
   y ^= (y << 7) & 0x9d2c5680UL;
   y ^= (y << 15) & 0xefc60000UL;
   y ^= (y >> 18);
-  
+
   return y;
 }
 
@@ -89,7 +89,7 @@ static uint32_t
 mt_exp_rand (uint32_t mean, uint32_t max_value)
 {
   double mean_d = mean;
-  double erand  = log (1.0 / (mt_random (&static_mtrand) / 
+  double erand  = log (1.0 / (mt_random (&static_mtrand) /
                               (double)UINT32_MAX));
   uint32_t x = (uint32_t) (mean_d * erand + 0.5);
 
@@ -1961,7 +1961,7 @@ test_recode_command2 (xd3_stream *stream, int has_source,
 
   /* Now decode */
   sprintf (dcmd, "%s -fd ", program_name);
-  
+
   if (has_source)
     {
       strcat (dcmd, "-s ");
@@ -1987,14 +1987,14 @@ test_recode_command2 (xd3_stream *stream, int has_source,
     }
 
   return 0;
-}  
+}
 
 static int
 test_recode_command (xd3_stream *stream, int ignore)
 {
   /* Things to test:
    * - with and without a source file (recode does not change)
-   * 
+   *
    * (recode may or may not change -- 8 variations)
    * - with and without adler32
    * - with and without app header
@@ -2146,6 +2146,7 @@ test_source_decompression (xd3_stream *stream, int ignore)
   int ret;
   char buf[TESTBUFSIZE];
   const main_extcomp *ext;
+  xoff_t dsize;
 
   mt_init (& static_mtrand, 0x9f73f7fc);
 
@@ -2153,53 +2154,73 @@ test_source_decompression (xd3_stream *stream, int ignore)
   if ((ret = test_make_inputs (stream, NULL, NULL))) { return ret; }
 
   /* Use gzip. */
-  if ((ext = main_get_compressor ("G")) == NULL) { DP(RINT "skipped"); return 0; }
+  if ((ext = main_get_compressor ("G")) == NULL)
+    {
+      DP(RINT "skipped");
+      return 0;
+    }
 
   /* Save an uncompressed copy. */
   if ((ret = test_save_copy (TEST_TARGET_FILE))) { return ret; }
 
-  /* Compress the target. */
-  sprintf (buf, "%s %s < %s > %s", ext->recomp_cmdname,
-           ext->recomp_options, TEST_TARGET_FILE, TEST_SOURCE_FILE);
+  /* Compress the source. */
+  sprintf (buf, "%s -1 %s < %s > %s", ext->recomp_cmdname,
+           ext->recomp_options, TEST_COPY_FILE, TEST_SOURCE_FILE);
   if ((ret = do_cmd (stream, buf))) { return ret; }
-
-  /* Copy back to the source. */
-  sprintf (buf, "cp -f %s %s", TEST_SOURCE_FILE, TEST_TARGET_FILE);
+  /* Compress the target. */
+  sprintf (buf, "%s -9 %s < %s > %s", ext->recomp_cmdname,
+           ext->recomp_options, TEST_COPY_FILE, TEST_TARGET_FILE);
   if ((ret = do_cmd (stream, buf))) { return ret; }
 
   /* Now the two identical files are compressed.  Delta-encode the target,
    * with decompression. */
-  sprintf (buf, "%s -v -eq -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_TARGET_FILE, TEST_DELTA_FILE);
+  sprintf (buf, "%s -e -A -vfq -s%s %s %s", program_name, TEST_SOURCE_FILE,
+           TEST_TARGET_FILE, TEST_DELTA_FILE);
   if ((ret = do_cmd (stream, buf))) { return ret; }
 
+  /* Check that the compressed file is small (b/c inputs are
+   * identical). */
+  if ((ret = test_file_size (TEST_DELTA_FILE, & dsize))) { return ret; }
+  /* Deltas for identical files should be very small. */
+  if (dsize > 200)
+    {
+      DP(RINT "external compression did not happen\n");
+      stream->msg = "external compression did not happen";
+      return XD3_INTERNAL;
+    }
+
   /* Decode the delta file with recompression disabled, should get an
    * uncompressed file out. */
-  sprintf (buf, "%s -v -dq -R -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE);
+  sprintf (buf, "%s -v -dq -R -s%s %s %s", program_name,
+           TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE);
   if ((ret = do_cmd (stream, buf))) { return ret; }
-  if ((ret = compare_files (stream, TEST_COPY_FILE, TEST_RECON_FILE))) { return ret; }
+  if ((ret = compare_files (stream, TEST_COPY_FILE,
+                            TEST_RECON_FILE))) { return ret; }
 
   /* Decode the delta file with recompression, should get a compressed file
    * out.  But we can't compare compressed files directly. */
-  sprintf (buf, "%s -v -dqf -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE);
+  sprintf (buf, "%s -v -dqf -s%s %s %s", program_name,
+           TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE);
   if ((ret = do_cmd (stream, buf))) { return ret; }
-  sprintf (buf, "%s %s < %s > %s", ext->decomp_cmdname, ext->decomp_options, TEST_RECON_FILE, TEST_RECON2_FILE);
+  sprintf (buf, "%s %s < %s > %s", ext->decomp_cmdname, ext->decomp_options,
+           TEST_RECON_FILE, TEST_RECON2_FILE);
   if ((ret = do_cmd (stream, buf))) { return ret; }
-  if ((ret = compare_files (stream, TEST_COPY_FILE, TEST_RECON2_FILE))) { return ret; }
+  if ((ret = compare_files (stream, TEST_COPY_FILE,
+                            TEST_RECON2_FILE))) { return ret; }
 
   /* Encode with decompression disabled */
-  sprintf (buf, "%s -v -feqD -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_TARGET_FILE, TEST_DELTA_FILE);
+  sprintf (buf, "%s -e -D -vfq -s%s %s %s", program_name,
+           TEST_SOURCE_FILE, TEST_TARGET_FILE, TEST_DELTA_FILE);
   if ((ret = do_cmd (stream, buf))) { return ret; }
 
-  /* Decode the delta file with recompression enabled, it doesn't matter,
-   * should get the compressed file out. */
-  sprintf (buf, "%s -v -fdq -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE);
+  /* Decode the delta file with decompression disabled, should get the
+   * identical compressed file out. */
+  sprintf (buf, "%s -d -D -vfq -s%s %s %s", program_name,
+           TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE);
   if ((ret = do_cmd (stream, buf))) { return ret; }
-  if ((ret = compare_files (stream, TEST_TARGET_FILE, TEST_RECON_FILE))) { return ret; }
+  if ((ret = compare_files (stream, TEST_TARGET_FILE,
+                            TEST_RECON_FILE))) { return ret; }
 
-  /* Try again with recompression disabled, it doesn't make a difference. */
-  sprintf (buf, "%s -v -fqRd -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE);
-  if ((ret = do_cmd (stream, buf))) { return ret; }
-  if ((ret = compare_files (stream, TEST_TARGET_FILE, TEST_RECON_FILE))) { return ret; }
   test_cleanup();
   return 0;
 }