Fix secondary compression for recode. Now able to reproduce doytchin's

bug report by recoding an FGK as a DJW.

6 files changed, 59 insertions, 51 deletions

diff --git a/xdelta3/setup.py b/xdelta3/setup.py
index f50e34a..6e2bd66 100644
--- a/xdelta3/setup.py
+++ b/xdelta3/setup.py
@@ -49,7 +49,7 @@ xdelta3_ext = Extension('xdelta3main',
                                                ])
 
 # $Format: "REL='$Xdelta3Version$'" $
-REL='0q'
+REL='0p_0'
 
 # This provides xdelta3.main(), which calls the xdelta3 command-line main()
 # from python.
diff --git a/xdelta3/xdelta3-decode.h b/xdelta3/xdelta3-decode.h
index fd41675..0368cdc 100644
--- a/xdelta3/xdelta3-decode.h
+++ b/xdelta3/xdelta3-decode.h
@@ -495,6 +495,27 @@ xd3_decode_finish_window (xd3_stream *stream)
 }
 
 static int
+xd3_decode_secondary_sections (xd3_stream *secondary_stream)
+{
+#if SECONDARY_ANY
+  int ret;
+#define DECODE_SECONDARY_SECTION(UPPER,LOWER) \
+  ((secondary_stream->dec_del_ind & VCD_ ## UPPER ## COMP) && \
+   (ret = xd3_decode_secondary (secondary_stream, & secondary_stream-> LOWER ## _sect, \
+                                        & xd3_sec_ ## LOWER (secondary_stream))))
+
+  if (DECODE_SECONDARY_SECTION (DATA, data) ||
+      DECODE_SECONDARY_SECTION (INST, inst) ||
+      DECODE_SECONDARY_SECTION (ADDR, addr))
+    {
+      return ret;
+    }
+#undef DECODE_SECONDARY_SECTION
+#endif
+  return 0;
+}
+
+static int
 xd3_decode_sections (xd3_stream *stream)
 {
   usize_t need, more, take;
@@ -557,19 +578,7 @@ xd3_decode_sections (xd3_stream *stream)
 
   XD3_ASSERT (stream->dec_winbytes == need);
 
-#if SECONDARY_ANY
-#define DECODE_SECONDARY_SECTION(UPPER,LOWER) \
-  ((stream->dec_del_ind & VCD_ ## UPPER ## COMP) && \
-   (ret = xd3_decode_secondary (stream, & stream-> LOWER ## _sect, \
-                                        & xd3_sec_ ## LOWER (stream))))
-
-  if (DECODE_SECONDARY_SECTION (DATA, data) ||
-      DECODE_SECONDARY_SECTION (INST, inst) ||
-      DECODE_SECONDARY_SECTION (ADDR, addr))
-    {
-      return ret;
-    }
-#endif
+  if ((ret = xd3_decode_secondary_sections (stream))) { return ret; }
 
   if (stream->flags & XD3_SKIP_EMIT)
     {
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h
index 5d13ad6..c56bead 100644
--- a/xdelta3/xdelta3-main.h
+++ b/xdelta3/xdelta3-main.h
@@ -1261,6 +1261,7 @@ main_recode_copy (xd3_stream* stream,
                                   &output->base,
                                   &output->avail)))
     {
+      XPR(NT XD3_LIB_ERRMSG (stream, ret));
       return ret;
     }
 
@@ -2484,13 +2485,40 @@ main_input (xd3_cmd     cmd,
 
   start_time = get_millisecs_now ();
 
+  if (option_use_secondary)
+    {
+      /* The default secondary compressor is DJW, if it's compiled, being used, etc. */
+      if (option_secondary == NULL)
+        {
+          if (SECONDARY_DJW) { stream_flags |= XD3_SEC_DJW; }
+        }
+      else
+        {
+          if (strcmp (option_secondary, "fgk") == 0 && SECONDARY_FGK)
+            {
+              stream_flags |= XD3_SEC_FGK;
+            }
+          else if (strcmp (option_secondary, "djw") == 0 && SECONDARY_DJW)
+            {
+              stream_flags |= XD3_SEC_DJW;
+            }
+          else
+            {
+              XPR(NT "unrecognized secondary compressor type: %s\n", option_secondary);
+              return EXIT_FAILURE;
+            }
+        }
+    }
+
+  if (option_use_checksum) { stream_flags |= XD3_ADLER32; }
+
   /* main_input setup. */
   switch ((int) cmd)
     {
 #if VCDIFF_TOOLS
-           if (1) { case CMD_PRINTHDR:   stream_flags = XD3_JUST_HDR; }
-      else if (1) { case CMD_PRINTHDRS:  stream_flags = XD3_SKIP_WINDOW; }
-      else        { case CMD_PRINTDELTA: stream_flags = XD3_SKIP_EMIT; }
+           if (1) { case CMD_PRINTHDR:   stream_flags |= XD3_JUST_HDR; }
+      else if (1) { case CMD_PRINTHDRS:  stream_flags |= XD3_SKIP_WINDOW; }
+      else        { case CMD_PRINTDELTA: stream_flags |= XD3_SKIP_EMIT; }
       ifile->flags |= RD_NONEXTERNAL;
       input_func    = xd3_decode_input;
       output_func   = main_print_func;
@@ -2499,14 +2527,13 @@ main_input (xd3_cmd     cmd,
       break;
 
     case CMD_RECODE:
-
       // No source will be read
-      stream_flags |= XD3_ADLER32_NOVER;
+      stream_flags |= XD3_ADLER32_NOVER | XD3_SKIP_EMIT;
 
       XD3_ASSERT (recode_stream == NULL);
       recode_stream = (xd3_stream*) main_malloc(sizeof(xd3_stream));
 
-      int recode_flags = (stream.flags & XD3_SEC_TYPE);
+      int recode_flags = (stream_flags & XD3_SEC_TYPE);
       // TODO: what about sec_xxxx.ngroups = 1?
 
       xd3_config recode_config;
@@ -2521,6 +2548,7 @@ main_input (xd3_cmd     cmd,
       if ((ret = xd3_config_stream (recode_stream, &recode_config)) ||
           (ret = xd3_encode_init_buffers (recode_stream)))
         {
+          XPR(NT XD3_LIB_ERRMSG (& stream, ret));
           return EXIT_FAILURE;
         }
 
@@ -2537,31 +2565,6 @@ main_input (xd3_cmd     cmd,
       input_func  = xd3_encode_input;
       output_func = main_write_output;
 
-      if (option_use_checksum) { stream_flags |= XD3_ADLER32; }
-      if (option_use_secondary)
-        {
-          /* The default secondary compressor is DJW, if it's compiled, being used, etc. */
-          if (option_secondary == NULL)
-            {
-              if (SECONDARY_DJW) { stream_flags |= XD3_SEC_DJW; }
-            }
-          else
-            {
-              if (strcmp (option_secondary, "fgk") == 0 && SECONDARY_FGK)
-                {
-                  stream_flags |= XD3_SEC_FGK;
-                }
-              else if (strcmp (option_secondary, "djw") == 0 && SECONDARY_DJW)
-                {
-                  stream_flags |= XD3_SEC_DJW;
-                }
-              else
-                {
-                  XPR(NT "unrecognized secondary compressor type: %s\n", option_secondary);
-                  return EXIT_FAILURE;
-                }
-            }
-        }
       if (option_no_compress)      { stream_flags |= XD3_NOCOMPRESS; }
       if (option_use_altcodetable) { stream_flags |= XD3_ALT_CODE_TABLE; }
       if (option_smatch_config)
@@ -2615,7 +2618,6 @@ main_input (xd3_cmd     cmd,
 #endif
     case CMD_DECODE:
       if (option_use_checksum == 0) { stream_flags |= XD3_ADLER32_NOVER; }
-      stream_flags  = 0;
       ifile->flags |= RD_NONEXTERNAL;
       input_func    = xd3_decode_input;
       output_func   = main_write_output;
diff --git a/xdelta3/xdelta3-second.h b/xdelta3/xdelta3-second.h
index f925586..db3a68c 100644
--- a/xdelta3/xdelta3-second.h
+++ b/xdelta3/xdelta3-second.h
@@ -198,6 +198,7 @@ xd3_decode_secondary (xd3_stream      *stream,
 
   sect->buf     = sect->copied2;
   sect->buf_max = sect->copied2 + dec_size;
+  sect->size = dec_size;
 
   return 0;
 }
diff --git a/xdelta3/xdelta3.h b/xdelta3/xdelta3.h
index 2b6169b..07b2be0 100644
--- a/xdelta3/xdelta3.h
+++ b/xdelta3/xdelta3.h
@@ -100,11 +100,7 @@ typedef u_int16_t       uint16_t;
 typedef u_int32_t       uint32_t;
 #endif
 
-#if sizeof(long) < 8
-typedef u_int64_t       uint64_t;
-#else
 typedef long long unsigned int uint64_t;
-#endif
 
 #else
 #define WIN32_LEAN_AND_MEAN
diff --git a/xdelta3/xdelta3.prj b/xdelta3/xdelta3.prj
index 130abed..42ddee9 100644
--- a/xdelta3/xdelta3.prj
+++ b/xdelta3/xdelta3.prj
@@ -9,7 +9,7 @@
 (Checkin-Login jmacd)
 (Populate-Ignore ("\\.svn"))
 (Project-Keywords
- (Xdelta3Version "0q")
+ (Xdelta3Version "0p_0")
  )
 (Files
   (COPYING (xdelta3/b/29_COPYING 1.1 744))