Add "xdelta3 recode" function, which is currently outputing incorrect

VCDIFF windows.

5 files changed, 226 insertions, 66 deletions

diff --git a/xdelta3/xdelta3-decode.h b/xdelta3/xdelta3-decode.h
index 34c0fea..fd41675 100644
--- a/xdelta3/xdelta3-decode.h
+++ b/xdelta3/xdelta3-decode.h
@@ -99,22 +99,20 @@ xd3_decode_setup_buffers (xd3_stream *stream)
 static int
 xd3_decode_allocate (xd3_stream  *stream,
                      usize_t       size,
-                     uint8_t    **copied1,
-                     usize_t      *alloc1,
-                     uint8_t    **copied2,
-                     usize_t      *alloc2)
+                     uint8_t    **buf_ptr,
+                     usize_t      *buf_alloc)
 {
-  if (*copied1 != NULL && *alloc1 < size)
+  if (*buf_ptr != NULL && *buf_alloc < size)
     {
-      xd3_free (stream, *copied1);
-      *copied1 = NULL;
+      xd3_free (stream, *buf_ptr);
+      *buf_ptr = NULL;
     }
 
-  if (*copied1 == NULL)
+  if (*buf_ptr == NULL)
     {
-      *alloc1 = xd3_round_blksize (size, XD3_ALLOCSIZE);
+      *buf_alloc = xd3_round_blksize (size, XD3_ALLOCSIZE);
 
-      if ((*copied1 = xd3_alloc (stream, *alloc1, 1)) == NULL)
+      if ((*buf_ptr = xd3_alloc (stream, *buf_alloc, 1)) == NULL)
         {
           return ENOMEM;
         }
@@ -161,9 +159,7 @@ xd3_decode_section (xd3_stream *stream,
               if ((ret = xd3_decode_allocate (stream,
                                               section->size,
                                               & section->copied1,
-                                              & section->alloc1,
-                                              & section->copied2,
-                                              & section->alloc2))) { return ret; }
+                                              & section->alloc1))) { return ret; }
 
               section->buf = section->copied1;
             }
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h
index 09a801a..57ad3d2 100644
--- a/xdelta3/xdelta3-main.h
+++ b/xdelta3/xdelta3-main.h
@@ -159,6 +159,7 @@ typedef enum
   CMD_PRINTHDR,
   CMD_PRINTHDRS,
   CMD_PRINTDELTA,
+  CMD_RECODE,
 #if XD3_ENCODER
   CMD_ENCODE,
 #endif
@@ -304,6 +305,9 @@ static int lru_filled = 0;
 /* Hacks for VCDIFF tools */
 static int allow_fake_source = 0;
 
+/* State for xdelta3 recode */
+static xd3_stream *recode_stream = NULL;
+
 /* This array of compressor types is compiled even if EXTERNAL_COMPRESSION is false just so
  * the program knows the mapping of IDENT->NAME. */
 static main_extcomp extcomp_types[] =
@@ -982,7 +986,7 @@ main_file_seek (main_file *xfile, xoff_t pos)
 
 /* The following macros let VCDIFF printing something printf-like with
  * main_file_write(), e.g.,:
- * 
+ *
  *   VC(UT "trying to be portable: %d\n", x)VE;
  */
 #define SNPRINTF_BUFSIZE 1024
@@ -1220,6 +1224,97 @@ main_print_func (xd3_stream* stream, main_file *xfile)
 
   return ret;
 }
+
+static int
+main_recode_copy (xd3_stream* stream,
+                  xd3_output* output,
+                  xd3_desect* input)
+{
+  int ret;
+
+  XD3_ASSERT(output != NULL);
+  XD3_ASSERT(output->next_page == NULL);
+
+  if ((ret = xd3_decode_allocate (recode_stream,
+                                  input->size,
+                                  &output->base,
+                                  &output->avail)))
+    {
+      return ret;
+    }
+
+  memcpy (output->base, input->buf, input->size);
+  output->next = input->size;
+  return 0;
+}
+
+// Re-encode one window
+static int
+main_recode_func (xd3_stream* stream, main_file *xfile)
+{
+  int ret;
+
+  XD3_ASSERT(stream->dec_state == DEC_FINISH);
+  XD3_ASSERT(recode_stream->enc_state == ENC_INIT ||
+             recode_stream->enc_state == ENC_INPUT);
+
+  // Copy partial decoder output to partial encoder inputs
+  if ((ret = main_recode_copy (recode_stream,
+                               DATA_HEAD(recode_stream),
+                               &stream->data_sect)) ||
+      (ret = main_recode_copy (recode_stream,
+                               INST_HEAD(recode_stream),
+                               &stream->inst_sect)) ||
+      (ret = main_recode_copy (recode_stream,
+                               ADDR_HEAD(recode_stream),
+                               &stream->addr_sect)))
+    {
+      return ret;
+    }
+
+  // This jumps to xd3_emit_hdr()
+  recode_stream->enc_state = ENC_FLUSH;
+  //recode_stream->avail_in = stream->dec_tgtlen;
+  //recode_stream->n_emit = stream->dec_tgtlen;
+
+  // Output loop
+  for (;;)
+    {
+      switch((ret = xd3_encode_input (recode_stream)))
+        {
+        case XD3_INPUT: {
+          /* finished recoding one window */
+          return 0;
+        }
+        case XD3_OUTPUT: {
+          /* main_file_write below */
+          break;
+        }
+        case XD3_GOTHEADER:
+        case XD3_WINSTART:
+        case XD3_WINFINISH: {
+          /* ignore */
+          continue;
+        }
+        case XD3_GETSRCBLK:
+        case 0: {
+            stream->msg = "invalid operation";
+            return XD3_INTERNAL;
+          }
+        default:
+          return ret;
+        }
+
+      if ((ret = main_file_write(xfile,
+                                 recode_stream->next_out,
+                                 recode_stream->avail_out, "write failed")))
+        {
+          return ret;
+        }
+
+      xd3_consume_output (recode_stream);
+    }
+}
 #endif /* VCDIFF_TOOLS */
 
 /******************************************************************************************
@@ -2054,7 +2149,7 @@ main_set_source (xd3_stream *stream, int cmd, main_file *sfile, xd3_source *sour
   source->ioh      = sfile;
   source->curblkno = (xoff_t) -1;
   source->curblk   = NULL;
-  
+
 #if EXTERNAL_COMPRESSION
   if (option_decompress_inputs)
     {
@@ -2149,7 +2244,7 @@ main_set_source (xd3_stream *stream, int cmd, main_file *sfile, xd3_source *sour
   if (option_verbose)
     {
       static char buf[32];
-      
+
       XPR(NT "source %s winsize %s size %"Q"u\n",
           sfile->filename, main_format_bcnt(option_srcwinsz, buf), source->size);
     }
@@ -2371,6 +2466,7 @@ main_input (xd3_cmd     cmd,
 
   config.alloc = main_alloc;
   config.freef = main_free1;
+  // TODO: what about sec_xxxx.ngroups = 1?
   config.sec_data.ngroups = 1;
   config.sec_addr.ngroups = 1;
   config.sec_inst.ngroups = 1;
@@ -2394,6 +2490,31 @@ main_input (xd3_cmd     cmd,
       stream_flags |= XD3_ADLER32_NOVER;
       stdout_only   = 1;
       break;
+
+    case CMD_RECODE:
+
+      XD3_ASSERT (recode_stream == NULL);
+      recode_stream = (xd3_stream*) main_malloc(sizeof(xd3_stream));
+
+      xd3_config recode_config;
+      xd3_init_config(&recode_config, 0);
+
+      // TODO: what about sec_xxxx.ngroups = 1?
+      config.alloc = main_alloc;
+      config.freef = main_free1;
+
+      if ((ret = xd3_config_stream (recode_stream, &recode_config)) ||
+          (ret = xd3_encode_init_buffers (recode_stream)))
+        {
+          return EXIT_FAILURE;
+        }
+
+      // TODO: xd3_set_appheader (recode_stream)
+
+      ifile->flags |= RD_NONEXTERNAL;
+      input_func    = xd3_decode_input;
+      output_func   = main_recode_func;
+      break;
 #endif
 #if XD3_ENCODER
     case CMD_ENCODE:
@@ -2615,7 +2736,8 @@ main_input (xd3_cmd     cmd,
               }
             else if (cmd == CMD_PRINTHDR ||
                      cmd == CMD_PRINTHDRS ||
-                     cmd == CMD_PRINTDELTA)
+                     cmd == CMD_PRINTDELTA ||
+                     cmd == CMD_RECODE)
               {
                 if (xd3_decoder_needs_source (& stream) && sfile->filename == NULL)
                   {
@@ -2676,7 +2798,7 @@ main_input (xd3_cmd     cmd,
                             stream.current_window * option_winsize,
                             (stream.current_window+1) * option_winsize);
                       }
-                    
+
                     /* Limited instruction buffer size affects source copies */
                     if (option_verbose > 1 && stream.i_slots_used > stream.iopt_size)
                       {
@@ -2836,6 +2958,13 @@ main_cleanup (void)
   lru_misses = 0;
   lru_filled = 0;
 
+  if (recode_stream != NULL)
+    {
+      xd3_free_stream (recode_stream);
+      main_free (recode_stream);
+      recode_stream = NULL;
+    }
+
   XD3_ASSERT (main_mallocs == 0);
 }
 
@@ -3044,6 +3173,7 @@ main (int argc, char **argv)
           else if (strcmp (my_optstr, "printhdr") == 0) { cmd = CMD_PRINTHDR; }
           else if (strcmp (my_optstr, "printhdrs") == 0) { cmd = CMD_PRINTHDRS; }
           else if (strcmp (my_optstr, "printdelta") == 0) { cmd = CMD_PRINTDELTA; }
+          else if (strcmp (my_optstr, "recode") == 0) { cmd = CMD_RECODE; }
 #endif
 
           /* If no option was found and still no command, let the default command be
@@ -3216,6 +3346,7 @@ main (int argc, char **argv)
     case CMD_PRINTDELTA:
 #if XD3_ENCODER
     case CMD_ENCODE:
+    case CMD_RECODE:
 #endif
     case CMD_DECODE:
       ret = main_input (cmd, & ifile, & ofile, & sfile);
@@ -3285,6 +3416,7 @@ main_help (void)
   DP(RINT "    printdelta  print information about the entire delta\n");
   DP(RINT "    printhdr    print information about the first window\n");
   DP(RINT "    printhdrs   print information about all windows\n");
+  DP(RINT "    recode      encode with new application/secondary settings\n");
 #endif
   DP(RINT "standard options:\n");
   DP(RINT "   -0 .. -9     compression level\n");
diff --git a/xdelta3/xdelta3-second.h b/xdelta3/xdelta3-second.h
index a8196fb..f925586 100644
--- a/xdelta3/xdelta3-second.h
+++ b/xdelta3/xdelta3-second.h
@@ -173,7 +173,7 @@ xd3_decode_secondary (xd3_stream      *stream,
 
   /* Decode the size, allocate the buffer. */
   if ((ret = xd3_read_size (stream, & sect->buf, sect->buf_max, & dec_size)) ||
-      (ret = xd3_decode_allocate (stream, dec_size, & sect->copied2, & sect->alloc2, NULL, NULL)))
+      (ret = xd3_decode_allocate (stream, dec_size, & sect->copied2, & sect->alloc2)))
     {
       return ret;
     }
diff --git a/xdelta3/xdelta3.c b/xdelta3/xdelta3.c
index 738bcb0..fb6191d 100644
--- a/xdelta3/xdelta3.c
+++ b/xdelta3/xdelta3.c
@@ -427,15 +427,15 @@ XD3_MAKELIST(xd3_rlist, xd3_rinst, link);
 
 #define ENC_SECTS         4  /* Number of separate output sections. */
 
-#define HDR_TAIL(s)  (stream->enc_tails[0])
-#define DATA_TAIL(s) (stream->enc_tails[1])
-#define INST_TAIL(s) (stream->enc_tails[2])
-#define ADDR_TAIL(s) (stream->enc_tails[3])
+#define HDR_TAIL(s)  ((s)->enc_tails[0])
+#define DATA_TAIL(s) ((s)->enc_tails[1])
+#define INST_TAIL(s) ((s)->enc_tails[2])
+#define ADDR_TAIL(s) ((s)->enc_tails[3])
 
-#define HDR_HEAD(s)  (stream->enc_heads[0])
-#define DATA_HEAD(s) (stream->enc_heads[1])
-#define INST_HEAD(s) (stream->enc_heads[2])
-#define ADDR_HEAD(s) (stream->enc_heads[3])
+#define HDR_HEAD(s)  ((s)->enc_heads[0])
+#define DATA_HEAD(s) ((s)->enc_heads[1])
+#define INST_HEAD(s) ((s)->enc_heads[2])
+#define ADDR_HEAD(s) ((s)->enc_heads[3])
 
 #define SIZEOF_ARRAY(x) (sizeof(x) / sizeof(x[0]))
 
@@ -577,8 +577,7 @@ static int         xd3_emit_uint32_t (xd3_stream *stream, xd3_output **output, u
 #endif /* XD3_ENCODER */
 
 static int         xd3_decode_allocate (xd3_stream  *stream, usize_t       size,
-                                        uint8_t    **copied1, usize_t      *alloc1,
-                                        uint8_t    **copied2, usize_t      *alloc2);
+                                        uint8_t    **copied1, usize_t      *alloc1);
 
 static void        xd3_compute_code_table_string (const xd3_dinst *code_table, uint8_t *str);
 static void*       xd3_alloc (xd3_stream *stream, usize_t      elts, usize_t      size);
@@ -1975,9 +1974,11 @@ static int
 xd3_alloc_cache (xd3_stream *stream)
 {
   if (((stream->acache.s_near > 0) &&
-       (stream->acache.near_array = xd3_alloc (stream, stream->acache.s_near, sizeof (usize_t))) == NULL) ||
+       (stream->acache.near_array =
+        xd3_alloc (stream, stream->acache.s_near, sizeof (usize_t))) == NULL) ||
       ((stream->acache.s_same > 0) &&
-       (stream->acache.same_array = xd3_alloc (stream, stream->acache.s_same * 256, sizeof (usize_t))) == NULL))
+       (stream->acache.same_array =
+        xd3_alloc (stream, stream->acache.s_same * 256, sizeof (usize_t))) == NULL))
     {
       return ENOMEM;
     }
@@ -3312,7 +3313,11 @@ xd3_emit_hdr (xd3_stream *stream)
 
       /* Secondary compressor ID */
 #if SECONDARY_ANY
-      if (use_secondary && (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), stream->sec_type->id))) { return ret; }
+      if (use_secondary &&
+          (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), stream->sec_type->id)))
+        {
+          return ret;
+        }
 #endif
 
       /* Compressed code table */
@@ -3321,19 +3326,26 @@ xd3_emit_hdr (xd3_stream *stream)
           usize_t code_table_size;
           const uint8_t *code_table_data;
 
-          if ((ret = stream->comp_table_func (stream, & code_table_data, & code_table_size))) { return ret; }
+          if ((ret = stream->comp_table_func (stream, & code_table_data, & code_table_size)))
+            {
+              return ret;
+            }
 
           if ((ret = xd3_emit_size (stream, & HDR_TAIL (stream), code_table_size + 2)) ||
               (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), stream->code_table_desc->near_modes)) ||
               (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), stream->code_table_desc->same_modes)) ||
-              (ret = xd3_emit_bytes (stream, & HDR_TAIL (stream), code_table_data, code_table_size))) { return ret; }
+              (ret = xd3_emit_bytes (stream, & HDR_TAIL (stream), code_table_data, code_table_size)))
+            {
+              return ret;
+            }
         }
 
       /* Application header */
       if (use_appheader)
         {
           if ((ret = xd3_emit_size (stream, & HDR_TAIL (stream), stream->enc_appheadsz)) ||
-              (ret = xd3_emit_bytes (stream, & HDR_TAIL (stream), stream->enc_appheader, stream->enc_appheadsz)))
+              (ret = xd3_emit_bytes (stream, & HDR_TAIL (stream), stream->enc_appheader,
+                                     stream->enc_appheadsz)))
             {
               return ret;
             }
@@ -3387,7 +3399,7 @@ xd3_emit_hdr (xd3_stream *stream)
   inst_len = xd3_sizeof_output (INST_HEAD (stream));
   addr_len = xd3_sizeof_output (ADDR_HEAD (stream));
 
-  /* The enc_len field is redundent... doh! */
+  /* The enc_len field is a redundency for future extensions.*/
   enc_len = (1 + (xd3_sizeof_size (tgt_len) +
                   xd3_sizeof_size (data_len) +
                   xd3_sizeof_size (inst_len) +
@@ -3508,11 +3520,29 @@ xd3_alloc_iopt (xd3_stream *stream, int elts)
   return 0;
 }
 
+/* This function allocates the encoder output buffers. */
+static int
+xd3_encode_init_buffers (xd3_stream *stream)
+{
+  int i;
+
+  for (i = 0; i < ENC_SECTS; i += 1)
+    {
+      if ((stream->enc_heads[i] =
+           stream->enc_tails[i] =
+           xd3_alloc_output (stream, NULL)) == NULL)
+        {
+          return ENOMEM;
+        }
+    }
+
+  return 0;
+}  
+
 /* This function allocates all memory initially used by the encoder. */
 static int
 xd3_encode_init (xd3_stream *stream)
 {
-  int i;
   int large_comp = (stream->src != NULL);
   int small_comp = ! (stream->flags & XD3_NOCOMPRESS);
 
@@ -3537,15 +3567,8 @@ xd3_encode_init (xd3_stream *stream)
                           & stream->small_hash);
     }
 
-  for (i = 0; i < ENC_SECTS; i += 1)
-    {
-      if ((stream->enc_heads[i] =
-           stream->enc_tails[i] =
-           xd3_alloc_output (stream, NULL)) == NULL)
-        {
-          goto fail;
-        }
-    }
+  /* data buffers */
+  if (xd3_encode_init_buffers(stream) != 0) { goto fail; }
 
   /* iopt buffer */
   xd3_rlist_init (& stream->iopt_used);
@@ -3725,10 +3748,17 @@ xd3_encode_input (xd3_stream *stream)
 
       /* Flush the instrution buffer, then possibly add one more instruction, then emit
        * the header. */
-      stream->enc_state = ENC_FLUSH;
       if ((ret = xd3_iopt_flush_instructions (stream, 1)) ||
-          (ret = xd3_iopt_add_finalize (stream)) ||
-          (ret = xd3_emit_hdr (stream)))
+          (ret = xd3_iopt_add_finalize (stream)))
+        {
+          return ret;
+        }
+
+      stream->enc_state = ENC_FLUSH;
+
+    case ENC_FLUSH:
+      /* Note: main_recode_func() bypasses string-matching by setting ENC_FLUSH. */
+      if ((ret = xd3_emit_hdr (stream)))
         {
           return ret;
         }
diff --git a/xdelta3/xdelta3.h b/xdelta3/xdelta3.h
index 67a4cac..e43574f 100644
--- a/xdelta3/xdelta3.h
+++ b/xdelta3/xdelta3.h
@@ -494,11 +494,28 @@ struct _xd3_hinst
 struct _xd3_output
 {
   uint8_t    *base;
-  usize_t      next;
-  usize_t      avail;
+  usize_t     next;
+  usize_t     avail;
   xd3_output *next_page;
 };
 
+/* used by the decoder to buffer input in sections. */
+struct _xd3_desect
+{
+  const uint8_t *buf;
+  const uint8_t *buf_max;
+  uint32_t       size;
+  usize_t        pos;
+
+  /* used in xdelta3-decode.h */
+  uint8_t       *copied1;
+  usize_t        alloc1;
+
+  /* used in xdelta3-second.h */
+  uint8_t       *copied2;
+  usize_t        alloc2;
+};
+
 /* the VCDIFF address cache, see the RFC */
 struct _xd3_addr_cache
 {
@@ -545,21 +562,6 @@ struct _xd3_slist
   usize_t     last_pos;
 };
 
-/* a decoder section (data, inst, or addr).  there is an optimization to avoid copying
- * these sections if all the input is available, related to the copied field below.
- * secondation compression uses the copied2 field. */
-struct _xd3_desect
-{
-  const uint8_t *buf;
-  const uint8_t *buf_max;
-  uint32_t       size;
-  usize_t        pos;
-  uint8_t       *copied1;
-  usize_t        alloc1;
-  uint8_t       *copied2;
-  usize_t        alloc2;
-};
-
 /******************************************************************************************
  public types
  ******************************************************************************************/