Implement more of "xdelta3 merge" command. Compile fixes.

Work on xdelta3-regtest.py.

8 files changed, 250 insertions, 126 deletions

diff --git a/xdelta3/Makefile b/xdelta3/Makefile
index 21e088a..aa35a4a 100644
--- a/xdelta3/Makefile
+++ b/xdelta3/Makefile
@@ -3,6 +3,7 @@
 
 UNAME = $(shell uname)
 CYGWIN = $(findstring CYGWIN, $(UNAME))
+DARWIN = $(findstring Darwin, $(UNAME))
 PYVER = 2.5
 
 ifeq ("$(CYGWIN)", "")
@@ -50,6 +51,7 @@ PYTHON = python
 
 WIXDIR = "/cygdrive/c/Program Files/wix2.0.4820"
 
+# -arch x86_64
 CFLAGS= -Wall -Wshadow -fno-builtin
 
 # $Format: "REL=$Xdelta3Version$" $
@@ -67,7 +69,7 @@ EXTRA = Makefile COPYING linkxd3lib.c badcopy.c xdelta3.swig \
         xdelta3.py xdelta3_wrap.c xdelta3.wxs xdelta3.wxi \
         README readme.txt
 
-SWIG_FLAGS = -DXD3_DEBUG=0 \
+SWIG_FLAGS = -DXD3_DEBUG=1 \
               -DEXTERNAL_COMPRESSION=0 \
               -DXD3_USE_LARGEFILE64=1 \
               -DGENERIC_ENCODE_TABLES=1 \
@@ -199,11 +201,17 @@ xdelta3module.dll: xdelta3_wrap.o xdelta3.o
                 -o xdelta3module.dll
         cp $(SWIGTGT) /usr/lib/python$(PYVER)/site-packages
 
+ifeq ("$(DARWIN)", "")
 xdelta3module.so: xdelta3_wrap.o xdelta3.o
         ld -shared xdelta3.o xdelta3_wrap.o \
                 -o xdelta3module.so \
                 /usr/lib/libpython$(PYVER).so \
                 -lc
+else
+xdelta3module.so: xdelta3_wrap.o xdelta3.o
+        gcc -Wl,-F. -bundle -undefined dynamic_lookup $(CFLAGS) \
+                xdelta3.o xdelta3_wrap.o -o xdelta3module.so
+endif
 
 xdelta3-decoder: $(SOURCES)
         $(CC) -O3 -Wall -Wshadow xdelta3.c \
diff --git a/xdelta3/setup.py b/xdelta3/setup.py
index b2c9022..edcb1fd 100644
--- a/xdelta3/setup.py
+++ b/xdelta3/setup.py
@@ -46,6 +46,7 @@ xdelta3_ext = Extension('xdelta3main',
                         extra_compile_args = [ '-O3',
                                                '-g',
                                                '-fno-builtin',
+                                               # '-arch', 'x86_64',
                                                ])
 
 # $Format: "REL='$Xdelta3Version$'" $
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h
index 8848ed4..db4c011 100644
--- a/xdelta3/xdelta3-main.h
+++ b/xdelta3/xdelta3-main.h
@@ -1538,7 +1538,7 @@ main_recode_func (xd3_stream* stream, main_file *ofile)
       option_appheader != NULL)
     {
       xd3_set_appheader (recode_stream, option_appheader,
-                         strlen (option_appheader));
+                         strlen ((char*) option_appheader));
     }
   else if (option_use_appheader != 0 &&
            option_appheader == NULL)
@@ -1619,7 +1619,7 @@ main_init_recode_stream (void)
 
   if ((ret = main_set_secondary_flags (&recode_config)) ||
       (ret = xd3_config_stream (recode_stream, &recode_config)) ||
-      (ret = xd3_encode_init_buffers (recode_stream)))
+      (ret = xd3_encode_init_partial (recode_stream)))
     {
       XPR(NT XD3_LIB_ERRMSG (recode_stream, ret));
       xd3_free_stream (recode_stream);
@@ -1706,47 +1706,141 @@ main_merge_func (xd3_stream* stream, main_file *no_write)
   return 0;
 }
 
+
+#define MERGE_IN_PROGRESS 0
 #if MERGE_IN_PROGRESS
 /* This is called after all windows have been read, as a final step in
  * main_input().  This is only called for the final merge step. */
 static int
-main_merge_output (xd3_stream *stream)
+main_merge_output (xd3_stream *stream, main_file *ofile)
 {
-  int inst_pos = 0;
+  int ret;
+  usize_t inst_pos = 0;
   xoff_t output_pos = 0;
-  usize_t window_pos = 0;
+  xd3_source recode_source;
+
+  /* Enter the ENC_INPUT state and bypass the next_in == NULL test
+   * and (leftover) input buffering logic. */
+  XD3_ASSERT(recode_stream->enc_state == ENC_INIT);
+  recode_stream->enc_state = ENC_INPUT;
+  recode_stream->next_in = main_bdata;
+  recode_stream->flags |= XD3_FLUSH;
 
-  while (output_pos < option_winsize)
+  while (inst_pos < stream->whole_target_instlen)
     {
-      xd3_winst *inst = &stream->whole_target_inst[inst_pos];
-      usize_t take = min(inst->size, option_winsize - output_pos);
+      xoff_t window_srcmin = XOFF_T_MAX;
+      xoff_t window_srcmax = 0;
+      usize_t window_pos = 0;
 
-      switch (inst->type)
-        {
-        case XD3_RUN:
-          if ((ret = xd3_emit_run (recode_stream, window_pos, take,
-                                   stream->whole_target_adds[inst->addr])))
-            {
-              return ret;
-            }
-          break;
-
-        case XD3_ADD:
-          /* Adds are implicit, put them into the input buffer. */
-          memcpy (main_bdata, stream->whole_target_adds + inst->addr, take);
-          break;
-
-        default: /* XD3_COPY + mode */
-          if ((ret = xd3_found_match (stream, window_pos, take, inst->addr,
-                                      IS_SOURCE)))
-            {
-              return ret;
-            }
-          break;
-        }
+      XD3_ASSERT (recode_stream->enc_state == ENC_INPUT);
+
+      if ((ret = xd3_encode_input (recode_stream)) != XD3_WINSTART)
+        {
+          XPR(NT "invalid merge state: %s\n", xd3_mainerror (ret));
+          return XD3_INVALID;
+        }
+
+      /* TODO: The inner loop termination condition can be used to
+       * change window sizes, however keeping the last delta in the
+       * chain's window size allows re-using its adler32 checksums.
+       * For now this will change window size and not use adler32. */
+      while (window_pos < option_winsize &&
+             inst_pos < stream->whole_target_instlen)
+        {
+          xd3_winst *inst = &stream->whole_target_inst[inst_pos];
+          usize_t take = min(inst->size, option_winsize - window_pos);
 
-      //xd3_avail_input (recode_stream, main_bdata,
+          switch (inst->type)
+            {
+            case XD3_RUN:
+              if ((ret = xd3_emit_run (recode_stream, window_pos, take,
+                                       stream->whole_target_adds[inst->addr])))
+                {
+                  return ret;
+                }
+              break;
+
+            case XD3_ADD:
+              /* Adds are implicit, put them into the input buffer. */
+              memcpy (main_bdata + window_pos, 
+                      stream->whole_target_adds + inst->addr, take);
+              break;
+
+            default: /* XD3_COPY + copy mode */
+              window_srcmin = min(window_srcmin, inst->addr);
+              window_srcmax = max(window_srcmax, inst->addr + take);
+              if ((ret = xd3_found_match (recode_stream, window_pos, take, inst->addr,
+                                          inst->mode == VCD_SOURCE)))
+                {
+                  return ret;
+                }
+              break;
+            }
+
+          window_pos += take;
+          output_pos += take;
+
+          if (take == inst->size)
+            {
+              inst_pos += 1;
+            }
+          else
+            {
+              /* Modify the instruction for the next pass. */
+              /* TODO: this seems to be broken, as initial testing indicates. */
+              if (inst->type != XD3_RUN)
+                {
+                  inst->addr += take;
+                }
+              inst->size -= take;
+            }
+        }
+
+      xd3_avail_input (recode_stream, main_bdata, window_pos);
+
+      recode_stream->enc_state = ENC_INSTR;
+      recode_stream->srcwin_decided = 1;
+      recode_stream->src = &recode_source;
+      recode_source.srclen = window_srcmax - window_srcmin;
+      recode_source.srcbase = window_srcmin;
+      recode_stream->taroff = recode_source.srclen;
+
+      for (;;)
+        {
+          switch ((ret = xd3_encode_input (recode_stream)))
+            {
+            case XD3_INPUT: {
+              goto done_window;
+            }
+            case XD3_OUTPUT: {
+              /* main_file_write below */
+              break;
+            }
+            case XD3_GOTHEADER:
+            case XD3_WINSTART:
+            case XD3_WINFINISH: {
+              /* ignore */
+              continue;
+            }
+            case XD3_GETSRCBLK:
+            case 0: {
+              return XD3_INTERNAL;
+            }
+            default:
+              return ret;
+            }
+
+          if ((ret = main_write_output(recode_stream, ofile)))
+            {
+              return ret;
+            }
+
+          xd3_consume_output (recode_stream);
+        }
+    done_window:
+      (void) 0;
     }
+
   return 0;
 }
 #endif
@@ -3158,7 +3252,7 @@ main_input (xd3_cmd     cmd,
 
                 /* Check if the user expected a source to be required although
                  * it was not. */
-                if (have_src && ! need_src && ! option_quiet)
+                if (have_src && ! need_src && option_verbose)
                   {
                     XPR(NT "warning: output window %"Q"u does not "
                         "copy source\n", stream.current_window);
@@ -3244,7 +3338,7 @@ main_input (xd3_cmd     cmd,
                     main_file_isopen (sfile))
                   {
                     /* Warn when no source copies are found */
-                    if (! xd3_encoder_used_source (& stream))
+                    if (option_verbose && ! xd3_encoder_used_source (& stream))
                       {
                         XPR(NT "warning: input window %"Q"u..%"Q"u has "
                             "no source copies\n",
@@ -3321,6 +3415,14 @@ done:
       main_file_close (sfile);
     }
 
+#if MERGE_IN_PROGRESS
+  if (cmd == CMD_MERGE &&
+      (ret = main_merge_output (& stream, ofile)))
+    {
+      return EXIT_FAILURE;
+    }
+#endif
+
   /* If output file is not open yet because of delayed-open, it means
    * we never encountered a window in the delta, but it could have had
    * a VCDIFF header?  TODO: solve this elsewhere.  For now, it prints
@@ -3350,14 +3452,6 @@ done:
     }
 #endif
 
-#if MERGE_IN_PROGRESS
-  if (cmd == CMD_MERGE &&
-      (ret = main_merge_output (& stream)))
-    {
-      return EXIT_FAILURE;
-    }
-#endif
-
   if ((ret = xd3_close_stream (& stream)))
     {
       XPR(NT XD3_LIB_ERRMSG (& stream, ret));
@@ -3659,8 +3753,10 @@ main (int argc, char **argv)
           else if (strcmp (my_optstr, "printdelta") == 0)
             { cmd = CMD_PRINTDELTA; }
           else if (strcmp (my_optstr, "recode") == 0) { cmd = CMD_RECODE; }
+#if MERGE_IN_PROGRESS
           else if (strcmp (my_optstr, "merge") == 0) { cmd = CMD_MERGE; }
 #endif
+#endif
 
           /* If no option was found and still no command, let the default
            * command be encode.  The remaining args are treated as
@@ -3778,6 +3874,7 @@ main (int argc, char **argv)
 
           sfilename = my_optarg;
           break;
+#if MERGE_IN_PROGRESS
         case 'm':
           if ((merge = (main_merge*)
                main_malloc (sizeof (main_merge))) == NULL)
@@ -3787,7 +3884,7 @@ main (int argc, char **argv)
           main_merge_list_push_back (& merge_order, merge);
           merge->filename = my_optarg;
           break;
-
+#endif
         case 'V':
           ret = main_version (); goto exit;
         default:
@@ -3902,7 +3999,6 @@ main (int argc, char **argv)
 
   while (! main_merge_list_empty (& merge_order))
     {
-      /* TODO: More merge cleanup, once implemented. */
       merge = main_merge_list_pop_front (& merge_order);
       main_free (merge);
     }
diff --git a/xdelta3/xdelta3-regtest.py b/xdelta3/xdelta3-regtest.py
index e86c621..f9a11bd 100755
--- a/xdelta3/xdelta3-regtest.py
+++ b/xdelta3/xdelta3-regtest.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python2.5
 # xdelta 3 - delta compression tools and library
-# Copyright (C) 2003, 2006, 2007.  Joshua P. MacDonald
+# Copyright (C) 2003, 2006, 2007, 2008.  Joshua P. MacDonald
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -19,7 +19,6 @@
 # TODO: test 1.5 vs. greedy
 
 import os, sys, math, re, time, types, array, random
-import xdelta3main
 import xdelta3
 
 #RCSDIR = '/mnt/polaroid/Polaroid/orbit_linux/home/jmacd/PRCS'
@@ -27,7 +26,10 @@ import xdelta3
 #SAMPLEDIR = "/tmp/WESNOTH_tmp/diff"
 
 #RCSDIR = 'G:/jmacd/PRCS_copy'
-SAMPLEDIR = "C:/sample_data/Wesnoth/tar"
+#SAMPLEDIR = "C:/sample_data/Wesnoth/tar"
+
+#RCSDIR = '/Users/jmacd/src/ftp.kernel.org/pub/scm/linux/kernel/bkcvs/linux-2.4/net/x25'
+RCSDIR = '/Users/jmacd/src/ftp.kernel.org'
 
 #
 MIN_SIZE       = 0
@@ -38,8 +40,6 @@ SKIP_TRIALS    = 2
 MIN_TRIALS     = 3
 MAX_TRIALS     = 15
 
-SKIP_DECODE = 1
-
 # 10 = fast 1.5 = slow
 MIN_STDDEV_PCT = 1.5
 
@@ -57,7 +57,7 @@ MAX_RUN = 1000 * 1000 * 10
 
 # Testwide defaults
 ALL_ARGS = [
-    # -v
+    '-vv'
     ]
 
 # The first 7 args go to -C
@@ -231,11 +231,6 @@ class TimedTest:
                                        lambda x: x.Encode(self.target, self.source, DFILE))
         self.encode_size = runnable.EncodeSize(DFILE)
 
-        if SKIP_DECODE:
-            self.decode_time = StatList([1, 1], 'not decoded')
-            return
-        #end
-
         self.decode_time = self.DoTest(RFILE,
                                        lambda x: x.Decode(DFILE, self.source, RFILE),
                                        )
@@ -404,9 +399,10 @@ class Xdelta3Runner:
 
     def Main(self, args):
         try:
-            xdelta3main.main(args)
+            #print 'Run %s' % (' '.join(args))
+            xdelta3.xd3_main_cmdline(args)
         except Exception, e:
-            raise CommandError(args, "xdelta3.main exception")
+            raise CommandError(args, "xdelta3.main exception: %s" % e)
         #end
     #end
 #end
@@ -904,10 +900,7 @@ class RandomTest:
     #end
 
     def __str__(self):
-        decodestr = ''
-        if not SKIP_DECODE:
-            decodestr = ' %.6f' % self.decodetime
-        #end
+        decodestr = ' %.6f' % self.decodetime
         return 'time %.6f%s size %d%s << %s >>%s' % (
             self.time(), ((self.time_pos != None) and (" (%s)" % self.time_pos) or ""),
             self.size(), ((self.size_pos != None) and (" (%s)" % self.size_pos) or ""),
@@ -1202,19 +1195,20 @@ if __name__ == "__main__":
         RunCommand(['rm', '-rf', TMPDIR])
         os.mkdir(TMPDIR)
 
-        #rcsf = GetTestRcsFiles()
+        rcsf = GetTestRcsFiles()
         #generator = rcsf.Generator()
 
-        sample = SampleDataTest([SAMPLEDIR])
-        generator = sample.Generator()
+        #sample = SampleDataTest([SAMPLEDIR])
+        #generator = sample.Generator()
 
-        rand = random.Random(135135135135135)
-        RunTestLoop(rand, generator, TEST_ROUNDS)
+        #rand = random.Random(135135135135135)
+        #RunTestLoop(rand, generator, TEST_ROUNDS)
 
         #RunSpeedTest()
 
         #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9']))
-        #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-S', 'djw']))
+        x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-S', 'djw']))
+        x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-1', '-S', 'djw']))
         #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-T']))
 
         #x1r = rcsf.AllPairsByDate(Xdelta1RunClass())
diff --git a/xdelta3/xdelta3-test.h b/xdelta3/xdelta3-test.h
index 281d1b6..5389d57 100644
--- a/xdelta3/xdelta3-test.h
+++ b/xdelta3/xdelta3-test.h
@@ -576,7 +576,7 @@ static int
 test_forward_match (xd3_stream *stream, int unused)
 {
   int i;
-  char buf1[256], buf2[256];
+  uint8_t buf1[256], buf2[256];
 
   memset(buf1, 0, 256);
   memset(buf2, 0, 256);
@@ -615,7 +615,7 @@ test_address_cache (xd3_stream *stream, int unused)
   stream->acache.s_near = stream->code_table_desc->near_modes;
   stream->acache.s_same = stream->code_table_desc->same_modes;
 
-  if ((ret = xd3_encode_init (stream))) { return ret; }
+  if ((ret = xd3_encode_init_partial (stream))) { return ret; }
 
   addrs = xd3_alloc (stream, sizeof (usize_t), ADDR_CACHE_ROUNDS);
   modes = xd3_alloc (stream, sizeof (uint8_t), ADDR_CACHE_ROUNDS);
@@ -758,7 +758,7 @@ test_compress_text (xd3_stream  *stream,
 
   (*encoded_size) = 0;
 
-  xd3_set_appheader (stream, test_apphead, strlen (test_apphead));
+  xd3_set_appheader (stream, test_apphead, strlen ((char*) test_apphead));
 
   if ((ret = xd3_encode_stream (stream, test_text, sizeof (test_text),
                                 encoded, encoded_size, 4*sizeof (test_text)))) { goto fail; }
@@ -830,8 +830,8 @@ test_decompress_text (xd3_stream *stream, uint8_t *enc, usize_t enc_size, usize_
 
   if ((ret = xd3_get_appheader (stream, & apphead, & apphead_size))) { goto fail; }
 
-  if (apphead_size != strlen (test_apphead) ||
-      memcmp (apphead, test_apphead, strlen (test_apphead)) != 0)
+  if (apphead_size != strlen ((char*) test_apphead) ||
+      memcmp (apphead, test_apphead, strlen ((char*) test_apphead)) != 0)
     {
       stream->msg = "incorrect appheader";
       ret = XD3_INTERNAL;
@@ -2459,7 +2459,7 @@ test_string_matching (xd3_stream *stream, int ignore)
       config.smatcher_soft.long_enough  = 10;
 
       if ((ret = xd3_config_stream (stream, & config))) { return ret; }
-      if ((ret = xd3_encode_init (stream))) { return ret; }
+      if ((ret = xd3_encode_init_full (stream))) { return ret; }
 
       xd3_avail_input (stream, (uint8_t*)test->input, len);
 
@@ -2686,7 +2686,7 @@ xd3_selftest (void)
 #ifndef WIN32
   if (getuid() == 0)
     {
-      DP(RINT "This test should not be run as root.\n");
+      DP(RINT "xdelta3: This test should not be run as root.\n");
       ret = XD3_INVALID;
       goto failure;
     }
diff --git a/xdelta3/xdelta3-test.py b/xdelta3/xdelta3-test.py
index 2c37030..7fcc3d5 100755
--- a/xdelta3/xdelta3-test.py
+++ b/xdelta3/xdelta3-test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2.4
+#!/usr/bin/python2.5
 # xdelta 3 - delta compression tools and library
 # Copyright (C) 2003, 2006, 2007.  Joshua P. MacDonald
 #
@@ -142,6 +142,14 @@ assert result1 == 0 and result9 == 0
 assert len(level1) > len(level9)
 
 #
-#
+# Issue 65
+print 'encode: 65 ...'
+source = 'Hello World' 
+target = 'Hello everyone' 
+result, patch = xdelta3.xd3_encode_memory(target, source, len(target))
+assert result != 0
+
+result, patch = xdelta3.xd3_encode_memory(target, source, 2 * len(target))
+assert result == 0
 
 print 'PASS'
diff --git a/xdelta3/xdelta3.c b/xdelta3/xdelta3.c
index 1800c90..7e798bd 100644
--- a/xdelta3/xdelta3.c
+++ b/xdelta3/xdelta3.c
@@ -1908,6 +1908,16 @@ xd3_sizeof_uint64_t (uint64_t num)
 static int
 xd3_alloc_cache (xd3_stream *stream)
 {
+  if (stream->acache.near_array != NULL)
+    {
+      xd3_free (stream, stream->acache.near_array);
+    }
+
+  if (stream->acache.same_array != NULL)
+    {
+      xd3_free (stream, stream->acache.same_array);
+    }
+
   if (((stream->acache.s_near > 0) &&
        (stream->acache.near_array = (usize_t*)
         xd3_alloc (stream, stream->acache.s_near, sizeof (usize_t)))
@@ -2087,6 +2097,8 @@ xd3_alloc (xd3_stream *stream,
   if (a != NULL)
     {
       IF_DEBUG (stream->alloc_cnt += 1);
+      IF_DEBUG1 (DP(RINT "[stream %p malloc] size %u ptr %p\n", 
+                    stream, elts * size, a));
     }
   else
     {
@@ -2104,6 +2116,8 @@ xd3_free (xd3_stream *stream,
     {
       IF_DEBUG (stream->free_cnt += 1);
       XD3_ASSERT (stream->free_cnt <= stream->alloc_cnt);
+      IF_DEBUG1 (DP(RINT "[stream %p free] %p\n", 
+                    stream, ptr));
       stream->free (stream->opaque, ptr);
     }
 }
@@ -2781,10 +2795,11 @@ xd3_iopt_finish_encoding (xd3_stream *stream, xd3_rinst *inst)
             stream->l_tcpy += inst->size;
           }
 
-        XD3_ASSERT (inst->size >= MIN_MATCH);
-
+        /* Note: used to assert inst->size >= MIN_MATCH, but not true
+         * for merge operations & identical match heuristics. */
         /* the "here" position is always offset by taroff */
-        if ((ret = xd3_encode_address (stream, addr, inst->pos + stream->taroff, & inst->type)))
+        if ((ret = xd3_encode_address (stream, addr, inst->pos + stream->taroff, 
+                                       & inst->type)))
           {
             return ret;
           }
@@ -3262,8 +3277,6 @@ xd3_emit_run (xd3_stream *stream, usize_t pos, usize_t size, uint8_t run_c)
   xd3_rinst* ri;
   int ret;
 
-  XD3_ASSERT (pos + size <= stream->avail_in);
-
   if ((ret = xd3_iopt_get_slot (stream, & ri))) { return ret; }
 
   ri->type = XD3_RUN;
@@ -3283,8 +3296,6 @@ xd3_found_match (xd3_stream *stream, usize_t pos,
   xd3_rinst* ri;
   int ret;
 
-  XD3_ASSERT (pos + size <= stream->avail_in);
-
   if ((ret = xd3_iopt_get_slot (stream, & ri))) { return ret; }
 
   ri->type = XD3_CPY;
@@ -3574,12 +3585,43 @@ xd3_alloc_iopt (xd3_stream *stream, int elts)
   return 0;
 }
 
-/* This function allocates the encoder output buffers. */
+/* This function allocates all memory initially used by the encoder. */
 static int
-xd3_encode_init_buffers (xd3_stream *stream)
+xd3_encode_init (xd3_stream *stream, int full_init)
 {
   int i;
 
+  if (full_init)
+    {
+      int large_comp = (stream->src != NULL);
+      int small_comp = ! (stream->flags & XD3_NOCOMPRESS);
+
+      /* Memory allocations for checksum tables are delayed until
+       * xd3_string_match_init in the first call to string_match--that way
+       * identical or short inputs require no table allocation. */
+      if (large_comp)
+        {
+          usize_t hash_values = (stream->srcwin_maxsz / stream->smatcher.large_step);
+
+          xd3_size_hashtable (stream,
+                              hash_values,
+                              & stream->large_hash);
+        }
+
+      if (small_comp)
+        {
+          /* TODO: This is under devel: used to have min(sprevsz) here, which sort
+           * of makes sense, but observed fast performance w/ larger tables, which
+           * also sort of makes sense. @@@ */
+          usize_t hash_values = stream->winsize;
+
+          xd3_size_hashtable (stream,
+                              hash_values,
+                              & stream->small_hash);
+        }
+    }
+
+  /* data buffers */
   for (i = 0; i < ENC_SECTS; i += 1)
     {
       if ((stream->enc_heads[i] =
@@ -3590,43 +3632,6 @@ xd3_encode_init_buffers (xd3_stream *stream)
         }
     }
 
-  return 0;
-}
-
-/* This function allocates all memory initially used by the encoder. */
-int
-xd3_encode_init (xd3_stream *stream)
-{
-  int large_comp = (stream->src != NULL);
-  int small_comp = ! (stream->flags & XD3_NOCOMPRESS);
-
-  /* Memory allocations for checksum tables are delayed until
-   * xd3_string_match_init in the first call to string_match--that way
-   * identical or short inputs require no table allocation. */
-  if (large_comp)
-    {
-      usize_t hash_values = (stream->srcwin_maxsz / stream->smatcher.large_step);
-
-      xd3_size_hashtable (stream,
-                          hash_values,
-                          & stream->large_hash);
-    }
-
-  if (small_comp)
-    {
-      /* TODO: This is under devel: used to have min(sprevsz) here, which sort
-       * of makes sense, but observed fast performance w/ larger tables, which
-       * also sort of makes sense. @@@ */
-      usize_t hash_values = stream->winsize;
-
-      xd3_size_hashtable (stream,
-                          hash_values,
-                          & stream->small_hash);
-    }
-
-  /* data buffers */
-  if (xd3_encode_init_buffers(stream) != 0) { goto fail; }
-
   /* iopt buffer */
   xd3_rlist_init (& stream->iopt_used);
   xd3_rlist_init (& stream->iopt_free);
@@ -3648,6 +3653,18 @@ xd3_encode_init (xd3_stream *stream)
   return ENOMEM;
 }
 
+int
+xd3_encode_init_full (xd3_stream *stream)
+{
+  return xd3_encode_init (stream, 1);
+}
+
+int
+xd3_encode_init_partial (xd3_stream *stream)
+{
+  return xd3_encode_init (stream, 0);
+}
+
 /* Called after the ENC_POSTOUT state, this puts the output buffers
  * back into separate lists and re-initializes some variables.  (The
  * output lists were spliced together during the ENC_FLUSH state.) */
@@ -3708,7 +3725,7 @@ xd3_encode_input (xd3_stream *stream)
     {
     case ENC_INIT:
       /* Only reached on first time through: memory setup. */
-      if ((ret = xd3_encode_init (stream))) { return ret; }
+      if ((ret = xd3_encode_init_full (stream))) { return ret; }
 
       stream->enc_state = ENC_INPUT;
 
diff --git a/xdelta3/xdelta3.h b/xdelta3/xdelta3.h
index 4d04b49..9b009f1 100644
--- a/xdelta3/xdelta3.h
+++ b/xdelta3/xdelta3.h
@@ -1159,13 +1159,13 @@ int     xd3_decoder_needs_source (xd3_stream *stream);
 /* To generate a VCDIFF encoded delta with xd3_encode_init() from
  * another format, use:
  *
- *   xd3_encode_init() -- initialze encoder state
+ *   xd3_encode_init_partial() -- initialze encoder state (w/o hash tables)
  *   xd3_init_cache() -- reset VCDIFF address cache
  *   xd3_found_match() -- to report a copy instruction
  *
  * set stream->enc_state to ENC_INSTR and call xd3_encode_input as usual.
  */
-int xd3_encode_init (xd3_stream *stream);
+int xd3_encode_init_partial (xd3_stream *stream);
 void xd3_init_cache (xd3_addr_cache* acache);
 int xd3_found_match (xd3_stream *stream,
                      usize_t pos, usize_t size,