Implmeent xd3_encode_memory() and xd3_decode_memory(), add comments.

8 files changed, 344 insertions, 135 deletions

diff --git a/xdelta3/Makefile b/xdelta3/Makefile
index ea86e3d..b300310 100755
--- a/xdelta3/Makefile
+++ b/xdelta3/Makefile
@@ -13,13 +13,19 @@ SOURCES = xdelta3-cfgs.h \
           xdelta3.c \
           xdelta3.h
 
-TARGETS = xdelta3 xdelta3-debug xdelta3-64 xdelta3-everything \
-          xdelta3-Opg xdelta3-64-O xdelta3-Op xdelta3-O \
+TARGETS = xdelta3-debug \
+          xdelta3 \
+          xdelta3-32 \
+          xdelta3-64 \
+          xdelta3-everything \
+          xdelta3-Opg \
+          xdelta3-64-O \
+          xdelta3-Op \
           xdelta3-decoder xdelta3-decoder-nomain.o \
-          $(PYTGT) \
           xdelta3-nosec.o xdelta3-all.o xdelta3-fgk.o xdelta3-djw.o \
           xdelta3-noext xdelta3-tools xdelta3-tune \
-          xdelta3-notools
+          xdelta3-notools \
+          $(PYTGT) \
 
 PYTHON = python
 PYTGT = build/lib.linux-i686-2.4/xdelta3.so
@@ -37,8 +43,13 @@ all: xdelta3-debug xdelta3 $(PYTGT)
 
 all-targets: $(TARGETS)
 
+all-targets-test: all-targets test
+
 pytgt: $(PYTGT)
 
+test:
+        ./xdelta3-debug test
+
 tar:
         tar --exclude ".svn" -czf /tmp/$(RELDIR)-tmp.tar.gz $(SOURCES) $(PYFILES) $(EXTRA)
         rm -rf /tmp/$(RELDIR)
@@ -81,8 +92,8 @@ xdelta3-decoder-nomain.o: $(SOURCES) linkxd3lib.c
             -o xdelta3-decoder-nomain.o
         strip xdelta3-decoder-nomain.o
 
-xdelta3-O: $(SOURCES)
-        $(CC) -g -O2 -Wall -Wshadow xdelta3.c -o xdelta3-O -DXD3_MAIN=1 -DSECONDARY_DJW=1 -DREGRESSION_TEST=1 -lm
+xdelta3-32: $(SOURCES)
+        $(CC) -g -O2 -Wall -Wshadow xdelta3.c -o xdelta3-32 -DXD3_MAIN=1 -DSECONDARY_DJW=1 -DREGRESSION_TEST=1 -lm
 
 xdelta3-O++: $(SOURCES)
         $(CXX) -g -O2 -Wall -Wshadow xdelta3.c -o xdelta3-O++ -DXD3_MAIN=1 -DSECONDARY_DJW=1 -DREGRESSION_TEST=1 -lm
diff --git a/xdelta3/linkxd3lib.c b/xdelta3/linkxd3lib.c
index feed876..6064467 100755
--- a/xdelta3/linkxd3lib.c
+++ b/xdelta3/linkxd3lib.c
@@ -27,7 +27,7 @@ int main() {
   use (xd3_set_source (& stream, & source));
   xd3_set_flags (& stream, 0);
   
-  use (xd3_decode_completely_stream (& stream, NULL, 0, NULL, NULL, 0));
+  use (xd3_decode_stream (& stream, NULL, 0, NULL, NULL, 0));
   use (xd3_decode_input (&stream));
   use (xd3_decoder_needs_source (& stream));
   use (xd3_get_appheader (& stream, NULL, NULL));
@@ -37,7 +37,7 @@ int main() {
                              
 #if XD3_ENCODER
   use (xd3_encode_input (&stream));
-  use (xd3_encode_completely_stream (& stream, NULL, 0, NULL, NULL, 0));
+  use (xd3_encode_stream (& stream, NULL, 0, NULL, NULL, 0));
   use (xd3_set_appheader (& stream));
   use (xd3_encoder_used_source (& stream));
   use (xd3_encoder_srcbase (& stream));
diff --git a/xdelta3/setup.py b/xdelta3/setup.py
index 289219e..2fe3b19 100755
--- a/xdelta3/setup.py
+++ b/xdelta3/setup.py
@@ -39,6 +39,8 @@ xdelta3_ext = Extension('xdelta3',
 # $Format: "REL='$Xdelta3Version$'" $
 REL='0m'
 
+# This provides xdelta3.main(), which calls the xdelta3 command-line main()
+# from python.
 setup(name='xdelta3',
       version=REL,
       ext_modules=[xdelta3_ext])
diff --git a/xdelta3/xdelta3-regtest.py b/xdelta3/xdelta3-regtest.py
index 764eed4..656d617 100755
--- a/xdelta3/xdelta3-regtest.py
+++ b/xdelta3/xdelta3-regtest.py
@@ -46,8 +46,8 @@ MAX_RUN        = 1000 * 1000 * 10
 
 #
 #
-#RCSDIR = '/mnt/polaroid/Polaroid/orbit_linux/home/jmacd/PRCS'
-RCSDIR = 'Z:/Polaroid/orbit_linux/home/jmacd/PRCS'
+RCSDIR = '/mnt/polaroid/Polaroid/orbit_linux/home/jmacd/PRCS'
+#RCSDIR = 'Z:/Polaroid/orbit_linux/home/jmacd/PRCS'
 
 TMPDIR = '/tmp/xd3regtest.%d' % os.getpid()
 
@@ -712,15 +712,17 @@ class RandomTester:
         runner.extra = ['-I', '0', '-D', '-C', ','.join(strs)]
         result = TimeRun(runner.Runner(f1, 1, f2, 2))
 
-        print 'config %s dsize %d time %.7f in %u trials' % \
-              (' '.join(strs),
-               result.r1.dsize,
-               result.time.mean,
+        tr = RandomTestResult(config,
+                              result.time.mean,
+                              result.r1.dsize)
+
+        self.results.append(tr)
+
+        print 'Trial %d: %s in %u trials' % \
+              (trial_num++,
+               tr,
                result.trials)
 
-        self.results.append(RandomTestResult(config,
-                                             result.time.mean,
-                                             result.r1.dsize))
         return
     #end
 
@@ -782,19 +784,27 @@ def RunSpeed():
 if __name__ == "__main__":
     try:
         os.mkdir(TMPDIR)
-        #rcsf = Test()
-        #rcsf.PairsByDate(Xdelta3Pair())
-        #RunSpeed()
-        #f1, f2 = MakeBigFiles(rcsf)
-        f1 = '/tmp/big.1'
-        f2 = '/tmp/big.2'
-        test = RandomTester()
+        rcsf = Test()
+
         while 1:
-            while not test.HasEnoughResults():
-                test.RandomBigRun(f1, f2)
+            f1, f2 = MakeBigFiles(rcsf)
+            #f1 = '/tmp/big.1'
+            #f2 = '/tmp/big.2'
+            test = RandomTester()
+            while 1:
+                while not test.HasEnoughResults():
+                    test.RandomBigRun(f1, f2)
+                #end
+                test.ScoreTests()
             #end
-            test.ScoreTests()
         #end
+
+        # This tests pairwise (date-ordered) performance
+        #rcsf.PairsByDate(Xdelta3Pair())
+
+        # This tests the raw speed of 0-byte inputs
+        #RunSpeed()
+
     except CommandError:
         pass
     else:
diff --git a/xdelta3/xdelta3-test.h b/xdelta3/xdelta3-test.h
index af90a90..ea0bd49 100755
--- a/xdelta3/xdelta3-test.h
+++ b/xdelta3/xdelta3-test.h
@@ -670,7 +670,7 @@ static const uint8_t test_apphead[] = "header test";
 static int
 test_compress_text (xd3_stream  *stream,
                     uint8_t     *encoded,
-                    usize_t      *encoded_size)
+                    usize_t     *encoded_size)
 {
   int ret;
   xd3_config cfg;
@@ -701,8 +701,8 @@ test_compress_text (xd3_stream  *stream,
 
   xd3_set_appheader (stream, test_apphead, sizeof (test_apphead));
 
-  if ((ret = xd3_encode_completely_stream (stream, test_text, sizeof (test_text),
-                                    encoded, encoded_size, 4*sizeof (test_text)))) { goto fail; }
+  if ((ret = xd3_encode_stream (stream, test_text, sizeof (test_text),
+                                encoded, encoded_size, 4*sizeof (test_text)))) { goto fail; }
 
   if ((ret = xd3_close_stream (stream))) { goto fail; }
 
@@ -1384,7 +1384,7 @@ test_streaming (xd3_stream *in_stream, uint8_t *encbuf, uint8_t *decbuf, uint8_t
 
       if ((i % 200) == 199) { DOT (); }
 
-      if ((ret = xd3_process_completely_stream (& estream, xd3_encode_input, 0,
+      if ((ret = xd3_process_stream (& estream, xd3_encode_input, 0,
                                          encbuf, 1 << 20,
                                          delbuf, & delsize, 1 << 10)))
         {
@@ -1392,7 +1392,7 @@ test_streaming (xd3_stream *in_stream, uint8_t *encbuf, uint8_t *decbuf, uint8_t
           goto fail;
         }
 
-      if ((ret = xd3_process_completely_stream (& dstream, xd3_decode_input, 0,
+      if ((ret = xd3_process_stream (& dstream, xd3_decode_input, 0,
                                          delbuf, delsize,
                                          decbuf, & decsize, 1 << 20)))
         {
@@ -1921,7 +1921,7 @@ test_identical_behavior (xd3_stream *stream, int ignore)
   if ((ret = xd3_set_source (stream, & source))) { goto fail; }
 
   /* Decode. */
-  if ((ret = xd3_decode_completely_stream (stream, del, delpos, rec, & recsize, IDB_TGTSZ))) { goto fail; }
+  if ((ret = xd3_decode_stream (stream, del, delpos, rec, & recsize, IDB_TGTSZ))) { goto fail; }
 
   /* Check result size and data. */
   if (recsize != IDB_TGTSZ) { stream->msg = "wrong size reconstruction"; goto fail; }
@@ -2135,7 +2135,7 @@ test_iopt_flush_instructions (xd3_stream *stream, int ignore)
       target[tpos++] = i;
     }
 
-  if ((ret = xd3_encode_completely_stream (stream, target, tpos,
+  if ((ret = xd3_encode_stream (stream, target, tpos,
                                     delta, & delta_size, sizeof (delta))))
     {
       return ret;
@@ -2144,8 +2144,8 @@ test_iopt_flush_instructions (xd3_stream *stream, int ignore)
   xd3_free_stream(stream);
   if ((ret = xd3_config_stream (stream, & config))) { return ret; }
 
-  if ((ret = xd3_decode_completely_stream (stream, delta, delta_size,
-                                    recon, & recon_size, sizeof (recon))))
+  if ((ret = xd3_decode_stream (stream, delta, delta_size,
+                                recon, & recon_size, sizeof (recon))))
     {
       return ret;
     }
@@ -2206,6 +2206,43 @@ test_source_cksum_offset (xd3_stream *stream, int ignore)
   return 0;
 }
 
+static int
+test_in_memory (xd3_stream *stream, int ignore)
+{
+  // test_text is 256 bytes
+  char ibuf[sizeof(test_text)];
+  char dbuf[sizeof(test_text)];
+  char obuf[sizeof(test_text)];
+  int size = sizeof(test_text);
+  int dsize, osize;
+  int r1, r2;
+  int eflags = SECONDARY_DJW ? XD3_SEC_DJW : 0;
+
+  memcpy(ibuf, test_text, size);
+  memset(ibuf + 128, 0, 16);
+
+  r1 = xd3_encode_memory(ibuf, size,
+                         test_text, size,
+                         dbuf, &dsize, size, eflags);
+
+  r2 = xd3_decode_memory(dbuf, dsize,
+                         test_text, size,
+                         obuf, &osize, size, 0);
+
+  if (r1 != 0 || r2 != 0 || dsize >= (size/2) || dsize < 1 ||
+      osize != size) {
+    stream->msg = "encode/decode size error";
+    return XD3_INTERNAL;
+  }
+
+  if (memcmp(obuf, ibuf, size) != 0) {
+    stream->msg = "encode/decode data error";
+    return XD3_INTERNAL;
+  }
+
+  return 0;
+}
+
 /******************************************************************************************
  TEST MAIN
  ******************************************************************************************/
@@ -2248,6 +2285,7 @@ xd3_selftest (void)
   IF_GENCODETBL (DO_TEST (choose_instruction, XD3_ALT_CODE_TABLE, 0));
   IF_GENCODETBL (DO_TEST (encode_code_table, 0, 0));
 
+  DO_TEST (in_memory, 0, 0);
   DO_TEST (identical_behavior, 0, 0);
   DO_TEST (iopt_flush_instructions, 0, 0);
   DO_TEST (source_cksum_offset, 0, 0);
diff --git a/xdelta3/xdelta3.c b/xdelta3/xdelta3.c
index aaea811..5740777 100755
--- a/xdelta3/xdelta3.c
+++ b/xdelta3/xdelta3.c
@@ -763,9 +763,7 @@ static const xd3_sec_type djw_sec_type =
  * allowing to vary the distribution of single- and
  * double-instructions and change the number of near and same copy
  * modes.  More exotic tables are only possible by extending this
- * code, but a detailed experiment would need to be carried out first,
- * probably using separate code.  I would like to experiment with a
- * double-copy instruction, for example.
+ * code.  TODO: experiment with a double-copy instruction.
  *
  * For performance reasons, both the parametrized and non-parametrized
  * versions of xd3_choose_instruction remain.  The parametrized
@@ -1209,8 +1207,8 @@ int xd3_compute_code_table_encoding (xd3_stream *in_stream, const xd3_dinst *cod
 
   if ((ret = xd3_set_source (& stream, & source))) { goto fail; }
 
-  if ((ret = xd3_encode_completely_stream (& stream, code_string, CODE_TABLE_STRING_SIZE,
-                                    comp_string, comp_string_size, CODE_TABLE_VCDIFF_SIZE))) { goto fail; }
+  if ((ret = xd3_encode_stream (& stream, code_string, CODE_TABLE_STRING_SIZE,
+                                comp_string, comp_string_size, CODE_TABLE_VCDIFF_SIZE))) { goto fail; }
 
  fail:
 
@@ -1405,7 +1403,7 @@ xd3_apply_table_encoding (xd3_stream *in_stream, const uint8_t *data, usize_t si
 
   if ((ret = xd3_config_stream (& stream, NULL)) ||
       (ret = xd3_set_source (& stream, & source)) ||
-      (ret = xd3_decode_completely_stream (& stream, data, size, code_string, & code_size, sizeof (code_string))))
+      (ret = xd3_decode_stream (& stream, data, size, code_string, & code_size, sizeof (code_string))))
     {
       in_stream->msg = stream.msg;
       goto fail;
@@ -3809,17 +3807,15 @@ xd3_encode_input (xd3_stream *stream)
  Client convenience functions
  ******************************************************************************************/
 
-/* This function invokes either encode or decode to and from in-memory arrays.  The output array
- * must be large enough to hold the output or else ENOSPC is returned. */
 static int
-xd3_process_completely_stream (xd3_stream    *stream,
-                        int          (*func) (xd3_stream *),
-                        int            close_stream,
-                        const uint8_t *input,
-                        usize_t         input_size,
-                        uint8_t       *output,
-                        usize_t        *output_size,
-                        usize_t         avail_size)
+xd3_process_stream (xd3_stream    *stream,
+                    int          (*func) (xd3_stream *),
+                    int            close_stream,
+                    const uint8_t *input,
+                    usize_t        input_size,
+                    uint8_t       *output,
+                    usize_t       *output_size,
+                    usize_t        output_size_max)
 {
   (*output_size) = 0;
 
@@ -3842,13 +3838,17 @@ xd3_process_completely_stream (xd3_stream    *stream,
             stream->msg = "stream requires source input";
             return XD3_INTERNAL;
           }
-        case 0: /* there is no plain "success" return for xd3_encode/decode */
-          XD3_ASSERT (ret != 0);
+        case 0:
+          {
+            /* xd3_encode_input/xd3_decode_input never return 0 */
+            stream->msg = "invalid return: 0";
+            return XD3_INTERNAL;
+          }
         default:
           return ret;
         }
 
-      if (*output_size + stream->avail_out > avail_size)
+      if (*output_size + stream->avail_out > output_size_max)
         {
           stream->msg = "insufficient output space";
           return ENOSPC;
@@ -3864,34 +3864,129 @@ xd3_process_completely_stream (xd3_stream    *stream,
   return (close_stream == 0) ? 0 : xd3_close_stream (stream);
 }
 
+static int
+xd3_process_memory (int          (*func) (xd3_stream *),
+                    int            close_stream,
+                    const uint8_t *input,
+                    usize_t        input_size,
+                    const uint8_t *source,
+                    usize_t        source_size,
+                    uint8_t       *output,
+                    usize_t       *output_size,
+                    usize_t        output_size_max,
+                    int            flags) {
+  xd3_stream stream;
+  xd3_config config;
+  xd3_source src;
+  int ret;
+
+  /* TODO: for small inputs, the xd3_stream could be configured to allocate MUCH less
+   * memory.  Most of the allocations sizes are defaulted (see xdelta3.h), and assume
+   * large inputs. */
+  memset (& stream, 0, sizeof (stream));
+  memset (& config, 0, sizeof (config));
+
+  config.flags = flags;
+  config.srcwin_maxsz = source_size;
+  config.winsize = input_size;
+
+  if ((ret = xd3_config_stream (&stream, &config)) != 0)
+    {
+      goto exit;
+    }
+
+  if (source != NULL)
+    {
+      memset (& src, 0, sizeof (src));
+      src.size = source_size;
+      src.blksize = source_size;
+      src.onblk = source_size;
+      src.curblk = source;
+      src.curblkno = 0;
+
+      if ((ret = xd3_set_source (&stream, &src)) != 0)
+        {
+          goto exit;
+        }
+    }
+
+  if ((ret = xd3_process_stream (& stream,
+                                 func, 1,
+                                 input, input_size,
+                                 output,
+                                 output_size,
+                                 output_size_max)) != 0)
+    {
+      goto exit;
+    }
+
+ exit:
+  xd3_free_stream(&stream);
+  return ret;
+}
+
 int
-xd3_decode_completely_stream (xd3_stream    *stream,
-                       const uint8_t *input,
-                       usize_t         input_size,
-                       uint8_t       *output,
-                       usize_t        *output_size,
-                       usize_t         avail_size)
+xd3_decode_stream (xd3_stream    *stream,
+                   const uint8_t *input,
+                   usize_t         input_size,
+                   uint8_t       *output,
+                   usize_t        *output_size,
+                   usize_t         output_size_max)
 {
-  return xd3_process_completely_stream (stream, & xd3_decode_input, 1,
-                                 input, input_size,
-                                 output, output_size, avail_size);
+  return xd3_process_stream (stream, & xd3_decode_input, 1,
+                             input, input_size,
+                             output, output_size, output_size_max);
 }
 
+int
+xd3_decode_memory (const uint8_t *input,
+                   usize_t        input_size,
+                   const uint8_t *source,
+                   usize_t        source_size,
+                   uint8_t       *output,
+                   usize_t       *output_size,
+                   usize_t        output_size_max,
+                   int            flags) {
+  return xd3_process_memory (& xd3_decode_input, 1,
+                             input, input_size,
+                             source, source_size,
+                             output, output_size, output_size_max,
+                             flags);
+}
+
+
 #if XD3_ENCODER
 int
-xd3_encode_completely_stream (xd3_stream    *stream,
-                       const uint8_t *input,
-                       usize_t         input_size,
-                       uint8_t       *output,
-                       usize_t        *output_size,
-                       usize_t         avail_size)
+xd3_encode_stream (xd3_stream    *stream,
+                   const uint8_t *input,
+                   usize_t         input_size,
+                   uint8_t       *output,
+                   usize_t        *output_size,
+                   usize_t         output_size_max)
 {
-  return xd3_process_completely_stream (stream, & xd3_encode_input, 1,
-                                 input, input_size,
-                                 output, output_size, avail_size);
+  return xd3_process_stream (stream, & xd3_encode_input, 1,
+                             input, input_size,
+                             output, output_size, output_size_max);
+}
+
+int
+xd3_encode_memory (const uint8_t *input,
+                   usize_t        input_size,
+                   const uint8_t *source,
+                   usize_t        source_size,
+                   uint8_t       *output,
+                   usize_t        *output_size,
+                   usize_t        output_size_max,
+                   int            flags) {
+  return xd3_process_memory (& xd3_encode_input, 1,
+                             input, input_size,
+                             source, source_size,
+                             output, output_size, output_size_max,
+                             flags);
 }
 #endif
 
+
 /******************************************************************************************
  String matching helpers
  ******************************************************************************************/
@@ -3996,7 +4091,7 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
       stream->srcwin_cksum_pos = stream->maxsrcaddr;
     }
 
-  if (logical_input_cksum_pos < stream->srcwin_cksum_pos) 
+  if (logical_input_cksum_pos < stream->srcwin_cksum_pos)
     {
       logical_input_cksum_pos = stream->srcwin_cksum_pos;
     }
@@ -4004,7 +4099,7 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
   /* Advance an extra srcwin_size bytes */
   logical_input_cksum_pos += stream->srcwin_size;
 
-  IF_DEBUG1 (P(RINT "[srcwin_move_point] T=%"Q"u S=%"Q"u/%"Q"u\n", 
+  IF_DEBUG1 (P(RINT "[srcwin_move_point] T=%"Q"u S=%"Q"u/%"Q"u\n",
                stream->total_in + stream->input_position,
                stream->srcwin_cksum_pos,
                logical_input_cksum_pos));
@@ -4038,6 +4133,9 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
       diff = logical_input_cksum_pos - stream->srcwin_cksum_pos;
       onblk = min(blkoff + diff, onblk);
 
+      /* TODO: This block needs to be included in the template pass... duh.
+       */
+
       /* Note: I experimented with rewriting this block to use LARGE_CKSUM_UPDATE()
        * instead of recalculating the cksum every N bytes.  It seemed to make performance
        * worse. */
@@ -4066,7 +4164,7 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
 
 /* This function handles the 32/64bit ambiguity -- file positions are 64bit but the hash
  * table for source-offsets is 32bit. */
-static xoff_t 
+static xoff_t
 xd3_source_cksum_offset(xd3_stream *stream, usize_t low)
 {
   xoff_t scp = stream->srcwin_cksum_pos;
@@ -4179,7 +4277,7 @@ xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos)
       greedy_or_not = stream->unencoded_offset;
     }
 
-  
+
 
   /* Backward target match limit. */
   XD3_ASSERT (stream->input_position >= greedy_or_not);
@@ -4295,7 +4393,7 @@ xd3_source_extend_match (xd3_stream *stream)
                 {
                   break;
                 }
-                
+
               /* could be a XD3_GETSRCBLK failure. */
               return ret;
             }
@@ -4387,7 +4485,7 @@ xd3_source_extend_match (xd3_stream *stream)
 
       /* Correct the variables to remove match_back from the equation. */
       // IT'S A BUG!
-      
+
       usize_t target_position = stream->input_position - stream->match_back;
       usize_t match_length   = stream->match_back      + stream->match_fwd;
       xoff_t match_position  = stream->match_srcpos    - stream->match_back;
@@ -4554,6 +4652,8 @@ xd3_check_smatch (const uint8_t *ref0, const uint8_t *inp0,
  * comparison to zlib) in order to support the PROMOTE decision, which is to prefer the
  * most recently used matching address of a certain string to aid the VCDIFF same cache.
  *
+ * --- TODO: declare the PROMOTE experiment a failure -- remove the extra LIST --
+ *
  * Weak reasoning? it's time to modularize this routine...?  Let's say the PROMOTE
  * feature supported by this slow data structure contributes around 2% improvement in
  * compressed size, is there a better code table that doesn't use the SAME address cache,
diff --git a/xdelta3/xdelta3.h b/xdelta3/xdelta3.h
index fd91ac8..ca4c79a 100755
--- a/xdelta3/xdelta3.h
+++ b/xdelta3/xdelta3.h
@@ -597,6 +597,11 @@ struct _xd3_source
   const char         *name;          /* its name, for debug/print purposes */
   void               *ioh;           /* opaque handle */
 
+  /* getblk sets */
+  xoff_t              curblkno;      /* current block number: client sets after getblk request */
+  usize_t             onblk;         /* number of bytes on current block: client sets, xd3 verifies */
+  const uint8_t      *curblk;        /* current block array: client sets after getblk request */
+
   /* xd3 sets */
   usize_t             srclen;        /* length of this source window */
   xoff_t              srcbase;       /* offset of this source window in the source itself */
@@ -604,11 +609,6 @@ struct _xd3_source
   usize_t             cpyoff_blocks; /* offset of copy window in blocks */
   usize_t             cpyoff_blkoff; /* offset of copy window in blocks, remainder */
   xoff_t              getblkno;      /* request block number: xd3 sets current getblk request */
-
-  /* getblk sets */
-  xoff_t              curblkno;      /* current block number: client sets after getblk request */
-  usize_t             onblk;         /* number of bytes on current block: client sets, xd3 verifies */
-  const uint8_t      *curblk;        /* current block array: client sets after getblk request */
 };
 
 /* The primary xd3_stream object, used for encoding and decoding.  You may access only two
@@ -812,11 +812,86 @@ struct _xd3_stream
  PUBLIC FUNCTIONS
  ******************************************************************************************/
 
-/* The two I/O disciplines, encode and decode, have similar stream semantics.  It is
- * recommended that applications use the same code for compression and decompression -
- * because there are only a few differences in handling encoding/decoding.
+/* This function configures an xd3_stream using the provided in-memory input buffer,
+ * source buffer, output buffer, and flags.  The output array must be large enough or else
+ * ENOSPC will be returned.  This is the simplest in-memory encoding interface. */
+int     xd3_encode_memory (const uint8_t *input,
+                           usize_t        input_size,
+                           const uint8_t *source,
+                           usize_t        source_size,
+                           uint8_t       *output,
+                           usize_t       *output_size,
+                           usize_t        avail_output,
+                           int            flags);
+
+/* The reverse of xd3_encode_memory. */
+int     xd3_decode_memory (const uint8_t *input,
+                           usize_t        input_size,
+                           const uint8_t *source,
+                           usize_t        source_size,
+                           uint8_t       *output,
+                           usize_t       *output_size,
+                           usize_t        avail_output,
+                           int            flags);
+
+/* This function encodes an in-memory input.  Everything else about the xd3_stream is
+ * configurable.  The output array must be large enough to hold the output or else ENOSPC
+ * is returned.  The source (if any) should be set using xd3_set_source() with a
+ * single-block xd3_source.  This calls the underlying non-blocking interface,
+ * xd3_encode_input(), handling the necessary input/output states.  This method be
+ * considered a reference for any application using xd3_encode_input() directly.
+ *
+ *   xd3_stream stream;
+ *   xd3_config config;
+ *   xd3_source src;
+ * 
+ *   memset (& src, 0, sizeof (src));
+ *   memset (& stream, 0, sizeof (stream));
+ *   memset (& config, 0, sizeof (config));
  *
- * See also the xd3_avail_input() and xd3_consume_output() routines, inlined below.
+ *   if (source != NULL)
+ *     {
+ *       src.size = source_size;
+ *       src.blksize = source_size;
+ *       src.curblkno = 0;
+ *       src.onblk = source_size;
+ *       src.curblk = source;
+ *       xd3_set_source(&stream, &src);
+ *     }
+ *
+ *   config.flags = flags;
+ *   config.srcwin_maxsz = source_size;
+ *   config.winsize = input_size;
+ *
+ *   ... set smatcher, appheader, encoding-table, compression-level, etc.
+ *
+ *   xd3_config_stream(&stream, &config);
+ *   xd3_encode_stream(&stream, ...);
+ *   xd3_free_stream(&stream);
+ *
+ * DO NOT USE except for testing. These methods are allocate bad buffer sizes.
+ */
+int     xd3_encode_stream (xd3_stream    *stream,
+                           const uint8_t *input,
+                           usize_t         input_size,
+                           uint8_t       *output,
+                           usize_t        *output_size,
+                           usize_t         avail_output);
+
+/* The reverse of xd3_encode_stream. */
+int     xd3_decode_stream (xd3_stream    *stream,
+                           const uint8_t *input,
+                           usize_t        input_size,
+                           uint8_t       *output,
+                           usize_t       *output_size,
+                           usize_t        avail_size);
+
+/* This is the non-blocking interface.
+ *
+ * Handling input and output states is the same for encoding or decoding using the
+ * xd3_avail_input() and xd3_consume_output() routines, inlined below.
+ *
+ * Return values:
  *
  *   XD3_INPUT:  the process requires more input: call xd3_avail_input() then repeat
  *   XD3_OUTPUT: the process has more output: read stream->next_out, stream->avail_out,
@@ -844,14 +919,6 @@ struct _xd3_stream
  *               consistent for either encoding or decoding.
  *   XD3_GETSRCBLK: If the xd3_getblk() callback is NULL, this value is returned to
  *               initiate a non-blocking source read.
- *
- * For simple usage, see the xd3_process_completely_stream() function, which underlies
- * xd3_encode_completely_stream() and xd3_decode_completely_stream() [xdelta3.c].  For
- * real application usage, including the application header, the see command-line utility
- * [xdelta3-main.h].
- *
- * main_input() implements the command-line encode and decode as well as the optional
- * VCDIFF_TOOLS printhdr, printhdrs, and printdelta with a single loop [xdelta3-main.h].
  */
 int     xd3_decode_input  (xd3_stream    *stream);
 int     xd3_encode_input  (xd3_stream    *stream);
@@ -868,7 +935,7 @@ int     xd3_config_stream (xd3_stream    *stream,
  * resources it supplied. */
 int     xd3_close_stream (xd3_stream    *stream);
 
-/* This unconditionally closes/frees the stream, future close() will succeed.*/
+/* This unconditionally closes/frees the stream, future close() will succeed. */
 void    xd3_abort_stream (xd3_stream    *stream);
 
 /* xd3_free_stream frees all memory allocated for the stream.  The application is
@@ -884,36 +951,18 @@ void    xd3_free_stream   (xd3_stream    *stream);
 int     xd3_set_source    (xd3_stream    *stream,
                            xd3_source    *source);
 
-/* This function invokes xd3_encode_input using whole-file, in-memory inputs.  The output
- * array must be large enough to hold the output or else ENOSPC is returned. */
-int     xd3_encode_completely_stream (xd3_stream    *stream,
-                                      const uint8_t *input,
-                                      usize_t         input_size,
-                                      uint8_t       *output,
-                                      usize_t        *output_size,
-                                      usize_t         avail_output);
-
-/* This function invokes xd3_decode_input using whole-file, in-memory inputs.  The output
- * array must be large enough to hold the output or else ENOSPC is returned. */
-int     xd3_decode_completely_stream (xd3_stream    *stream,
-                                      const uint8_t *input,
-                                      usize_t         input_size,
-                                      uint8_t       *output,
-                                      usize_t        *output_size,
-                                      usize_t         avail_size);
-
 /* This should be called before the first call to xd3_encode_input() to include
  * application-specific data in the VCDIFF header. */
 void    xd3_set_appheader (xd3_stream    *stream,
                            const uint8_t *data,
-                           usize_t         size);
+                           usize_t        size);
 
 /* xd3_get_appheader may be called in the decoder after XD3_GOTHEADER.  For convenience,
  * the decoder always adds a single byte padding to the end of the application header,
  * which is set to zero in case the application header is a string. */
 int     xd3_get_appheader (xd3_stream     *stream,
                            uint8_t       **data,
-                           usize_t         *size);
+                           usize_t        *size);
 
 /* After receiving XD3_GOTHEADER, the decoder should check this function which returns 1
  * if the decoder will require source data. */
diff --git a/xdelta3/xdelta3.prj b/xdelta3/xdelta3.prj
index ae05516..684b109 100755
--- a/xdelta3/xdelta3.prj
+++ b/xdelta3/xdelta3.prj
@@ -1,12 +1,11 @@
 ;; -*- Prcs -*-
 (Created-By-Prcs-Version 1 3 4)
 (Project-Description "")
-(Project-Version xdelta3 0 9)
-(Parent-Version xdelta3 0 8)
-(Version-Log "rename encode/decode_completely to encode/decode_completely_stream
-")
+(Project-Version xdelta3 0 11)
+(Parent-Version xdelta3 0 10)
+(Version-Log "compiles on cygwin, copyright years updated, python experiments")
 (New-Version-Log "")
-(Checkin-Time "Sun, 28 Jan 2007 21:08:28 -0800")
+(Checkin-Time "Wed, 31 Jan 2007 22:47:01 -0800")
 (Checkin-Login jmacd)
 (Populate-Ignore ())
 (Project-Keywords
@@ -16,18 +15,18 @@
 (Files
   (COPYING (xdelta3/b/29_COPYING 1.1 644))
 
-  (xdelta3-cfgs.h (xdelta3/9_xdelta3-cf 1.2 744))
-  (xdelta3-decode.h (xdelta3/b/30_xdelta3-de 1.1 744))
-  (xdelta3-djw.h (xdelta3/8_xdelta3-dj 1.2 744))
-  (xdelta3-fgk.h (xdelta3/7_xdelta3-fg 1.2 744))
-  (xdelta3-list.h (xdelta3/6_xdelta3-li 1.2 644))
-  (xdelta3-main.h (xdelta3/5_xdelta3-ma 1.5 744))
-  (xdelta3-python.h (xdelta3/4_xdelta3-py 1.2 644))
-  (xdelta3-regtest.py (xdelta3/10_xdelta3-re 1.4 744))
-  (xdelta3-second.h (xdelta3/3_xdelta3-se 1.2 744))
-  (xdelta3-test.h (xdelta3/2_xdelta3-te 1.4 744))
-  (xdelta3.c (xdelta3/16_xdelta3.c 1.5 744))
-  (xdelta3.h (xdelta3/1_xdelta3.h 1.4 744))
+  (xdelta3-cfgs.h (xdelta3/9_xdelta3-cf 1.3 744))
+  (xdelta3-decode.h (xdelta3/b/30_xdelta3-de 1.3 744))
+  (xdelta3-djw.h (xdelta3/8_xdelta3-dj 1.4 744))
+  (xdelta3-fgk.h (xdelta3/7_xdelta3-fg 1.3 744))
+  (xdelta3-list.h (xdelta3/6_xdelta3-li 1.3 744))
+  (xdelta3-main.h (xdelta3/5_xdelta3-ma 1.7 744))
+  (xdelta3-python.h (xdelta3/4_xdelta3-py 1.3 744))
+  (xdelta3-regtest.py (xdelta3/10_xdelta3-re 1.6 744))
+  (xdelta3-second.h (xdelta3/3_xdelta3-se 1.3 744))
+  (xdelta3-test.h (xdelta3/2_xdelta3-te 1.5 744))
+  (xdelta3.c (xdelta3/16_xdelta3.c 1.6 744))
+  (xdelta3.h (xdelta3/1_xdelta3.h 1.5 744))
 
   (Makefile (xdelta3/0_Makefile 1.4 744))
   (setup.py (xdelta3/11_setup.py 1.2 744))
@@ -44,7 +43,7 @@
 
   (README (xdelta3/b/31_README 1.1 744))
   (badcopy.vcproj (xdelta3/b/32_badcopy.vc 1.1 744))
-  (xdelta3.vcproj (xdelta3/b/33_xdelta3.vc 1.1 744))
+  (xdelta3.vcproj (xdelta3/b/33_xdelta3.vc 1.2 744))
 )
 (Merge-Parents)
 (New-Merge-Parents)