Fixes bugs in XDELTA environment variable.

Removes stream.srcwin_size variable, uses stream.src->blksize instead, to address performance regression from xdelta1.  Changes the order of checksum computation to be like xdelta1, insert checksums in reverse to give preference to earlier matches.
Removes svn:executable from files.

27 files changed, 104 insertions, 58 deletions

diff --git a/xdelta3/COPYING b/xdelta3/COPYING
index 5b6e7c6..5b6e7c6 100755..100644
--- a/xdelta3/COPYING
+++ b/xdelta3/COPYING
diff --git a/xdelta3/Makefile b/xdelta3/Makefile
index ea48055..cb79e64 100755..100644
--- a/xdelta3/Makefile
+++ b/xdelta3/Makefile
@@ -30,22 +30,22 @@ TARGETS = xdelta3-debug \
           xdelta3-nosec.o xdelta3-all.o xdelta3-fgk.o \
           xdelta3-noext xdelta3-tools xdelta3-tune \
           xdelta3-notools \
-          $(PYTGT) \
+          xdelta3_wrap.c xdelta3.py \
+          $(PYTGT) $(SWIGTGT)
 
 PYTHON = python
 
-PYTGT = build/lib.linux-i686-2.4/xdelta3.so
-#PYTGT = build/lib.cygwin-1.5.24-i686-2.4/xdelta3main.dll
+SWIGTGT = xdelta3module.so
+SWIGTGT = xdelta3module.dll
 
-PYFILES = xdelta3-regtest.py setup.py
+PYTGT = build/lib.linux-i686-2.4/xdelta3.so
+PYTGT = build/lib.cygwin-1.5.24-i686-2.4/xdelta3main.dll
 
 EXTRA = Makefile COPYING linkxd3lib.c badcopy.c xdelta3.swig \
         draft-korn-vcdiff.txt xdelta3.vcproj badcopy.vcproj \
+        xdelta3-regtest.py xdelta3-test.py setup.py \
         xdelta3.py xdelta3_wrap.c
 
-SWIGTGT = xdelta3module.so
-#SWIGTGT = xdelta3module.dll
-
 SWIG_FLAGS = -DXD3_DEBUG=0 \
               -DXD3_USE_LARGEFILE64=1 \
               -DSECONDARY_DJW=1 \
@@ -80,7 +80,8 @@ tar:
 
 clean:
         rm -f $(TARGETS)
-        rm -rf build Debug Release
+        rm -rf build Debug Release core *.stackdump *.exe \
+                xdelta3.ncb xdelta3.suo xdelta3.sln
 
 xdelta3: $(SOURCES)
         $(CC) -O3 -Wall -Wshadow xdelta3.c -lm -o xdelta3 \
diff --git a/xdelta3/README b/xdelta3/README
index b139f7e..b139f7e 100755..100644
--- a/xdelta3/README
+++ b/xdelta3/README
diff --git a/xdelta3/badcopy.c b/xdelta3/badcopy.c
index 03abc63..03abc63 100755..100644
--- a/xdelta3/badcopy.c
+++ b/xdelta3/badcopy.c
diff --git a/xdelta3/draft-korn-vcdiff.txt b/xdelta3/draft-korn-vcdiff.txt
index 1487deb..1487deb 100755..100644
--- a/xdelta3/draft-korn-vcdiff.txt
+++ b/xdelta3/draft-korn-vcdiff.txt
diff --git a/xdelta3/linkxd3lib.c b/xdelta3/linkxd3lib.c
index 6064467..6064467 100755..100644
--- a/xdelta3/linkxd3lib.c
+++ b/xdelta3/linkxd3lib.c
diff --git a/xdelta3/rcs_junk.cc b/xdelta3/rcs_junk.cc
index ac49644..ac49644 100755..100644
--- a/xdelta3/rcs_junk.cc
+++ b/xdelta3/rcs_junk.cc
diff --git a/xdelta3/releases/xdelta30h.ppc-osx.bin b/xdelta3/releases/xdelta30h.ppc-osx.bin
index 7ce8f0d..7ce8f0d 100755..100644
--- a/xdelta3/releases/xdelta30h.ppc-osx.bin
+++ b/xdelta3/releases/xdelta30h.ppc-osx.bin
diff --git a/xdelta3/releases/xdelta30n.tar.gz b/xdelta3/releases/xdelta30n.tar.gz
index 66fadb4..66fadb4 100755..100644
--- a/xdelta3/releases/xdelta30n.tar.gz
+++ b/xdelta3/releases/xdelta30n.tar.gz
diff --git a/xdelta3/releases/xdelta30n.x86-32.exe b/xdelta3/releases/xdelta30n.x86-32.exe
index cd96077..cd96077 100755..100644
--- a/xdelta3/releases/xdelta30n.x86-32.exe
+++ b/xdelta3/releases/xdelta30n.x86-32.exe
diff --git a/xdelta3/releases/xdelta30o.tar.gz b/xdelta3/releases/xdelta30o.tar.gz
index b2de57b..b2de57b 100755..100644
--- a/xdelta3/releases/xdelta30o.tar.gz
+++ b/xdelta3/releases/xdelta30o.tar.gz
diff --git a/xdelta3/setup.py b/xdelta3/setup.py
index d7f11c4..d7f11c4 100755..100644
--- a/xdelta3/setup.py
+++ b/xdelta3/setup.py
diff --git a/xdelta3/xdelta3-cfgs.h b/xdelta3/xdelta3-cfgs.h
index c48df9f..c48df9f 100755..100644
--- a/xdelta3/xdelta3-cfgs.h
+++ b/xdelta3/xdelta3-cfgs.h
diff --git a/xdelta3/xdelta3-decode.h b/xdelta3/xdelta3-decode.h
index b1b5b88..b1b5b88 100755..100644
--- a/xdelta3/xdelta3-decode.h
+++ b/xdelta3/xdelta3-decode.h
diff --git a/xdelta3/xdelta3-djw.h b/xdelta3/xdelta3-djw.h
index deff7cb..deff7cb 100755..100644
--- a/xdelta3/xdelta3-djw.h
+++ b/xdelta3/xdelta3-djw.h
diff --git a/xdelta3/xdelta3-fgk.h b/xdelta3/xdelta3-fgk.h
index 34ee3ca..34ee3ca 100755..100644
--- a/xdelta3/xdelta3-fgk.h
+++ b/xdelta3/xdelta3-fgk.h
diff --git a/xdelta3/xdelta3-list.h b/xdelta3/xdelta3-list.h
index 8d49e45..8d49e45 100755..100644
--- a/xdelta3/xdelta3-list.h
+++ b/xdelta3/xdelta3-list.h
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h
index 1552809..7ccd63b 100755..100644
--- a/xdelta3/xdelta3-main.h
+++ b/xdelta3/xdelta3-main.h
@@ -346,7 +346,6 @@ main_config (void)
   P(RINT "XD3_ALLOCSIZE=%d\n", XD3_ALLOCSIZE);
   P(RINT "XD3_DEBUG=%d\n", XD3_DEBUG);
   P(RINT "XD3_DEFAULT_LEVEL=%d\n", XD3_DEFAULT_LEVEL);
-  P(RINT "XD3_DEFAULT_CKSUM_SIZE=%d\n", XD3_DEFAULT_CKSUM_ADVANCE);
   P(RINT "XD3_DEFAULT_IOPT_SIZE=%d\n", XD3_DEFAULT_IOPT_SIZE);
   P(RINT "XD3_DEFAULT_SPREVSZ=%d\n", XD3_DEFAULT_SPREVSZ);
   P(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ);
@@ -742,7 +741,13 @@ main_file_open (main_file *xfile, const char* name, int mode)
 
   xfile->mode = mode;
 
+  XD3_ASSERT (name != NULL);
   XD3_ASSERT (! main_file_isopen (xfile));
+  if (name[0] == 0)
+    {
+      XPR(NT "invalid file name: empty string\n");
+      return XD3_INVALID;
+    }
 
 #if XD3_STDIO
   xfile->file = fopen (name, XOPEN_STDIO);
@@ -1840,6 +1845,12 @@ main_get_appheader_params (main_file *file, char **parsed, int output, const cha
 
       if (other->filename != NULL) {
         /* Take directory from the other file, if it has one. */
+        /* TODO: This results in nonsense names like /dev/foo.tar.gz
+         * and probably the filename-default logic interferes with
+         * multi-file operation and the standard file extension?
+         * Possibly the name header is bad, should be off by default.
+         * Possibly we just want to remember external/compression
+         * settings. */
         char *last_slash = strrchr(other->filename, '/');
 
         if (last_slash != NULL) {
@@ -2029,17 +2040,25 @@ main_set_source (xd3_stream *stream, int cmd, main_file *sfile, xd3_source *sour
       sfile->nread = 0;
       source->size = UINT64_MAX;
     }
-  else if ((ret = main_file_open (sfile, sfile->filename, XO_READ)) ||
-           (ret = main_file_stat (sfile, & source->size, 1)))
+  else
     {
-      goto error;
+      if (option_verbose)
+        {
+          XPR(NT "source file: %s\n", sfile->filename);
+        }
+
+      if ((ret = main_file_open (sfile, sfile->filename, XO_READ)) ||
+          (ret = main_file_stat (sfile, & source->size, 1)))
+        {
+          goto error;
+        }
     }
 
   source->name     = sfile->filename;
   source->ioh      = sfile;
   source->curblkno = (xoff_t) -1;
   source->curblk   = NULL;
-
+  
 #if EXTERNAL_COMPRESSION
   if (option_decompress_inputs)
     {
@@ -2650,7 +2669,7 @@ main_input (xd3_cmd     cmd,
                       }
                     
                     /* Limited instruction buffer size affects source copies */
-                    if (stream.i_slots_used > stream.iopt_size)
+                    if (option_verbose > 1 && stream.i_slots_used > stream.iopt_size)
                       {
                         XPR(NT "warning: input position %"Q"u overflowed instruction buffer, "
                             "needed %u (vs. %u)\n",
@@ -2831,7 +2850,7 @@ setup_environment (int argc,
   (*env_free) = main_malloc(strlen(v) + 1);
   strcpy(*env_free, v);
 
-  /* Space needed for new argv: count spaces */
+  /* Space needed for extra args, at least # of spaces */
   n = argc + 1;
   for (p = *env_free; *p != 0; ) {
     if (*p++ == ' ') {
@@ -2839,7 +2858,6 @@ setup_environment (int argc,
     }
   }
 
-  (*argc_out) = n;
   (*argv_free) = main_malloc(sizeof(char*) * (n + 1));
   (*argv_out) = (*argv_free);
   (*argv_out)[0] = argv[0];
@@ -2851,7 +2869,7 @@ setup_environment (int argc,
     while (*p != ' ' && *p != 0) {
       p++;
     }
-    if (*p == ' ') {
+    while (*p == ' ') {
       *p++ = 0;
     }
   }
@@ -2860,7 +2878,11 @@ setup_environment (int argc,
     (*argv_out)[i++] = argv[i0];
   }
 
-  XD3_ASSERT (i == n);
+  /* Counting spaces is an upper bound, argv stays NULL terminated. */
+  (*argc_out) = i;
+  while (i <= n) {
+    (*argv_out)[i++] = NULL;
+  }
 }
 
 int
@@ -2881,8 +2903,8 @@ main (int argc, char **argv)
   char *sfilename;
   int env_argc;
   char **env_argv;
-  char **free_argv;  /* malloced */
-  char *free_value;  /* malloced */
+  char **free_argv;  /* malloc() in setup_environment() */
+  char *free_value;  /* malloc() in setup_environment() */
   int ret;
 
 #ifdef _WIN32
@@ -3248,7 +3270,7 @@ main (int argc, char **argv)
 static int
 main_help (void)
 {
-  /* TODO: update www/xdelta3-cmdline.html */
+  /* Note: update wiki when command-line features change */
   main_version ();
   P(RINT "usage: xdelta3 [command/options] [input [output]]\n");
   P(RINT "special command names:\n");
@@ -3296,5 +3318,9 @@ main_help (void)
   P(RINT "   -P           repeat count (for profiling)\n");
   P(RINT "   -T           use alternate code table\n");
 #endif
+  P(RINT "the XDELTA environment variable may contain extra args:\n");
+  P(RINT "   XDELTA=\"-s source-x.y.tar.gz\" \\\n");
+  P(RINT "   tar --use-compress-program=xdelta3 \\\n");
+  P(RINT "       -cf target-x.z.tar.gz.vcdiff target-x.y/\n");
   return EXIT_FAILURE;
 }
diff --git a/xdelta3/xdelta3-python.h b/xdelta3/xdelta3-python.h
index a91b7f2..a91b7f2 100755..100644
--- a/xdelta3/xdelta3-python.h
+++ b/xdelta3/xdelta3-python.h
diff --git a/xdelta3/xdelta3-regtest.py b/xdelta3/xdelta3-regtest.py
index db6f06f..928a5f0 100755
--- a/xdelta3/xdelta3-regtest.py
+++ b/xdelta3/xdelta3-regtest.py
@@ -85,6 +85,8 @@ def INPUT_SPEC(rand):
 RCSDIR = '/tmp/PRCS_read_copy'
 #RCSDIR = 'G:/jmacd/PRCS'
 
+SAMPLEDIR = "C:/sample_data/WESNOTH_tmp/tar'
+
 TMPDIR = '/tmp/xd3regtest.%d' % os.getpid()
 
 RUNFILE = os.path.join(TMPDIR, 'run')
diff --git a/xdelta3/xdelta3-second.h b/xdelta3/xdelta3-second.h
index c64ffe0..c64ffe0 100755..100644
--- a/xdelta3/xdelta3-second.h
+++ b/xdelta3/xdelta3-second.h
diff --git a/xdelta3/xdelta3-test.h b/xdelta3/xdelta3-test.h
index f4d5972..f4d5972 100755..100644
--- a/xdelta3/xdelta3-test.h
+++ b/xdelta3/xdelta3-test.h
diff --git a/xdelta3/xdelta3.c b/xdelta3/xdelta3.c
index 4e40ba4..cc11f5b 100755..100644
--- a/xdelta3/xdelta3.c
+++ b/xdelta3/xdelta3.c
@@ -1197,7 +1197,6 @@ int xd3_compute_code_table_encoding (xd3_stream *in_stream, const xd3_dinst *cod
 
   /* Be exhaustive. */
   config.sprevsz = 1<<11;
-  config.srcwin_size = CODE_TABLE_STRING_SIZE;
   config.srcwin_maxsz = CODE_TABLE_STRING_SIZE;
 
   config.smatch_cfg = XD3_SMATCH_SOFT;
@@ -2416,15 +2415,17 @@ xd3_config_stream(xd3_stream *stream,
 
   stream->winsize   = config->winsize   ? config->winsize : XD3_DEFAULT_WINSIZE;
   stream->sprevsz   = config->sprevsz   ? config->sprevsz : XD3_DEFAULT_SPREVSZ;
-  stream->iopt_size = config->iopt_size ? config->iopt_size : XD3_DEFAULT_IOPT_SIZE;
-  stream->srcwin_size  = config->srcwin_size ? config->srcwin_size : XD3_DEFAULT_CKSUM_ADVANCE;
   stream->srcwin_maxsz = config->srcwin_maxsz ? config->srcwin_maxsz : XD3_DEFAULT_SRCWINSZ;
 
-  if (stream->iopt_size == 0)
+  if (config->iopt_size == 0)
     {
       stream->iopt_size = XD3_ALLOCSIZE / sizeof(xd3_rinst);
       stream->iopt_unlimited = 1;
     }
+  else
+    {
+      stream->iopt_size = XD3_DEFAULT_IOPT_SIZE;
+    }
 
   stream->getblk    = config->getblk;
   stream->alloc     = config->alloc ? config->alloc : __xd3_alloc_func;
@@ -4210,7 +4211,7 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
     {
       *next_move_point = stream->input_position +
         (usize_t)(stream->srcwin_cksum_pos - logical_input_cksum_pos);
-      return 0;
+      goto finished;
     }
 
   /* If the stream has matched beyond the srcwin_cksum_pos (good), we shouldn't
@@ -4225,8 +4226,21 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
       logical_input_cksum_pos = stream->srcwin_cksum_pos;
     }
 
-  /* Advance an extra srcwin_size bytes */
-  logical_input_cksum_pos += stream->srcwin_size;
+  /* Advance at least one source block.  For the command-line, this
+   * means we read to EOF for files smaller than the source window.
+   * Note that this term (stream->src->blksize) is used again below.
+   *
+   * TODO: This is possibly a post-3.0o performance regression.  I
+   * have examples of files that perform better w/ xdelta1.  Note that
+   * xdelta1 fills its cksum table in reverse so that earlier matches
+   * are preferred.  Note that xdelta3 is designed to avoid
+   * unnecessary computation, which this might be if we have long
+   * matches ahead. OTOH, sometimes tar scrambles a file such that you
+   * copy from the end to the beginning right away, and this logic at
+   * least helps those cases.  Used to use 16K here, instead of
+   * stream->src->blksize.
+   */
+  logical_input_cksum_pos += stream->src->blksize;
 
   IF_DEBUG1 (P(RINT "[srcwin_move_point] T=%"Q"u S=%"Q"u/%"Q"u\n",
                stream->total_in + stream->input_position,
@@ -4240,7 +4254,6 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
       usize_t blkoff = stream->srcwin_cksum_pos % stream->src->blksize;
       usize_t onblk = xd3_bytes_on_srcblk (stream->src, blkno);
       int ret;
-      int diff;
 
       if (blkoff + stream->smatcher.large_look > onblk)
         {
@@ -4258,34 +4271,44 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
           return ret;
         }
 
-      onblk -= stream->smatcher.large_look;
-      diff = logical_input_cksum_pos - stream->srcwin_cksum_pos;
-      onblk = min(blkoff + diff, onblk);
-
-      /* Note: I experimented with rewriting this block to use LARGE_CKSUM_UPDATE()
-       * instead of recalculating the cksum every N bytes.  It seemed to make performance
-       * worse (except obviously for step == 1, not worth extra complexity) */
-      while (blkoff <= onblk)
+      /* Note: I experimented with rewriting this block to use
+       * LARGE_CKSUM_UPDATE() instead of recalculating the cksum every
+       * N bytes.  It seemed to make performance worse (except
+       * obviously for step == 1, not worth extra complexity).
+       *
+       * Prior to 3.0p, this loop inserted cksums forwards, now it
+       * goes in reverse like xdelta1. */
+      while (onblk >= stream->smatcher.large_step &&
+             onblk >= stream->smatcher.large_look)
         {
-          /* It's a big win to have the compiler optimize this loop w/ fi. */
-          uint32_t cksum = xd3_lcksum (stream->src->curblk + blkoff, stream->smatcher.large_look);
-          usize_t hval = xd3_checksum_hash (& stream->large_hash, cksum);
+          onblk -= stream->smatcher.large_step;
 
-          stream->large_table[hval] = stream->srcwin_cksum_pos + HASH_CKOFFSET;
+          uint32_t cksum = xd3_lcksum (stream->src->curblk + onblk,
+                                       stream->smatcher.large_look);
+          usize_t hval = xd3_checksum_hash (& stream->large_hash, cksum);
 
-          blkoff += stream->smatcher.large_step;
-          stream->srcwin_cksum_pos += stream->smatcher.large_step;
+          /* TODO: can you simplify this computation? */
+          stream->large_table[hval] = (stream->src->blksize * blkno + onblk) + HASH_CKOFFSET;
 
           IF_DEBUG (stream->large_ckcnt += 1);
         }
+
+      stream->srcwin_cksum_pos = (blkno + 1) * stream->src->blksize;
     }
 
-  if (stream->srcwin_cksum_pos > stream->src->size) {
-    // We do this so the xd3_source_cksum_offset() function, which uses the high/low
-    // 32-bits of srcwin_cksum_pos, is guaranteed never to return a position > src->size
-    stream->srcwin_cksum_pos = stream->src->size;
-  }
-  *next_move_point = stream->input_position + stream->srcwin_size;
+  if (stream->srcwin_cksum_pos > stream->src->size)
+    {
+      /* We do this so the xd3_source_cksum_offset() function, which
+       * uses the high/low 32-bits of srcwin_cksum_pos, is guaranteed
+       * never to return a position > src->size */
+      stream->srcwin_cksum_pos = stream->src->size;
+    }
+
+ finished:
+  /* How long until this function should be called again. */
+  /* TODO: Ack, this logic is underflowing the usize_t */
+  *next_move_point = stream->input_position +
+    (usize_t)(stream->srcwin_cksum_pos - logical_input_cksum_pos);
   return 0;
 }
 
@@ -4895,7 +4918,8 @@ XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream)
   usize_t         next_move_point;
 
   /* If there will be no compression due to settings or short input, skip it entirely. */
-  if (! (DO_SMALL || DO_LARGE || DO_RUN) || stream->input_position + SLOOK > stream->avail_in) { goto loopnomore; }
+  if (! (DO_SMALL || DO_LARGE || DO_RUN) ||
+      stream->input_position + SLOOK > stream->avail_in) { goto loopnomore; }
 
   if ((ret = xd3_string_match_init (stream))) { return ret; }
 
diff --git a/xdelta3/xdelta3.h b/xdelta3/xdelta3.h
index e6c9e20..8c11000 100755..100644
--- a/xdelta3/xdelta3.h
+++ b/xdelta3/xdelta3.h
@@ -38,11 +38,6 @@
 #define XD3_DEFAULT_WINSIZE (1U << 18)
 #endif
 
-/* This is a unit of how far to advance the checksum position in one go. */
-#ifndef XD3_DEFAULT_CKSUM_ADVANCE
-#define XD3_DEFAULT_CKSUM_ADVANCE (1U << 14)
-#endif
-
 /* Default total size of the source window used in xdelta3-main.h */
 #ifndef XD3_DEFAULT_SRCWINSZ
 #define XD3_DEFAULT_SRCWINSZ (1U << 26)
@@ -583,7 +578,6 @@ struct _xd3_config
   usize_t             winsize;       /* The encoder window size. */
   usize_t             sprevsz;       /* How far back small string matching goes */
   usize_t             iopt_size;     /* entries in the instruction-optimizing buffer */
-  usize_t             srcwin_size;   /* Initial size of the source-window lookahead */
   usize_t             srcwin_maxsz;  /* srcwin_size grows by a factor of 2 when no matches are found */
 
   xd3_getblk_func   *getblk;        /* The three callbacks. */
@@ -657,7 +651,6 @@ struct _xd3_stream
   usize_t           sprevmask;        /* small string, previous window size mask */
   uint              iopt_size;
   uint              iopt_unlimited;
-  uint              srcwin_size;
   uint              srcwin_maxsz;
 
   /* general configuration */
@@ -693,7 +686,7 @@ struct _xd3_stream
                                        * match in the buffer. */
 
   // SRCWIN
-  // these variables plus srcwin_size, srcwin_maxsz above (set by config)
+  // these variables plus srcwin_maxsz above (set by config)
   int                srcwin_decided;    /* boolean: true if the srclen,srcbase have been decided. */
   xoff_t             srcwin_cksum_pos;  /* Source checksum position */
 
diff --git a/xdelta3/xdelta3.prj b/xdelta3/xdelta3.prj
index 52ad972..52ad972 100755..100644
--- a/xdelta3/xdelta3.prj
+++ b/xdelta3/xdelta3.prj
diff --git a/xdelta3/xdelta3.swig b/xdelta3/xdelta3.swig
index 696974f..696974f 100755..100644
--- a/xdelta3/xdelta3.swig
+++ b/xdelta3/xdelta3.swig
diff --git a/xdelta3/xdelta3.vcproj b/xdelta3/xdelta3.vcproj
index e1c3625..e1c3625 100755..100644
--- a/xdelta3/xdelta3.vcproj
+++ b/xdelta3/xdelta3.vcproj