Merge frmo origin

17 files changed, 318 insertions, 254 deletions

diff --git a/xdelta3/Makefile.am b/xdelta3/Makefile.am
index 11f75b0..f437764 100644
--- a/xdelta3/Makefile.am
+++ b/xdelta3/Makefile.am
@@ -61,6 +61,9 @@ if DEBUG_SYMBOLS
   common_CFLAGS += -g
 endif
 
+#common_CFLAGS += -fsanitize=address -fno-omit-frame-pointer
+#common_CFLAGS += -O2
+
 # For additional debugging, add -DXD3_DEBUG=1, 2, 3, ...
 xdelta3_CFLAGS = $(C_WFLAGS) $(common_CFLAGS) -DXD3_DEBUG=0
 xdelta3_LDADD = -lm
diff --git a/xdelta3/go/src/regtest.go b/xdelta3/go/src/regtest.go
index bcf0de9..a7cab33 100644
--- a/xdelta3/go/src/regtest.go
+++ b/xdelta3/go/src/regtest.go
@@ -13,8 +13,8 @@ import (
 
 const (
         xdataset = "/volume/home/jmacd/src/testdata"
-        xcompare = "/volume/home/jmacd/src/xdelta3-devel/xdelta3/xdelta3"
-        xdelta3 = "/volume/home/jmacd/src/xdelta-64bithash/xdelta3/xdelta3"
+        xcompare = "/volume/home/jmacd/src/xdelta-devel/xdelta3/xdelta3"
+        xdelta3  = "/volume/home/jmacd/src/xdelta-64bithash/xdelta3/xdelta3"
         seed = 1422253499919909358
 )
 
@@ -35,12 +35,10 @@ func (c Config) smokeTest(t *xdelta.TestGroup, p xdelta.Program) {
 
         enc, err := t.Exec("encode", p, true, []string{"-e"})
         if err != nil {
-                fmt.Println(err)
                 t.Panic(err)
         }
         dec, err := t.Exec("decode", p, true, []string{"-d"})
         if err != nil {
-                fmt.Println(err)
                 t.Panic(err)
         }
 
@@ -234,7 +232,7 @@ func (cfg Config) offsetTest(t *xdelta.TestGroup, p xdelta.Program, offset, leng
 
         // The decoder output ("read", above) is compared with the
         // test-provided output ("write", below).  The following
-        // generates the input and output twice.
+        // generates two identical inputs.
         t.WriteRstreams("encode", seed, offset, length, enc.Srcin, enc.Stdin)
         t.WriteRstreams("decode", seed, offset, length, dec.Srcin, write)
         t.Wait(enc, dec)
diff --git a/xdelta3/go/src/xdelta/run.go b/xdelta3/go/src/xdelta/run.go
index 6523a1c..448fabe 100644
--- a/xdelta3/go/src/xdelta/run.go
+++ b/xdelta3/go/src/xdelta/run.go
@@ -55,7 +55,8 @@ func (r *Runner) RunTest(name string, f func (t *TestGroup)) {
         c := make(chan interface{})
         go func() {
                 defer func() {
-                        c <- recover()
+                        rec := recover()
+                        c <- rec
                 }()
                 fmt.Println("Testing", name, "...")
                 f(t)
diff --git a/xdelta3/go/src/xdelta/tgroup.go b/xdelta3/go/src/xdelta/tgroup.go
index b1b04ec..602b1e1 100644
--- a/xdelta3/go/src/xdelta/tgroup.go
+++ b/xdelta3/go/src/xdelta/tgroup.go
@@ -58,7 +58,9 @@ func (g *Goroutine) OK() {
 
 func (g *Goroutine) Panic(err error) {
         g.finish(err)
-        runtime.Goexit()
+        if g != g.TestGroup.main {
+                runtime.Goexit()
+        }
 }
 
 func (t *TestGroup) Main() *Goroutine { return t.main }
diff --git a/xdelta3/run_release.sh b/xdelta3/run_release.sh
index f38c44d..eb08afc 100755
--- a/xdelta3/run_release.sh
+++ b/xdelta3/run_release.sh
@@ -123,6 +123,7 @@ function buildit {
     local FULLD="${SRCDIR}/${D}"
     local CFLAGS="${march} ${cargs} -I${SRCDIR}/build/lib-${LIBBM}/include"
     local CXXFLAGS="${march} ${cargs} -I${SRCDIR}/build/lib-${LIBBM}/include"
+    local CPPFLAGS="-I${SRCDIR}/build/lib-${LIBBM}/include"
     local LDFLAGS="${march} -L${SRCDIR}/build/lib-${LIBBM}/lib"
 
     mkdir -p ${D}
@@ -179,6 +180,7 @@ EOF
                   --enable-debug-symbols \
                   "CFLAGS=${CFLAGS}" \
                   "CXXFLAGS=${CXXFLAGS}" \
+                  "CPPFLAGS=${CPPFLAGS}" \
                   "LDFLAGS=${LDFLAGS}" \
                   "CC=${USECC}" \
                   "CXX=${USECXX}"
diff --git a/xdelta3/testing/regtest.cc b/xdelta3/testing/regtest.cc
index bee7477..e83af4b 100644
--- a/xdelta3/testing/regtest.cc
+++ b/xdelta3/testing/regtest.cc
@@ -92,7 +92,7 @@ public:
     bool done = false;
     bool done_after_input = false;
 
-    IF_DEBUG1 (XPR(NTR "source %"Q"[%"Q"] target %"Q" winsize %lu\n",
+    IF_DEBUG1 (XPR(NTR "source %"Q"u[%"Q"u] target %"Q"u winsize %"Z"u\n",
                   source_file.Size(), options.block_size,
                   target_file.Size(),
                   Constants::WINDOW_SIZE));
@@ -153,7 +153,7 @@ public:
         xd3_source *src = (encoding ? &encode_source : &decode_source);
         Block *block = (encoding ? &encode_source_block : &decode_source_block);
         if (encoding) {
-          IF_DEBUG1(XPR(NTR "[srcblock] %"Q"u last srcpos %"Q"u "
+          IF_DEBUG2(XPR(NTR "[srcblock] %"Q"u last srcpos %"Q"u "
                         "encodepos %"Q"u\n",
                         encode_source.getblkno,
                         encode_stream.match_last_srcpos,
@@ -790,32 +790,34 @@ void TestSmallStride() {
   usize_t size = Constants::BLOCK_SIZE * 4;
   spec0.GenerateFixedSize(size);
 
-  /* TODO Need to study the actual causes of missed adds for tests
-   * less than 30 bytes. */
-  const int s = 30;
-  usize_t adds = 0;
-  ChangeList cl;
-  for (usize_t j = s; j < size; j += s, ++adds)
-    {
-      cl.push_back(Change(Change::MODIFY, 1, j));
-    }
+  // Note: Not very good performance due to hash collisions, note 3x
+  // multiplier below.
+  for (int s = 15; s < 101; s++) {
+    usize_t changes = 0;
+    ChangeList cl;
+    for (usize_t j = s; j < size; j += s, ++changes)
+      {
+        cl.push_back(Change(Change::MODIFY, 1, j));
+      }
 
-  FileSpec spec1(&rand);
-  spec0.ModifyTo(ChangeListMutator(cl), &spec1);
+    FileSpec spec1(&rand);
+    spec0.ModifyTo(ChangeListMutator(cl), &spec1);
 
-  Options options;
-  options.encode_srcwin_maxsz = size;
-  options.iopt_size = 128;
-  options.smatch_cfg = XD3_SMATCH_SLOW;
-  options.size_known = false;
+    Options options;
+    options.encode_srcwin_maxsz = size;
+    options.iopt_size = 128;
+    options.smatch_cfg = XD3_SMATCH_SLOW;
+    options.size_known = false;
 
-  Block block;
-  InMemoryEncodeDecode(spec0, spec1, &block, options);
-  Delta delta(block);
+    Block block;
+    InMemoryEncodeDecode(spec0, spec1, &block, options);
+    Delta delta(block);
 
-  // Allow an additional two byte of add per window
-  usize_t allowance = 2 * size / Constants::WINDOW_SIZE;
-  CHECK_GE(adds + allowance, delta.AddedBytes());
+    IF_DEBUG1(DP(RINT "[stride=%d] changes=%u adds=%"Q"u\n",
+                 s, changes, delta.AddedBytes()));
+    double allowance = Constants::BLOCK_SIZE < 8192 || s < 30 ? 3.0 : 1.1;
+    CHECK_GE(allowance * changes, (double)delta.AddedBytes());
+  }
 }
 
 void TestCopyWindow() {
@@ -927,85 +929,73 @@ void TestCopyFromEnd() {
 }
 
 void TestHalfBlockCopy() {
-  MTRandom rand;
-  FileSpec spec0(&rand);
-  FileSpec spec1(&rand);
-
-  spec0.GenerateFixedSize(Constants::BLOCK_SIZE * 4);
-
-  // Create a half-block copy, 2.5 blocks apart, from the second half
-  // of the source version to the first half of the target version.
-  //       0             1             2             3
-  // spec0 [bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb][ccccc][bbbbb]
-  // spec1 [aaaaa][ccccc][aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa]
-  ChangeList cl1;
-  cl1.push_back(Change(Change::MODIFY,
-                       Constants::BLOCK_SIZE / 2,  // size
-                       0));
-  cl1.push_back(Change(Change::COPYOVER,
-                       Constants::BLOCK_SIZE / 2,  // size
-                       Constants::BLOCK_SIZE * 3,  // offset
-                       Constants::BLOCK_SIZE / 2));
-  cl1.push_back(Change(Change::MODIFY,
-                       Constants::BLOCK_SIZE * 3,
-                       Constants::BLOCK_SIZE));
-  spec0.ModifyTo(ChangeListMutator(cl1), &spec1);
-
-  const int onecopy_adds =
-    4 * Constants::BLOCK_SIZE - Constants::BLOCK_SIZE / 2;
-  const int nocopy_adds = 4 * Constants::BLOCK_SIZE;
-
-  // Note the case b=4 is contrived: the caller should use a single block
-  // containing the entire source, if possible.
-  for (int b = 1; b <= 4; b++)
+  // Create a half-block copy, 7.5 blocks apart, in a pair of files:
+  //       0             1     ...     6             7
+  // spec0 [bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb][ccccc][bbbb_]
+  // spec1 [aaaaa][ccccc][aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_]
+  // where stage=
+  // 0: the final block is full
+  //   a. (source)spec1->(target)spec0 copies block C: reads 8 source
+  //      blocks during target block 0.
+  //   b. (source)spec0->(target)spec1 does not copy block C b/c attempt
+  //      to read past EOF empties block 0 from (virtual) block cache
+  // 1: the final block is less than full.
+  //   a. (same) copies block C
+  //   b. (same) copies block C, unlike 0a, no attempt to read past EOF
+  //
+  // "virtual" above refers to XD3_TOOFARBACK, since there is no caching
+  // in the API, there is simply a promise not to request blocks that are
+  // beyond source->max_winsize from the last known source file position.
+  for (int stage = 0; stage < 2; stage++)
     {
+      IF_DEBUG1 (DP(RINT "half_block_copy stage %d\n", stage));
+
+      MTRandom rand;
+      FileSpec spec0(&rand);
+      FileSpec spec1(&rand);
+
+      spec0.GenerateFixedSize(Constants::BLOCK_SIZE * 8 - stage);
+
+      ChangeList cl1;
+      cl1.push_back(Change(Change::MODIFY,
+                           Constants::BLOCK_SIZE / 2,  // size
+                           0));
+      cl1.push_back(Change(Change::COPYOVER,
+                           Constants::BLOCK_SIZE / 2,  // size
+                           Constants::BLOCK_SIZE * 7,  // offset
+                           Constants::BLOCK_SIZE / 2));
+      cl1.push_back(Change(Change::MODIFY,
+                           Constants::BLOCK_SIZE * 7,
+                           Constants::BLOCK_SIZE - stage));
+      spec0.ModifyTo(ChangeListMutator(cl1), &spec1);
+
       Options options;
-      options.encode_srcwin_maxsz = Constants::BLOCK_SIZE * b;
+      options.encode_srcwin_maxsz = Constants::BLOCK_SIZE * 8;
 
       Block block0;
       Block block1;
       InMemoryEncodeDecode(spec0, spec1, &block0, options);
       InMemoryEncodeDecode(spec1, spec0, &block1, options);
+
       Delta delta0(block0);
       Delta delta1(block1);
 
-      // The first block never copies from the last source block, by
-      // design, because if the last source block is available when
-      // the first target block is ready, the caller is expected to
-      // use a single block.
-      CHECK_EQ(nocopy_adds, delta0.AddedBytes());
-      if (Constants::BLOCK_SIZE < 8192 || b > 2)
-        {
-          // For small-block inputs, the entire file is read into one
-          // block (the min source window size is 16kB).
-          //
-          // For large blocks, at least 3 blocks of source window are
-          // needed.
-          CHECK_EQ(onecopy_adds, delta1.AddedBytes());
-        }
+      const int yes =
+        Constants::BLOCK_SIZE * 8 - Constants::BLOCK_SIZE / 2;
+      const int no =
+        Constants::BLOCK_SIZE * 8 - Constants::BLOCK_SIZE / 2;
+
+      if (stage == 0)
+        {
+          CHECK_EQ(yes, delta0.AddedBytes());
+          CHECK_EQ(no, delta1.AddedBytes());
+        }
       else
-        {
-          // When there are fewer than 3 source blocks.
-          CHECK_EQ(nocopy_adds, delta1.AddedBytes());
-        }
+        {
+          CHECK_EQ(yes, delta0.AddedBytes());
+          CHECK_EQ(yes, delta1.AddedBytes());
+        }
     }
-
-  Options options;
-  options.encode_srcwin_maxsz = Constants::BLOCK_SIZE * 4;
-  options.block_size = Constants::BLOCK_SIZE * 4;
-
-  // Test the whole-buffer case.
-  Block block0;
-  Block block1;
-  InMemoryEncodeDecode(spec0, spec1, &block0, options);
-  InMemoryEncodeDecode(spec1, spec0, &block1, options);
-  Delta delta0(block0);
-  Delta delta1(block1);
-  // This <= >= are only for blocksize = 512, which has irregular readsize.
-  CHECK_LE(onecopy_adds, delta0.AddedBytes());
-  CHECK_GE(onecopy_adds + 1, delta0.AddedBytes());
-
-  CHECK_EQ(onecopy_adds, delta1.AddedBytes());
 }
 
 void FourWayMergeTest(const FileSpec &spec0,
@@ -1310,3 +1300,4 @@ int main(int argc, char **argv)
 
   return 0;
 }
+
diff --git a/xdelta3/testing/test.h b/xdelta3/testing/test.h
index 30d13a3..7de24fb 100644
--- a/xdelta3/testing/test.h
+++ b/xdelta3/testing/test.h
@@ -22,8 +22,8 @@ extern "C" {
     __typeof__(x) _y(y); \
     if (!(_x OP _y)) { \
       cerr << __FILE__ << ":" << __LINE__ << " Check failed: " << #x " " #OP " " #y << endl; \
-      cerr << __FILE__ << ":" << __LINE__ << " Expected: " << _x << endl; \
-      cerr << __FILE__ << ":" << __LINE__ << " Actual: " << _y << endl; \
+      cerr << __FILE__ << ":" << __LINE__ << " {0} " << _x << endl; \
+      cerr << __FILE__ << ":" << __LINE__ << " {1} " << _y << endl; \
     abort(); \
     } } while (false)
 #undef CHECK
diff --git a/xdelta3/xdelta3-blkcache.h b/xdelta3/xdelta3-blkcache.h
index 8de5d13..20a2a4a 100644
--- a/xdelta3/xdelta3-blkcache.h
+++ b/xdelta3/xdelta3-blkcache.h
@@ -183,6 +183,7 @@ main_set_source (xd3_stream *stream, xd3_cmd cmd,
   if (!sfile->size_known && source->onblk < blksize)
     {
       source_size = source->onblk;
+      source->onlastblk = source_size;
       sfile->size_known = 1;
     }
 
@@ -194,8 +195,10 @@ main_set_source (xd3_stream *stream, xd3_cmd cmd,
       /* Modify block 0, change blocksize. */
       blksize = option_srcwinsz / MAX_LRU_SIZE;
       source->blksize = blksize;
-      source->onblk = blksize;  /* xd3 sets onblk */
-      /* Note: source->max_winsize is unchanged. */
+      source->onblk = blksize;
+      source->onlastblk = blksize;
+      source->max_blkno = MAX_LRU_SIZE - 1;
+
       lru[0].size = blksize;
       lru_size = MAX_LRU_SIZE;
 
@@ -309,9 +312,13 @@ main_getblk_lru (xd3_source *source, xoff_t blkno,
               main_blklru_list_remove (blru);
               main_blklru_list_push_back (& lru_list, blru);
               (*blrup) = blru;
+              IF_DEBUG1 (DP(RINT "[getblk_lru] HIT blkno = %"Z"u lru_size=%d\n",
+                    blkno, lru_size));
               return 0;
             }
         }
+      IF_DEBUG1 (DP(RINT "[getblk_lru] MISS blkno = %"Z"u lru_size=%d\n",
+                    blkno, lru_size));
     }
 
   if (do_src_fifo)
@@ -467,7 +474,6 @@ main_getblk_func (xd3_stream *stream,
   main_file *sfile = (main_file*) source->ioh;
   main_blklru *blru;
   int is_new;
-  int did_seek = 0;
   size_t nread = 0;
 
   if (allow_fake_source)
@@ -504,20 +510,10 @@ main_getblk_func (xd3_stream *stream,
         {
           return ret;
         }
-
-      /* Indicates that another call to main_getblk_lru() may be
-       * needed */
-      did_seek = 1;
     }
 
   XD3_ASSERT (sfile->source_position == pos);
 
-  if (did_seek &&
-      (ret = main_getblk_lru (source, blkno, & blru, & is_new)))
-    {
-      return ret;
-    }
-
   if ((ret = main_read_primary_input (sfile,
                                       (uint8_t*) blru->blk,
                                       source->blksize,
diff --git a/xdelta3/xdelta3-decode.h b/xdelta3/xdelta3-decode.h
index abe8c83..0cb8857 100644
--- a/xdelta3/xdelta3-decode.h
+++ b/xdelta3/xdelta3-decode.h
@@ -162,6 +162,9 @@ xd3_decode_allocate (xd3_stream  *stream,
                      uint8_t    **buf_ptr,
                      usize_t      *buf_alloc)
 {
+  IF_DEBUG2 (DP(RINT "[xd3_decode_allocate] size %u alloc %u\n",
+                size, *buf_alloc));
+  
   if (*buf_ptr != NULL && *buf_alloc < size)
     {
       xd3_free (stream, *buf_ptr);
@@ -204,6 +207,8 @@ xd3_decode_section (xd3_stream *stream,
           /* No allocation/copy needed */
           section->buf = stream->next_in;
           sect_take    = section->size;
+          IF_DEBUG1 (DP(RINT "[xd3_decode_section] zerocopy %u @ %u avail %u\n",
+                        sect_take, section->pos, stream->avail_in));
         }
       else
         {
@@ -227,6 +232,10 @@ xd3_decode_section (xd3_stream *stream,
               section->buf = section->copied1;
             }
 
+          IF_DEBUG2 (DP(RINT "[xd3_decode_section] take %u @ %u [need %u] avail %u\n",
+                        sect_take, section->pos, sect_need, stream->avail_in));
+          XD3_ASSERT (section->pos + sect_take <= section->alloc1);
+
           memcpy (section->copied1 + section->pos,
                   stream->next_in,
                   sect_take);
@@ -241,6 +250,7 @@ xd3_decode_section (xd3_stream *stream,
 
   if (section->pos < section->size)
     {
+      IF_DEBUG1 (DP(RINT "[xd3_decode_section] further input required %u\n", section->size - section->pos));
       stream->msg = "further input required";
       return XD3_INPUT;
     }
@@ -1156,9 +1166,9 @@ xd3_decode_input (xd3_stream *stream)
                           &src->cpyoff_blocks,
                           &src->cpyoff_blkoff);
           
-          IF_DEBUG1(DP(RINT
-                       "decode cpyoff %"Q" "
-                       "cpyblkno %"Q" "
+          IF_DEBUG2(DP(RINT
+                       "[decode_cpyoff] %"Q"u "
+                       "cpyblkno %"Q"u "
                        "cpyblkoff %u "
                        "blksize %u\n",
                        stream->dec_cpyoff,
diff --git a/xdelta3/xdelta3-djw.h b/xdelta3/xdelta3-djw.h
index e6be032..414702a 100644
--- a/xdelta3/xdelta3-djw.h
+++ b/xdelta3/xdelta3-djw.h
@@ -1456,7 +1456,7 @@ djw_decode_symbol (xd3_stream     *stream,
       if (*input == input_end)
         {
           stream->msg = "secondary decoder end of input";
-          return XD3_INTERNAL;
+          return XD3_INVALID_INPUT;
         }
 
       bstate->cur_byte = *(*input)++;
@@ -1479,7 +1479,7 @@ djw_decode_symbol (xd3_stream     *stream,
 
  corrupt:
   stream->msg = "secondary decoder invalid code";
-  return XD3_INTERNAL;
+  return XD3_INVALID_INPUT;
 }
 
 static int
@@ -1606,7 +1606,7 @@ djw_decode_1_2 (xd3_stream     *stream,
   if (rep != 0)
     {
       stream->msg = "secondary decoder invalid repeat code";
-      return XD3_INTERNAL;
+      return XD3_INVALID_INPUT;
     }
   
   return 0;
@@ -1654,7 +1654,7 @@ xd3_decode_huff (xd3_stream     *stream,
   if (output_bytes == 0)
     {
       stream->msg = "secondary decoder invalid input";
-      return XD3_INTERNAL;
+      return XD3_INVALID_INPUT;
     }
 
   /* Decode: number of groups */
@@ -1796,7 +1796,11 @@ xd3_decode_huff (xd3_stream     *stream,
                 gp_maxlen  = maxlen[gp];
               }
 
-            XD3_ASSERT (output_end - output > 0);
+            if (output_end < output)
+              {
+                stream->msg = "secondary decoder invalid input";
+                return XD3_INVALID_INPUT;
+              }
             
             /* Decode next sector. */
             n = xd3_min (sector_size, (usize_t) (output_end - output));
diff --git a/xdelta3/xdelta3-internal.h b/xdelta3/xdelta3-internal.h
index 6482721..d54584d 100644
--- a/xdelta3/xdelta3-internal.h
+++ b/xdelta3/xdelta3-internal.h
@@ -335,6 +335,7 @@ xd3_emit_uint64_t (xd3_stream *stream, xd3_output **output, uint64_t num)
 
 #if SIZEOF_USIZE_T == 4
 #define USIZE_T_MAX             UINT32_MAX
+#define USIZE_T_MAXBLKSZ        0x80000000U
 #define xd3_decode_size         xd3_decode_uint32_t
 #define xd3_emit_size           xd3_emit_uint32_t
 #define xd3_sizeof_size         xd3_sizeof_uint32_t
@@ -342,9 +343,10 @@ xd3_emit_uint64_t (xd3_stream *stream, xd3_output **output, uint64_t num)
 #define xd3_large_cksum         xd3_large32_cksum
 #define xd3_large_cksum_update  xd3_large32_cksum_update
 #define xd3_hash_multiplier     xd3_hash_multiplier32
-#define XD3_MAXSRCWINSZ (1ULL << 31)
+#define XD3_MAXSRCWINSZ         (1ULL << 31)
 #elif SIZEOF_USIZE_T == 8
 #define USIZE_T_MAX             UINT64_MAX
+#define USIZE_T_MAXBLKSZ        0x8000000000000000ULL
 #define xd3_decode_size         xd3_decode_uint64_t
 #define xd3_emit_size           xd3_emit_uint64_t
 #define xd3_sizeof_size         xd3_sizeof_uint64_t
@@ -352,7 +354,7 @@ xd3_emit_uint64_t (xd3_stream *stream, xd3_output **output, uint64_t num)
 #define xd3_large_cksum         xd3_large64_cksum
 #define xd3_large_cksum_update  xd3_large64_cksum_update
 #define xd3_hash_multiplier     xd3_hash_multiplier64
-#define XD3_MAXSRCWINSZ (1ULL << 61)
+#define XD3_MAXSRCWINSZ         (1ULL << 61)
 #endif /* SIZEOF_USIZE_T */
 
 #if SIZEOF_XOFF_T == 4
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h
index cfc952c..e43f6b3 100644
--- a/xdelta3/xdelta3-main.h
+++ b/xdelta3/xdelta3-main.h
@@ -870,6 +870,8 @@ main_file_open (main_file *xfile, const char* name, int mode)
       return XD3_INVALID;
     }
 
+  IF_DEBUG1(DP(RINT "[main] open source %s\n", name));
+
 #if XD3_STDIO
   xfile->file = fopen (name, XOPEN_STDIO);
 
@@ -1041,6 +1043,7 @@ main_file_read (main_file  *ifile,
                 const char *msg)
 {
   int ret = 0;
+  IF_DEBUG1(DP(RINT "[main] read %s up to %"Z"u\n", ifile->filename, size));
 
 #if XD3_STDIO
   size_t result;
@@ -1081,6 +1084,8 @@ main_file_write (main_file *ofile, uint8_t *buf, usize_t size, const char *msg)
 {
   int ret = 0;
 
+  IF_DEBUG1(DP(RINT "[main] write %u\n bytes", size));
+  
 #if XD3_STDIO
   usize_t result;
 
@@ -1150,6 +1155,8 @@ main_write_output (xd3_stream* stream, main_file *ofile)
 {
   int ret;
 
+  IF_DEBUG1(DP(RINT "[main] write(%s) %u\n bytes", ofile->filename, stream->avail_out));
+
   if (option_no_output)
     {
       return 0;
@@ -3335,7 +3342,7 @@ main_input (xd3_cmd     cmd,
                             main_format_bcnt (stream.total_in, &trdb),
                             main_format_bcnt (stream.total_out, &twdb),
                             main_format_millis (millis, &tm),
-                            main_format_bcnt (sfile->source_position, &srcpos));
+                            main_format_bcnt (stream.srcwin_cksum_pos, &srcpos));
                       }
                     else
                       {
diff --git a/xdelta3/xdelta3-merge.h b/xdelta3/xdelta3-merge.h
index 72bc892..a09f814 100644
--- a/xdelta3/xdelta3-merge.h
+++ b/xdelta3/xdelta3-merge.h
@@ -492,10 +492,15 @@ xd3_merge_source_copy (xd3_stream *stream,
             }
           else
             {
-              // TODO: this is slow because of the recursion, which
-              // could reach a depth equal to the number of target
-              // copies, and this is compression-inefficient because
-              // it can produce duplicate adds.
+              // Note: A better implementation will construct the
+              // mapping of output ranges, starting from the input
+              // range, applying deltas in forward order, using an
+              // interval tree.  This code uses recursion to construct
+              // each copied range, recursively (using binary search
+              // in xd3_merge_find_position).
+              //
+              // TODO: This code can cause stack overflow. Fix as
+              // described above.
               xd3_winst tinst;
               tinst.type = XD3_CPY;
               tinst.mode = iinst.mode;
@@ -555,12 +560,14 @@ int xd3_merge_inputs (xd3_stream *stream,
           ret = xd3_merge_add (stream, input, iinst);
           break;
         default:
-          /* TODO: VCD_TARGET support is completely untested all
-           * throughout. */
-          if (iinst->mode == 0 || iinst->mode == VCD_TARGET)
+          if (iinst->mode == 0)
             {
               ret = xd3_merge_target_copy (stream, iinst);
             }
+          else if (iinst->mode == VCD_TARGET)
+            {
+              ret = XD3_INVALID_INPUT;
+            }
           else
             {
               ret = xd3_merge_source_copy (stream, source, iinst);
diff --git a/xdelta3/xdelta3-second.h b/xdelta3/xdelta3-second.h
index 309c947..899902e 100644
--- a/xdelta3/xdelta3-second.h
+++ b/xdelta3/xdelta3-second.h
@@ -298,9 +298,12 @@ xd3_encode_secondary (xd3_stream      *stream,
 
   if (comp_size < (orig_size - SECONDARY_MIN_SAVINGS) || cfg->inefficient)
     {
-      IF_DEBUG1(DP(RINT "secondary saved %u bytes: %u -> %u (%0.2f%%)\n",
-                   orig_size - comp_size, orig_size, comp_size,
-               100.0 * (double) comp_size / (double) orig_size));
+      if (comp_size < orig_size)
+        {
+          IF_DEBUG1(DP(RINT "[encode_secondary] saved %u bytes: %u -> %u (%0.2f%%)\n",
+                       orig_size - comp_size, orig_size, comp_size,
+                       100.0 * (double) comp_size / (double) orig_size));
+        }
 
       xd3_free_output (stream, *head);
 
diff --git a/xdelta3/xdelta3-test.h b/xdelta3/xdelta3-test.h
index ddbfd65..335de28 100644
--- a/xdelta3/xdelta3-test.h
+++ b/xdelta3/xdelta3-test.h
@@ -3009,9 +3009,9 @@ int xd3_selftest (void)
 #endif
 
   DO_TEST (recode_command, 0, 0);
+  IF_LZMA (DO_TEST (secondary_lzma_default, 0, 0));
 #endif
 
-  IF_LZMA (DO_TEST (secondary_lzma_default, 0, 0));
   IF_LZMA (DO_TEST (secondary_lzma, 0, 1));
   IF_DJW (DO_TEST (secondary_huff, 0, DJW_MAX_GROUPS));
   IF_FGK (DO_TEST (secondary_fgk, 0, 1));
diff --git a/xdelta3/xdelta3.c b/xdelta3/xdelta3.c
index d95d97d..9253d68 100644
--- a/xdelta3/xdelta3.c
+++ b/xdelta3/xdelta3.c
@@ -519,7 +519,7 @@ static void xd3_verify_large_state (xd3_stream *stream,
                                     usize_t x_cksum);
 static void xd3_verify_small_state (xd3_stream    *stream,
                                     const uint8_t *inp,
-                                    uint32_t          x_cksum);
+                                    uint32_t       x_cksum);
 
 #endif /* XD3_DEBUG */
 #endif /* XD3_ENCODER */
@@ -1067,7 +1067,17 @@ xd3_round_blksize (usize_t sz, usize_t blksz)
 
   XD3_ASSERT (xd3_check_pow2 (blksz, NULL) == 0);
 
-  return mod ? (sz + (blksz - mod)) : sz;
+  if (mod == 0)
+    {
+      return sz;
+    }
+
+  if (sz > USIZE_T_MAXBLKSZ)
+    {
+      return USIZE_T_MAXBLKSZ;
+    }
+
+  return sz + (blksz - mod);
 }
 
 /***********************************************************************
@@ -1848,7 +1858,7 @@ xd3_config_stream(xd3_stream *stream,
 inline
 xoff_t xd3_source_eof(const xd3_source *src)
 {
-  xoff_t r = (src->blksize * src->max_blkno) + src->onlastblk;
+  xoff_t r = (src->max_blkno << src->shiftby) + (xoff_t)src->onlastblk;
   return r;
 }
 
@@ -1862,7 +1872,7 @@ usize_t xd3_bytes_on_srcblk (xd3_source *src, xoff_t blkno)
 }
 
 /* This function interfaces with the client getblk function, checks
- * its results, updates frontier_blkno, max_blkno, onlastblk, eof_known. */
+ * its results, updates max_blkno, onlastblk, eof_known. */
 static int
 xd3_getblk (xd3_stream *stream, xoff_t blkno)
 {
@@ -1875,6 +1885,7 @@ xd3_getblk (xd3_stream *stream, xoff_t blkno)
 
       if (stream->getblk == NULL)
         {
+          IF_DEBUG2 (DP(RINT "[getblk] XD3_GETSRCBLK %"Q"u\n", blkno));
           stream->msg = "getblk source input";
           return XD3_GETSRCBLK;
         }
@@ -1882,48 +1893,34 @@ xd3_getblk (xd3_stream *stream, xoff_t blkno)
       ret = stream->getblk (stream, source, blkno);
       if (ret != 0)
         {
-          IF_DEBUG1 (DP(RINT "[getblk] app error blkno %"Q": %s\n",
+          IF_DEBUG2 (DP(RINT "[getblk] app error blkno %"Q"u: %s\n",
                         blkno, xd3_strerror (ret)));
           return ret;
         }
-      IF_DEBUG2 (DP(RINT "[getblk] read source block %"Q" onblk "
-                    "%u blksize %u\n", blkno, source->onblk, source->blksize));
+
+      IF_DEBUG2 (DP(RINT "[getblk] read source block %"Q"u onblk "
+                    "%u blksize %u max_blkno %"Q"u\n", blkno, source->onblk,
+                    source->blksize, source->max_blkno));
     }
 
-  if (blkno >= source->frontier_blkno)
+  if (blkno > source->max_blkno)
     {
-      if (blkno > source->max_blkno)
-        {
-          source->max_blkno = blkno;
-          source->onlastblk = source->onblk;
-        }
+      source->max_blkno = blkno;
 
       if (source->onblk == source->blksize)
         {
-          source->frontier_blkno = blkno + 1;
-
-          IF_DEBUG2 (DP(RINT "[getblk] full source blkno %"Q": "
-                        "source length unknown %"Q"\n",
+          IF_DEBUG1 (DP(RINT "[getblk] full source blkno %"Q"u: "
+                        "source length unknown %"Q"u\n",
                         blkno,
                         xd3_source_eof (source)));
         }
-      else
+      else if (!source->eof_known)
         {
-          source->frontier_blkno = blkno;
-
-          if (xd3_bytes_on_srcblk (source, blkno) != 0)
-            {
-              source->frontier_blkno += 1;
-            }
-
-          if (!source->eof_known)
-            {
-              IF_DEBUG2 (DP(RINT "[getblk] eof block has %d bytes; "
-                            "source length known %"Q"\n",
-                            xd3_bytes_on_srcblk (source, blkno),
-                            xd3_source_eof (source)));
-              source->eof_known = 1;
-            }
+          IF_DEBUG1 (DP(RINT "[getblk] eof block has %d bytes; "
+                        "source length known %"Q"u\n",
+                        xd3_bytes_on_srcblk (source, blkno),
+                        xd3_source_eof (source)));
+          source->eof_known = 1;
         }
     }
 
@@ -1932,13 +1929,8 @@ xd3_getblk (xd3_stream *stream, xoff_t blkno)
   if (blkno == source->max_blkno)
     {
       /* In case the application sets the source as 1 block w/ a
-         preset buffer. */
+       * preset buffer. */
       source->onlastblk = source->onblk;
-
-      if (source->onblk == source->blksize)
-        {
-          source->frontier_blkno = blkno + 1;
-        }
     }
   return 0;
 }
@@ -1975,7 +1967,6 @@ xd3_set_source (xd3_stream *stream,
       IF_DEBUG1 (DP(RINT "raising src_maxsize to %u\n", src->blksize));
     }
   src->max_winsize = xd3_max (src->max_winsize, XD3_ALLOCSIZE);
-
   return 0;
 }
 
@@ -1989,11 +1980,13 @@ xd3_set_source_and_size (xd3_stream *stream,
       stream->src->eof_known = 1;
       IF_DEBUG2 (DP(RINT "[set source] size known %"Q"\n",
                     source_size));
-
       xd3_blksize_div(source_size,
                       stream->src,
                       &stream->src->max_blkno,
                       &stream->src->onlastblk);
+
+      IF_DEBUG1 (DP(RINT "[set source] size known %"Q"u max_blkno %"Q"u\n",
+                    source_size, stream->src->max_blkno));
     }
   return ret;
 }
@@ -2047,8 +2040,8 @@ xd3_close_stream (xd3_stream *stream)
           break;
         default:
           /* If decoding, should be ready for the next window. */
-          stream->msg = "EOF in decode";
-          return XD3_INTERNAL;
+          stream->msg = "eof in decode";
+          return XD3_INVALID_INPUT;
         }
     }
 
@@ -2207,8 +2200,6 @@ xd3_iopt_finish_encoding (xd3_stream *stream, xd3_rinst *inst)
       }
     case XD3_RUN:
       {
-        XD3_ASSERT (inst->size >= MIN_MATCH);
-
         if ((ret = xd3_emit_byte (stream, & DATA_TAIL (stream), inst->xtra))) { return ret; }
 
         stream->n_run += 1;
@@ -3194,6 +3185,7 @@ xd3_encode_input (xd3_stream *stream)
                    * or else it can get stuck in a match-backward
                    * (getsrcblk) then match-forward (getsrcblk),
                    * find insufficient match length, then repeat
+
                    * exactly the same search.
                    */
                   stream->input_position += stream->match_fwd;
@@ -3219,7 +3211,7 @@ xd3_encode_input (xd3_stream *stream)
 
     case ENC_INSTR:
       /* Note: Jump here to encode VCDIFF deltas w/o using this
-       * string-matching code.  Merging code code enters here. */
+       * string-matching code.  Merging code enters here. */
 
       /* Flush the instrution buffer, then possibly add one more
        * instruction, then emit the header. */
@@ -3674,7 +3666,7 @@ xd3_srcwin_setup (xd3_stream *stream)
   /* Otherwise, we have to make a guess.  More copies may still be
    * issued, but we have to decide the source window base and length
    * now.  
-   * TODO: This >> 2 is arbitrary--try other values. */
+   * TODO: This may not working well in practice, more testing needed. */
   src->srcbase = stream->match_minaddr;
   src->srclen  = xd3_max ((usize_t) length,
                           stream->avail_in + (stream->avail_in >> 2));
@@ -3707,9 +3699,8 @@ xd3_srcwin_setup (xd3_stream *stream)
 static int
 xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos)
 {
-  xd3_source *src = stream->src;
+  xd3_source *const src = stream->src;
   usize_t greedy_or_not;
-  xoff_t frontier_pos;
 
   stream->match_maxback = 0;
   stream->match_maxfwd  = 0;
@@ -3733,18 +3724,24 @@ xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos)
   /* Implement src->max_winsize, which prevents the encoder from seeking
    * back further than the LRU cache maintaining FIFO discipline, (to
    * avoid seeking). */
-  frontier_pos = stream->src->frontier_blkno * stream->src->blksize;
-  IF_DEBUG2(DP(RINT "[match_setup] frontier_pos %"Q", srcpos %"Q", "
-               "src->max_winsize %"Q"\n",
-               frontier_pos, srcpos, stream->src->max_winsize));
-  if (srcpos < frontier_pos &&
-      frontier_pos - srcpos > stream->src->max_winsize) {
-    IF_DEBUG1(DP(RINT "[match_setup] rejected due to src->max_winsize "
-                 "distance eof=%"Q" srcpos=%"Q" maxsz=%"Q"\n",
-                 xd3_source_eof (stream->src),
-                 srcpos, stream->src->max_winsize));
-    goto bad;
-  }
+  if (srcpos < stream->srcwin_cksum_pos &&
+      stream->srcwin_cksum_pos - srcpos > src->max_winsize)
+    {
+      IF_DEBUG2(DP(RINT "[match_setup] rejected due to src->max_winsize "
+                   "distance eof=%"Q"u srcpos=%"Q"u max_winsz=%"Q"u\n",
+                   xd3_source_eof (src),
+                   srcpos, src->max_winsize));
+      goto bad;
+    }
+
+  /* There are cases where the above test does not reject a match that
+   * will experience XD3_TOOFARBACK at the first xd3_getblk call
+   * because the input may have advanced up to one block beyond the
+   * actual EOF. */
+  IF_DEBUG2(DP(RINT "[match_setup] %"Q"u srcpos %"Q"u, "
+               "src->max_winsize %"Q"u\n",
+               stream->total_in + stream->input_position,
+               srcpos, src->max_winsize));
 
   /* Going backwards, the 1.5-pass algorithm allows some
    * already-matched input may be covered by a longer source match.
@@ -3788,9 +3785,9 @@ xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos)
           stream->match_maxback = (usize_t) srcpos;
         }
 
-      if (stream->src->eof_known)
+      if (src->eof_known)
         {
-          xoff_t srcavail = xd3_source_eof (stream->src) - srcpos;
+          xoff_t srcavail = xd3_source_eof (src) - srcpos;
 
           if (srcavail < stream->match_maxfwd)
             {
@@ -3834,8 +3831,8 @@ xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos)
           stream->match_maxfwd = srcavail;
         }
 
-      IF_DEBUG1(DP(RINT
-                   "[match_setup] srcpos %"Q" (tgtpos %"Q") "
+      IF_DEBUG2(DP(RINT
+                   "[match_setup] srcpos %"Q"u (tgtpos %"Q"u) "
                    "restricted maxback %u maxfwd %u\n",
                    srcpos,
                    stream->total_in + stream->input_position,
@@ -3852,6 +3849,7 @@ xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos)
 
  bad:
   stream->match_state  = MATCH_SEARCHING;
+  stream->match_last_srcpos = srcpos;
   return 1;
 }
 
@@ -3903,7 +3901,7 @@ static int
 xd3_source_extend_match (xd3_stream *stream)
 {
   int ret;
-  xd3_source *src = stream->src;
+  xd3_source *const src = stream->src;
   xoff_t matchoff;  /* matchoff is the current right/left-boundary of
                        the source match being tested. */
   usize_t streamoff; /* streamoff is the current right/left-boundary
@@ -3923,7 +3921,7 @@ xd3_source_extend_match (xd3_stream *stream)
   if (stream->match_state == MATCH_BACKWARD)
     {
       /* Note: this code is practically duplicated below, substituting
-       * match_fwd/match_back and direction.  TODO: Consolidate? */
+       * match_fwd/match_back and direction. */
       matchoff  = stream->match_srcpos - stream->match_back;
       streamoff = stream->input_position - stream->match_back;
       xd3_blksize_div (matchoff, src, &tryblk, &tryoff);
@@ -3940,10 +3938,22 @@ xd3_source_extend_match (xd3_stream *stream)
 
           if ((ret = xd3_getblk (stream, tryblk)))
             {
-              /* if search went too far back, continue forward. */
               if (ret == XD3_TOOFARBACK)
                 {
-                  break;
+                  IF_DEBUG2(DP(RINT "[maxback] %"Q"u TOOFARBACK: %u INP %"Q"u CKSUM %"Q"u\n",
+                               tryblk, stream->match_back,
+                               stream->total_in + stream->input_position,
+                               stream->srcwin_cksum_pos));
+
+                  /* the starting position is too far back. */
+                  if (stream->match_back == 0)
+                    {
+                      XD3_ASSERT(stream->match_fwd == 0);
+                      goto donefwd;
+                    }
+
+                  /* search went too far back, continue forward. */
+                  goto doneback;
                 }
 
               /* could be a XD3_GETSRCBLK failure. */
@@ -3989,7 +3999,14 @@ xd3_source_extend_match (xd3_stream *stream)
 
       if ((ret = xd3_getblk (stream, tryblk)))
         {
-          XD3_ASSERT (ret != XD3_TOOFARBACK);
+          if (ret == XD3_TOOFARBACK)
+            {
+              IF_DEBUG2(DP(RINT "[maxfwd] %"Q"u TOOFARBACK: %u INP %"Q"u CKSUM %"Q"u\n",
+                           tryblk, stream->match_fwd,
+                           stream->total_in + stream->input_position,
+                           stream->srcwin_cksum_pos));
+              goto donefwd;
+            }
 
           /* could be a XD3_GETSRCBLK failure. */
           return ret;
@@ -4020,8 +4037,14 @@ xd3_source_extend_match (xd3_stream *stream)
         }
     }
 
+ donefwd:
   stream->match_state = MATCH_SEARCHING;
 
+  IF_DEBUG2(DP(RINT "[extend match] input %"Q"u srcpos %"Q"u len %u\n",
+               stream->input_position + stream->total_in,
+               stream->match_srcpos,
+               stream->match_fwd));
+
   /* If the match ends short of the last instruction end, we probably
    * don't want it.  There is the possibility that a copy ends short
    * of the last copy but also goes further back, in which case we
@@ -4073,8 +4096,9 @@ xd3_source_extend_match (xd3_stream *stream)
 
       IF_DEBUG2 ({
         static int x = 0;
-        DP(RINT "[source match:%d] <inp %"Q" %"Q">  <src %"Q" %"Q"> (%s) [ %u bytes ]\n",
+        DP(RINT "[source match:%d] length %u <inp %"Q"u %"Q"u>  <src %"Q"u %"Q"u> (%s) [ %u bytes ]\n",
            x++,
+           match_length,
            stream->total_in + target_position,
            stream->total_in + target_position + match_length,
            match_position,
@@ -4315,17 +4339,18 @@ xd3_verify_run_state (xd3_stream    *stream,
  * stream->input_position reaches the value returned as
  * *next_move_point.  NB: this is one of the most expensive functions
  * in this code and also the most critical for good compression.
- * TODO: optimize the inner loop
  */
 static int
 xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
 {
-  xoff_t logical_input_cksum_pos;
-  xoff_t source_size;
+  /* the source file is indexed until this point */
+  xoff_t target_cksum_pos;
+  /* the absolute target file input position */
+  xoff_t absolute_input_pos;
 
   if (stream->src->eof_known)
     {
-      source_size = xd3_source_eof (stream->src);
+      xoff_t source_size = xd3_source_eof (stream->src);
       XD3_ASSERT(stream->srcwin_cksum_pos <= source_size);
 
       if (stream->srcwin_cksum_pos == source_size)
@@ -4335,19 +4360,31 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
         }
     }
 
-  /* Begin by advancing at twice the input rate, up to half the
-   * maximum window size. */
-  logical_input_cksum_pos = xd3_min((stream->total_in + stream->input_position) * 2,
-                                (stream->total_in + stream->input_position) +
-                                  (stream->src->max_winsize / 2));
+  absolute_input_pos = stream->total_in + stream->input_position;
 
-  /* If srcwin_cksum_pos is already greater, wait until the difference
-   * is met. */
-  if (stream->srcwin_cksum_pos > logical_input_cksum_pos)
+  /* Immediately read the entire window. 
+   *
+   * Note: this reverses a long held policy, at this point in the
+   * code, of advancing relatively slowly as the input is read, which
+   * results in better compression for very-similar inputs, but worse
+   * compression where data is deleted near the beginning of the file.
+   * 
+   * The new policy is simpler, somewhat slower and can benefit, or
+   * slightly worsen, compression performance. */
+  if (absolute_input_pos < stream->src->max_winsize / 2)
     {
-      *next_move_point = stream->input_position +
-        (usize_t)(stream->srcwin_cksum_pos - logical_input_cksum_pos);
-      return 0;
+      target_cksum_pos = stream->src->max_winsize;
+    }
+  else
+    {
+      /* TODO: The addition of 2 blocks here is arbitrary.  Do a
+       * better job of stream alignment based on observed source copy
+       * addresses, and when both input sizes are known, the
+       * difference in size. */
+      target_cksum_pos = absolute_input_pos +
+        stream->src->max_winsize / 2 +
+        stream->src->blksize * 2;
+      target_cksum_pos &= ~stream->src->maskby;
     }
 
   /* A long match may have extended past srcwin_cksum_pos.  Don't
@@ -4357,27 +4394,12 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
       stream->srcwin_cksum_pos = stream->maxsrcaddr;
     }
 
-  if (logical_input_cksum_pos < stream->srcwin_cksum_pos)
+  if (target_cksum_pos < stream->srcwin_cksum_pos)
     {
-      logical_input_cksum_pos = stream->srcwin_cksum_pos;
+      target_cksum_pos = stream->srcwin_cksum_pos;
     }
 
-  /* Advance at least one source block.  With the command-line
-   * defaults this means:
-   *
-   * if (src->size <= src->max_winsize), index the entire source at once
-   * using the position of the first non-match.  This is good for
-   * small inputs, especially when the content may have moved anywhere
-   * in the file (e.g., tar files).
-   *
-   * if (src->size > src->max_winsize), index at least one block ahead
-   * of the logical position.  This is good for different reasons:
-   * when a long match spanning several source blocks is encountered,
-   * this avoids computing checksums for those blocks.
-   */
-  logical_input_cksum_pos += stream->src->blksize;
-
-  while (stream->srcwin_cksum_pos < logical_input_cksum_pos &&
+  while (stream->srcwin_cksum_pos < target_cksum_pos &&
          (!stream->src->eof_known ||
           stream->srcwin_cksum_pos < xd3_source_eof (stream->src)))
     {
@@ -4399,17 +4421,19 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
             {
               ret = XD3_INTERNAL;
             }
+
           IF_DEBUG1 (DP(RINT
-                        "[srcwin_move_point] async getblk return for %"Q"\n",
-                        blkno));
+                        "[srcwin_move_point] async getblk return for %"Q"u: %s\n",
+                        blkno, xd3_strerror (ret)));
           return ret;
         }
 
       IF_DEBUG1 (DP(RINT
-                    "[srcwin_move_point] T=%"Q"{%"Q"} S=%"Q" EOF=%"Q" %s\n",
+                    "[srcwin_move_point] block %"Q"u T=%"Q"u S=%"Q"u L=%"Q"u EOF=%"Q"u %s\n",
+                    blkno,
                     stream->total_in + stream->input_position,
-                    logical_input_cksum_pos,
                     stream->srcwin_cksum_pos,
+                    target_cksum_pos,
                     xd3_source_eof (stream->src),
                     stream->src->eof_known ? "known" : "unknown"));
 
@@ -4418,7 +4442,7 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
       if (blkpos < (ssize_t) stream->smatcher.large_look)
         {
           stream->srcwin_cksum_pos = (blkno + 1) * stream->src->blksize;
-          IF_DEBUG1 (DP(RINT "[srcwin_move_point] continue (end-of-block)\n"));
+          IF_DEBUG2 (DP(RINT "[srcwin_move_point] continue (end-of-block): %"Q"u\n", blkpos));
           continue;
         }
 
@@ -4468,8 +4492,7 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
 
   if (stream->src->eof_known)
     {
-      source_size = xd3_source_eof (stream->src);
-
+      xoff_t source_size = xd3_source_eof (stream->src);
       if (stream->srcwin_cksum_pos >= source_size)
         {
           /* This invariant is needed for xd3_source_cksum_offset() */
@@ -4482,9 +4505,22 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point)
     }
 
   /* How long until this function should be called again. */
-  XD3_ASSERT(stream->srcwin_cksum_pos >= logical_input_cksum_pos);
-  *next_move_point = stream->input_position + 1 +
-    (usize_t)(stream->srcwin_cksum_pos - logical_input_cksum_pos);
+  XD3_ASSERT(stream->srcwin_cksum_pos >= target_cksum_pos);
+
+  *next_move_point = stream->input_position +
+    stream->src->blksize -
+    ((stream->srcwin_cksum_pos - target_cksum_pos) & stream->src->maskby);
+  
+  IF_DEBUG2 (DP(RINT
+                "[srcwin_move_point] finished T=%"Q"u "
+                "S=%"Q"u L=%"Q"u EOF=%"Q"u %s again in %u\n",
+                stream->total_in + stream->input_position,
+                stream->srcwin_cksum_pos,
+                target_cksum_pos,
+                xd3_source_eof (stream->src),
+                stream->src->eof_known ? "known" : "unknown",
+                *next_move_point - stream->input_position));
+
   return 0;
 }
 
@@ -4543,6 +4579,8 @@ XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream)
   usize_t        match_offset = 0;
   usize_t        next_move_point;
 
+  IF_DEBUG2(DP(RINT "[string_match] initial entry %u\n", stream->input_position));
+
   /* If there will be no compression due to settings or short input,
    * skip it entirely. */
   if (! (DO_SMALL || DO_LARGE || DO_RUN) ||
@@ -4554,6 +4592,8 @@ XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream)
    * needs to be reset. */
  restartloop:
 
+  IF_DEBUG2(DP(RINT "[string_match] restartloop %u\n", stream->input_position));
+
   /* If there is not enough input remaining for any kind of match,
      skip it. */
   if (stream->input_position + SLOOK > stream->avail_in) { goto loopnomore; }
@@ -4597,7 +4637,9 @@ XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream)
     {
       /* Source window: next_move_point is the point that
        * stream->input_position must reach before computing more
-       * source checksum. */
+       * source checksum.  Note: this is called unconditionally
+       * the first time after reentry, subsequent calls will be
+       * avoided if next_move_point is > input_position */
       if ((ret = xd3_srcwin_move_point (stream, & next_move_point)))
         {
           return ret;
diff --git a/xdelta3/xdelta3.h b/xdelta3/xdelta3.h
index 63b76b0..f00f887 100644
--- a/xdelta3/xdelta3.h
+++ b/xdelta3/xdelta3.h
@@ -833,10 +833,6 @@ struct _xd3_source
   xoff_t              max_blkno;  /* Maximum block, if eof is known,
                                    * otherwise, equals frontier_blkno
                                    * (initially 0). */
-  xoff_t              frontier_blkno;  /* If eof is unknown, the next
-                                        * source position to be read.
-                                        * Otherwise, equal to
-                                        * max_blkno. */
   usize_t             onlastblk;  /* Number of bytes on max_blkno */
   int                 eof_known;  /* Set to true when the first
                                    * partial block is read. */