fix two alloc checks in djw.h

fix setting of stream->iopt_size
fix small_page_test.c

4 files changed, 102 insertions, 58 deletions

diff --git a/xdelta3/examples/Makefile b/xdelta3/examples/Makefile
index 657e08a..dd1f036 100755
--- a/xdelta3/examples/Makefile
+++ b/xdelta3/examples/Makefile
@@ -1,6 +1,9 @@
-CFLAGS = -g -O2 -Wall -I ..
+CFLAGS = -g -Wall -I ..
 
 SOURCES = small_page_test.c
 
 small_page_test: $(SOURCES)
-        $(CC) $(CFLAGS) small_page_test.c -o small_page_test -DXD3_USE_LARGEFILE64=1 -DSECONDARY_DJW=1
+        $(CC) $(CFLAGS) small_page_test.c -o small_page_test -DXD3_USE_LARGEFILE64=1 -DSECONDARY_DJW=1 -DXD3_DEBUG=1
+
+clean:
+        rm -f *.exe *.stackdump
diff --git a/xdelta3/examples/small_page_test.c b/xdelta3/examples/small_page_test.c
index 2d9fb1c..714fd6e 100755
--- a/xdelta3/examples/small_page_test.c
+++ b/xdelta3/examples/small_page_test.c
@@ -2,17 +2,19 @@
 
 #include <stdio.h>
 
-#define SPACE_MAX 32768
 #define PAGE_SIZE 4096
-#define OUTPUT_MAX 1024
-#define IOPT_SIZE 1024
-#define XD3_ALLOCSIZE 256
 
-// typedef void*  (xd3_alloc_func)    (void       *opaque,
-//                                  usize_t      items,
-//                                  usize_t      size);
-// typedef void   (xd3_free_func)     (void       *opaque,
-//                                  void       *address);
+#define SPACE_MAX 131072   // how much memory per process
+#define OUTPUT_MAX 1024    // max size for output
+#define XD3_ALLOCSIZE 256  // internal size for various buffers
+#define IOPT_SIZE 128      // instruction buffer
+
+// SPACE_MAX of 32K is sufficient for most inputs with XD3_COMPLEVEL_1
+// XD3_COMPLEVEL_9 requires about 4x more space than XD3_COMPLEVEL_1
+
+// TODO: experimental PROMOTE code in the encoder is using 16 bytes
+// per entry in a table, where 4 bytes are sufficient w/o this
+// (unimpressive) feature. Remove PROMOTE code.
 
 #include "xdelta3.h"
 #include "xdelta3.c"
@@ -22,6 +24,8 @@ typedef struct _context {
   int allocated;
 } context_t;
 
+static int max_allocated = 0;
+
 void*
 process_alloc (void* opaque, usize_t items, usize_t size)
 {
@@ -39,6 +43,11 @@ process_alloc (void* opaque, usize_t items, usize_t size)
   return ret;
 }
 
+void
+process_free (void* opaque, void *ptr)
+{
+}
+
 int
 process_page (int            is_encode,
               int          (*func) (xd3_stream *),
@@ -49,46 +58,61 @@ process_page (int            is_encode,
               usize_t       *output_size,
               usize_t        output_size_max,
               int            flags) {
-  xd3_stream *stream;
-  xd3_config *config;
-  xd3_source *src;
-  context_t *ctx = calloc(OUTPUT_MAX, 1);
+
+  /* On my x86 this is 1072 of objects on the stack */
+  xd3_stream stream;
+  xd3_config config;
+  xd3_source src;
+  context_t *ctx = calloc(SPACE_MAX, 1);
   int ret;
 
-  ctx->buffer = ((char*)ctx) + sizeof(*ctx);
+  if (ctx == NULL)
+    {
+      printf("calloc failed\n");
+      return -1;
+    }
+
+  ctx->buffer = (uint8_t*)ctx;
   ctx->allocated = sizeof(*ctx);
 
-  stream = process_alloc (ctx, 1, sizeof(*stream));
-  config = process_alloc (ctx, 1, sizeof(*config));
-  src = process_alloc (ctx, 1, sizeof(*src));
-
-  config->flags = flags;
-  config->winsize = PAGE_SIZE;
-  config->sprevsz = PAGE_SIZE;
-  config->srcwin_maxsz = PAGE_SIZE;
-  config->iopt_size = IOPT_SIZE;
-  config->alloc = &process_alloc;
-  config->opaque = (void*) ctx;
-
-  src->size = PAGE_SIZE;
-  src->blksize = PAGE_SIZE;
-  src->onblk = PAGE_SIZE;
-  src->curblk = source;
-  src->curblkno = 0;
-
-  if ((ret = xd3_config_stream (stream, config)) != 0 ||
-      (ret = xd3_set_source (stream, src)) != 0 ||
+  config.flags = flags;
+  config.winsize = PAGE_SIZE;
+  config.sprevsz = PAGE_SIZE;
+  config.srcwin_maxsz = PAGE_SIZE;
+  config.iopt_size = IOPT_SIZE;
+  config.alloc = &process_alloc;
+  config.freef = &process_free;
+  config.opaque = (void*) ctx;
+
+  src.size = PAGE_SIZE;
+  src.blksize = PAGE_SIZE;
+  src.onblk = PAGE_SIZE;
+  src.curblk = source;
+  src.curblkno = 0;
+
+  if ((ret = xd3_config_stream (&stream, &config)) != 0 ||
+      (ret = xd3_set_source (&stream, &src)) != 0 ||
       (ret = xd3_process_stream (is_encode,
-                                 stream,
+                                 &stream,
                                  func, 1,
-                                 input, PAGE_SIZE,
+                                 input, input_size,
                                  output, output_size,
                                  output_size_max)) != 0)
     {
-      // (void) 0;
+      if (stream.msg != NULL)
+        {
+          fprintf(stderr, "stream message: %s\n", stream.msg);
+        }
+    }
+
+  xd3_free_stream (&stream);
+  if (max_allocated < ctx->allocated)
+    {
+      max_allocated = ctx->allocated;
+      fprintf(stderr, "max allocated %d\n", max_allocated);
     }
 
-  xd3_free_stream (stream);
+  free(ctx);
   return ret;
 }
 
@@ -97,14 +121,14 @@ int test(int stride, int encode_flags)
   uint8_t frompg[PAGE_SIZE];
   uint8_t topg[PAGE_SIZE];
   uint8_t output[OUTPUT_MAX];
-  uint8_t reout[OUTPUT_MAX];
+  uint8_t reout[PAGE_SIZE];
   usize_t output_size;
   usize_t re_size;
   int i, j, ret;
 
   for (i = 0; i < PAGE_SIZE; i++)
     {
-      topg[i] = frompg[i] = lrand48();
+      topg[i] = frompg[i] = (rand() >> 3 ^ rand() >> 6 ^ rand() >> 9);
     }
 
   // change 1 byte every stride
@@ -122,26 +146,38 @@ int test(int stride, int encode_flags)
                            &output_size, OUTPUT_MAX,
                            encode_flags)) != 0)
     {
+      fprintf (stderr, "encode failed: stride %u flags 0x%x\n", stride, encode_flags);
       return ret;
     }
 
-  if ((ret = process_page (1, xd3_decode_input,
+  if ((ret = process_page (0, xd3_decode_input,
                            output, output_size,
                            frompg, reout,
                            &re_size, PAGE_SIZE,
                            0)) != 0)
     {
+      fprintf (stderr, "decode failed: stride %u output_size %u flags 0x%x\n",
+              stride, output_size, encode_flags);
       return ret;
     }
 
   if (output_size > OUTPUT_MAX || re_size != PAGE_SIZE)
     {
-      printf ("internal error\n");
+      fprintf (stderr, "internal error: %u != %u\n", output_size, re_size);
       return -1;
     }
 
-  printf("stride %d flags 0x%x size %u ", stride, encode_flags, output_size);
-  printf("%s\n", (ret == 0) ? "OK" : "FAIL");
+  for (i = 0; i < PAGE_SIZE; i++)
+    {
+      if (reout[i] != topg[i])
+        {
+          fprintf (stderr, "encode-decode error: position %d\n", i);
+          return -1;
+        }
+    }
+
+  fprintf(stderr, "stride %d flags 0x%x size %u ", stride, encode_flags, output_size);
+  fprintf(stderr, "%s\n", (ret == 0) ? "OK" : "FAIL");
 
   return 0;
 }
@@ -150,20 +186,15 @@ int main()
 {
   int stride;
   int level;
-  int ret;
 
   for (level = 1; level < 10; level = (level == 1 ? 3 : level + 3))
     {
       int lflag = level << XD3_COMPLEVEL_SHIFT;
 
-      for (stride = 0; stride <= PAGE_SIZE; stride += PAGE_SIZE / 64)
+      for (stride = 2; stride <= PAGE_SIZE; stride += 2)
         {
-          if ((ret = test(stride, lflag)) ||
-              (ret = test(stride, lflag | XD3_ADLER32)) ||
-              (ret = test(stride, lflag | XD3_SEC_DJW)))
-            {
-              return ret;
-            }
+          test(stride, lflag);
+          test(stride, lflag | XD3_SEC_DJW);
         }
     }
 
diff --git a/xdelta3/xdelta3-djw.h b/xdelta3/xdelta3-djw.h
index deff7cb..385775a 100644
--- a/xdelta3/xdelta3-djw.h
+++ b/xdelta3/xdelta3-djw.h
@@ -911,8 +911,15 @@ xd3_real_encode_huff (xd3_stream   *stream,
                                   DJW_SECTORSZ_BITS, (sector_size/DJW_SECTORSZ_MULT)-1))) { goto failure; }
 
       /* Dynamic allocation. */
-      if (gbest     == NULL) { gbest     = xd3_alloc (stream, gbest_max, 1); }
-      if (gbest_mtf == NULL) { gbest_mtf = xd3_alloc (stream, gbest_max, 1); }
+      if (gbest == NULL)
+        {
+          if ((gbest = xd3_alloc (stream, gbest_max, 1)) == NULL) { ret = ENOMEM; goto failure; }
+        }
+
+      if (gbest_mtf == NULL)
+        {
+          if ((gbest_mtf = xd3_alloc (stream, gbest_max, 1)) == NULL) { ret = ENOMEM; goto failure; }
+        }
 
       /* OPT: Some of the inner loops can be optimized, as shown in bzip2 */
 
@@ -1005,6 +1012,7 @@ xd3_real_encode_huff (xd3_stream   *stream,
               if (gcost[gp] < best) { best = gcost[gp]; winner = gp; }
             }
 
+          XD3_ASSERT(gbest_no < gbest_max);
           gbest[gbest_no++] = winner;
           IF_DEBUG1 (gcount[winner] += 1);
 
diff --git a/xdelta3/xdelta3.c b/xdelta3/xdelta3.c
index 5535d07..7042903 100644
--- a/xdelta3/xdelta3.c
+++ b/xdelta3/xdelta3.c
@@ -2357,7 +2357,7 @@ xd3_free_stream (xd3_stream *stream)
   memset (stream, 0, sizeof (xd3_stream));
 }
 
-#if (VCDIFF_TOOLS)
+#if (XD3_DEBUG > 1 || VCDIFF_TOOLS)
 static const char*
 xd3_rtype_to_string (xd3_rtype type, int print_mode)
 {
@@ -2424,7 +2424,7 @@ xd3_config_stream(xd3_stream *stream,
     }
   else
     {
-      stream->iopt_size = XD3_DEFAULT_IOPT_SIZE;
+      stream->iopt_size = config->iopt_size;
     }
 
   stream->getblk    = config->getblk;
@@ -3133,10 +3133,12 @@ xd3_iopt_get_slot (xd3_stream *stream, xd3_rinst** iptr)
       if (stream->iopt_unlimited)
         {
           int elts = XD3_ALLOCSIZE / sizeof(xd3_rinst);
+
           if ((ret = xd3_alloc_iopt (stream, elts)))
             {
               return ret;
             }
+
           stream->iopt_size += elts;
         }
       else