Cleanups, working on the python module, which has assertion failures.

9 files changed, 41 insertions, 90 deletions

diff --git a/xdelta3/Makefile b/xdelta3/Makefile
index 34437e9..d6a6b5b 100755
--- a/xdelta3/Makefile
+++ b/xdelta3/Makefile
@@ -29,10 +29,10 @@ PYFILES = xdelta3-regtest.py setup.py
 EXTRA = Makefile COPYING linkxd3lib.c badcopy.c www
 
 # $Format: "REL=$Xdelta3Version$" $
-REL=0g
+REL=0h_pre0
 RELDIR = xdelta3$(REL)
 
-all: xdelta3
+all: xdelta3 $(PYTGT)
 
 all-targets: $(TARGETS)
 
@@ -48,18 +48,19 @@ tar:
 clean:
         rm -f $(TARGETS)
 
-$(PYTGT): $(SOURCES)
-        $(PYTHON) setup.py install --compile --force
-
 xdelta3: $(SOURCES)
         $(CC) -O3 -Wall -Wshadow xdelta3.c -lm -o xdelta3 \
               -DXD3_DEBUG=0 \
               -DXD3_USE_LARGEFILE64=1 \
               -DREGRESSION_TEST=1 \
               -DSECONDARY_DJW=1 \
+              -DSECONDARY_FGK=1 \
               -DXD3_MAIN=1 \
               -DXD3_POSIX=1
 
+$(PYTGT): $(SOURCES)
+        $(PYTHON) setup.py install --verbose --compile --force
+
 xdelta3-1: $(SOURCES)
         $(CC) -g -Wall -Wshadow xdelta3.c -o xdelta3-1 -DXD3_MAIN=1 -DGENERIC_ENCODE_TABLES=1 \
                 -DXD3_USE_LARGEFILE64=1 -DREGRESSION_TEST=1 -DXD3_DEBUG=1 -DSECONDARY_DJW=1 -lm
diff --git a/xdelta3/junk.py b/xdelta3/junk.py
deleted file mode 100755
index 384951e..0000000
--- a/xdelta3/junk.py
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/usr/bin/python
-
-bytes = ''
-
-for x in range(0, 250):
-  bytes = bytes + ('%c%c%c%c=' % (x, x+1, x+2, x+3))
-
-for x in range(0, 250):
-  bytes = bytes + ('%c' % x)
-  
-print bytes
diff --git a/xdelta3/www/xdelta3-cmdline.html b/xdelta3/www/xdelta3-cmdline.html
index cabb547..19f59ae 100755
--- a/xdelta3/www/xdelta3-cmdline.html
+++ b/xdelta3/www/xdelta3-cmdline.html
@@ -128,33 +128,38 @@ VCDIFF addr section length:   3
 <pre>
 usage: xdelta3 [command/options] [input [output]]
 commands are:
-    encode      encodes the input
-    decode      decodes the input
     config      prints xdelta3 configuration
+    decode      decodes the input
+    encode      encodes the input
     test        run the builtin tests
 special commands for VCDIFF inputs:
+    printdelta  print information about the entire delta
     printhdr    print information about the first window
     printhdrs   print information about all windows
-    printdelta  print information about the entire delta
 options are:
+   -0 .. -9     compression level
+   -A [apphead] disable/provide application header
+   -B blksize   source file block size
+   -C           soft config (see xdelta3-cfgs.h)
    -c           use stdout instead of default
+   -D           disable external decompression (encode/decode)
    -d           same as decode command
    -e           same as encode command
    -f           force overwrite
+   -h           show help
+   -J           disable output (check/compute only)
+   -M memsize   memory budget for hash tables
    -n           disable checksum (encode/decode)
-   -D           disable external decompression (encode/decode)
-   -R           disable external recompression (decode)
    -N           disable small string-matching compression
+   -P           repeat count (for profiling)
+   -q           be quiet
+   -R           disable external recompression (decode)
    -S [djw|fgk] disable/enable secondary compression
-   -A [apphead] disable/provide application header
    -s source    source file to copy from (if any)
-   -B blksize   source file block size
-   -W winsize   input window buffer size
+   -T           use alternate code table
    -v           be verbose (max 2)
-   -q           be quiet
-   -h           show help
    -V           show version
-   -P           repeat count (for profiling)
+   -W winsize   input window buffer size
 </pre>
 <p>
 
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h
index 4a84e67..26a12e1 100755
--- a/xdelta3/xdelta3-main.h
+++ b/xdelta3/xdelta3-main.h
@@ -214,7 +214,7 @@ XD3_MAKELIST(main_blklru_list,main_blklru,link);
 /* Program options: various command line flags and options. */
 static int         option_stdout             = 0;
 static int         option_force              = 0;
-static int         option_verbose            = 0;
+static int         option_verbose            = 3;
 static int         option_quiet              = 0;
 static int         option_level              = 6;
 static int         option_use_appheader      = 1;
@@ -232,24 +232,6 @@ static usize_t     option_winsize            = XD3_DEFAULT_WINSIZE;
 static usize_t     option_srcwinsz           = XD3_DEFAULT_SRCWINSZ;
 static usize_t     option_memsize            = XD3_DEFAULT_MEMSIZE;
 
-/* Wishful TODO: Support should probably be for partial deltas & update-in-place deltas,
- * following the latest draft RFC specs partial deltas [the changes have moderate
- * complexity].  The following flags implement primitive controls to skip sections
- * of the input & output, mainly for debugging purposes. */
-
-/* DECODE-ONLY: Skips processing windows up to first_window and past last_window using the
- * XD3_SKIP_WINDOW flag, but main_ still reads reads/parses every window.  TODO: make it
- * meaningful for encode, etc... */
-/*static xoff_t      option_first_window = 0;*/
-/*static xoff_t      option_last_window  = XOFF_T_MAX;*/
-
-/* ENCODE-ONLY: Seeks to first_offset, EOF at last_offset, done entirely in this main_
- * routines, so the library actually sees a shortened input.  TODO: implement this for
- * decode, implement proper partial deltas, works with external compression?, works with
- * non-seekable inputs?, change ranges, etc... */
-/*static xoff_t      option_first_offset = 0;*/
-/*static xoff_t      option_last_offset  = XOFF_T_MAX;*/
-
 /* This controls the number of times main repeats itself, only for profiling. */
 static int option_profile_cnt = 0;
 
@@ -307,7 +289,7 @@ static int
 main_version (void)
 {
   /* $Format: "  P(RINT \"VERSION=3.$Xdelta3Version$\\n\");" $ */
-  P(RINT "VERSION=3.0g\n");
+  P(RINT "VERSION=3.0h_pre0\n");
   return EXIT_SUCCESS;
 }
 
@@ -345,7 +327,7 @@ main_malloc1 (usize_t size)
 {
   void* r = malloc (size);
   if (r == NULL) { XPR(NT "malloc: %s\n", xd3_strerror (ENOMEM)); }
-  else if (option_verbose > 2) { XPR(NT "malloc: %u\n", size); }
+  else if (option_verbose > 2) { XPR(NT "malloc: %u: %p\n", size, r); }
   return r;
 }
 
@@ -366,19 +348,20 @@ main_alloc (void   *opaque,
 }
 
 static void
-main_free (void *ptr)
+main_free1 (void *opaque, void *ptr)
 {
-  if (ptr)
-    {
-      IF_DEBUG (main_mallocs -= 1); 
-      free (ptr);
-    }
+  if (option_verbose > 2) { XPR(NT "free: %p\n", ptr); }
+  free (ptr);
 }
 
 static void
-main_free1 (void *opaque, void *ptr)
+main_free (void *ptr)
 {
-  free (ptr);
+  if (ptr)
+    {
+      IF_DEBUG (main_mallocs -= 1);
+      main_free1 (NULL, ptr);
+    }
 }
 
 /* This ensures that (ret = errno) always indicates failure, in case errno was
@@ -1820,19 +1803,6 @@ main_set_source (xd3_stream *stream, int cmd, main_file *sfile, xd3_source *sour
   lru_size = (option_srcwinsz / source->blksize);
   lru_size = max(1, lru_size);
 
-  if (lru_size > 128)
-    {
-      /* TODO: fix performance here, the LRU is scanned sequentially to find blocks,
-       * except in encode, where it uses FIFO.  Fix is to set source->blksize
-       * accordingly. */
-      if (IS_ENCODE(cmd) && ! option_quiet)
-        {
-          XPR(NT "warning: large source window (--B %u) "
-                 "hurts performance except for encoding\n",
-              option_srcwinsz);
-        }
-    }
-
   if ((lru = main_malloc (sizeof (main_blklru) * lru_size)) == NULL)
     {
       return ENOMEM;
@@ -2073,9 +2043,6 @@ main_getblk_func (xd3_stream *stream,
 /* This is a generic input function.  It calls the xd3_encode_input or xd3_decode_input
  * functions and makes calls to the various input handling routines above, which
  * coordinate external decompression.
- *
- * TODO config: Still need options for the at least: smatch config, memsize, sprevsz,
- * XD3_SEC_* flags, greedy/1.5
  */
 static int
 main_input (xd3_cmd     cmd,
@@ -2570,7 +2537,7 @@ main (int argc, char **argv)
    * main() and thus care about freeing all memory.  I never had much trust for getopt
    * anyway, it's too opaque.  This implements a fairly standard non-long-option getopt
    * with support for named operations (e.g., "xdelta3 [encode|decode|printhdr...] < in >
-   * out").  I'll probably add long options at some point. See TODO. */
+   * out").  I'll probably add long options at some point. */
   if (my_optstr)
     {
       if (*my_optstr == '-')    { my_optstr += 1; }
diff --git a/xdelta3/xdelta3-regtest.py b/xdelta3/xdelta3-regtest.py
index f3313a4..e8d0339 100755
--- a/xdelta3/xdelta3-regtest.py
+++ b/xdelta3/xdelta3-regtest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2.3
+#!/usr/bin/python2.4
 # xdelta 3 - delta compression tools and library
 # Copyright (C) 2003 and onward.  Joshua P. MacDonald
 #
@@ -51,7 +51,7 @@ FORK  = 0
 
 #
 #
-RCSDIR = '/Volumes/LACIE120RAID/orbit_linux/home/jmacd/PRCS/prcs/b'
+RCSDIR = '/mnt/polaroid/Polaroid/orbit_linux/home/jmacd/PRCS/prcs/b'
 
 TMPDIR = '/tmp/xd3regtest.%d' % os.getpid()
 
@@ -551,7 +551,7 @@ class Xdelta3Run1:
         if self.reps:
             assert(reps == 1)
             reps = self.reps
-        RunXdelta3(['-P', '%d' % reps, '-efq', self.file, DFILE],kind=self.kind)
+        RunXdelta3(['-vvv', '-P', '%d' % reps, '-efq', self.file, DFILE],kind=self.kind)
         if trial > 0:
             return None
         return Xdelta3Info(self.file,DFILE)
diff --git a/xdelta3/xdelta3-test.h b/xdelta3/xdelta3-test.h
index d076656..3c50bb6 100755
--- a/xdelta3/xdelta3-test.h
+++ b/xdelta3/xdelta3-test.h
@@ -45,14 +45,7 @@ static int test_exponential_dist (usize_t mean, usize_t max);
 
 #define CHECK(cond) if (!(cond)) { P(RINT "check failure: " #cond); abort(); }
 
-/* TODO
- *
- * 1. Test state changes: that config is called before open, open called before
- * encode/decode..., close incomplete works, invalid options, consume_output always
- * called, no mixing of encode/decode, etc.
- *
- * 2. Test window selection, window alignment, 1.5 pass alg vs. greedy
- */
+/* TODO: Test 1.5 pass alg vs. greedy */
 
 /******************************************************************************************
  TEST HELPERS
diff --git a/xdelta3/xdelta3.c b/xdelta3/xdelta3.c
index f7514e5..a26a93c 100755
--- a/xdelta3/xdelta3.c
+++ b/xdelta3/xdelta3.c
@@ -239,6 +239,7 @@
      xdelta3-main.h     The command-line interface, external compression
                         support, POSIX-specific, info & VCDIFF-debug tools.
      xdelta3-second.h   The common secondary compression routines.
+     xdelta3-decoder.h  All decoding routines.
      xdelta3-djw.h      The semi-adaptive huffman secondary encoder.
      xdelta3-fgk.h      The adaptive huffman secondary encoder.
      xdelta3-test.h     The unit test covers major algorithms,
@@ -268,8 +269,6 @@
                         generated using an expoential distribution
                         which approximates the expected error_prob
                         distribution.
-     show.c             Prints an offset/length segment from a file.
-     testh.c            Checks that xdelta3.h is can be #included
    --------------------------------------------------------------------
 
    This file itself is unusually large.  I hope to defend this layout
diff --git a/xdelta3/xdelta3.h b/xdelta3/xdelta3.h
index 90b2fd9..0f11854 100755
--- a/xdelta3/xdelta3.h
+++ b/xdelta3/xdelta3.h
@@ -940,7 +940,6 @@ void    xd3_avail_input  (xd3_stream    *stream,
    * a stream->winsize buffer.  This is to avoid an unwanted allocation. */
   XD3_ASSERT (idata != NULL);
 
-  /* TODO: Should check for a call to xd3_avail_input in the wrong state. */
   stream->next_in  = idata;
   stream->avail_in = isize;
 }
@@ -950,7 +949,6 @@ void    xd3_avail_input  (xd3_stream    *stream,
 static inline
 void xd3_consume_output (xd3_stream  *stream)
 {
-  /* TODO: Is it correct to set avail_in = 0 here, then check == 0 in avail_in? */
   stream->avail_out  = 0;
 }
 
diff --git a/xdelta3/xdelta3.prj b/xdelta3/xdelta3.prj
index 02a254d..9ce7618 100755
--- a/xdelta3/xdelta3.prj
+++ b/xdelta3/xdelta3.prj
@@ -9,7 +9,7 @@
 (Checkin-Login jmacd)
 (Populate-Ignore ())
 (Project-Keywords
- (Xdelta3Version "0g")
+ (Xdelta3Version "0h_pre0")
  (WWWLeftNavBar "<table cellpadding=\"20px\" width=700> <tr> <td class=\"leftbdr\" valign=top height=600 width=100> <div class=\"leftbody\"> <h1>Xdelta</h1> <a href=\"xdelta3.html\">overview</a><br> <a href=\"xdelta3-cmdline.html\">command&nbsp;line</a><br> <a href=\"xdelta3-api-guide.html\">api&nbsp;guide</a><br> <br><a href=\"http://xdelta.org\">xdelta.org</a></h2> </div> </td> <td valign=top width=500>")
  )
 (Files
@@ -41,7 +41,6 @@
   (junk.py ())
   (linkxd3lib.c (xdelta3/19_linkxd3lib 1.1 644))
   (rcs_junk.cc (xdelta3/15_rcs_junk.c 1.1 644))
-  (show.c (xdelta3/18_show.c 1.1 644))
 )
 (Merge-Parents)
 (New-Merge-Parents)