Fix some TODOs (option_srcwinsz up to 4GB?

6 files changed, 54 insertions, 80 deletions

diff --git a/xdelta3/Makefile.am b/xdelta3/Makefile.am
index 2b35e65..8283663 100644
--- a/xdelta3/Makefile.am
+++ b/xdelta3/Makefile.am
@@ -39,7 +39,7 @@ WFLAGS = -Wall -Wshadow -fno-builtin -Wextra -Wsign-compare \
          -Wextra -Wno-unused-parameter
 
 C_WFLAGS = $(WFLAGS) -pedantic -std=c99
-CXX_WFLAGS = $(WFLAGS) -pedantic
+CXX_WFLAGS = $(WFLAGS)
 
 common_CFLAGS = \
               -DGENERIC_ENCODE_TABLES=0 \
@@ -54,7 +54,8 @@ if DEBUG_SYMBOLS
   common_CFLAGS += -g
 endif
 
-xdelta3_CFLAGS = $(C_WFLAGS) $(common_CFLAGS) -DXD3_DEBUG=1
+# For additional debugging, add -DXD3_DEBUG=1, 2, 3, ...
+xdelta3_CFLAGS = $(C_WFLAGS) $(common_CFLAGS) -DXD3_DEBUG=0
 xdelta3_LDADD = -lm
 
 xdelta3decode_CFLAGS = \
diff --git a/xdelta3/xdelta3-blkcache.h b/xdelta3/xdelta3-blkcache.h
index e8a42a6..27263ed 100644
--- a/xdelta3/xdelta3-blkcache.h
+++ b/xdelta3/xdelta3-blkcache.h
@@ -1,6 +1,6 @@
 /* xdelta 3 - delta compression tools and library
  * Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007,
- * 2008, 2009, 2010
+ * 2008, 2009, 2010, 2011, 2012, 2013
  * Joshua P. MacDonald
  *
  *  This program is free software; you can redistribute it and/or modify
@@ -18,9 +18,6 @@
  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-/* TODO: This code is heavily revised from 3.0z but still needs major
- * refactoring. */
-
 #include "xdelta3-internal.h"
 
 typedef struct _main_blklru      main_blklru;
@@ -36,12 +33,13 @@ struct _main_blklru
 {
   uint8_t          *blk;
   xoff_t            blkno;
-  usize_t           size;
+  xoff_t            size;
   main_blklru_list  link;
 };
 
 #define MAX_LRU_SIZE 32U
 #define XD3_MINSRCWINSZ (XD3_ALLOCSIZE * MAX_LRU_SIZE)
+#define XD3_MAXSRCWINSZ (1ULL << 32)
 
 XD3_MAKELIST(main_blklru_list,main_blklru,link);
 
@@ -87,34 +85,25 @@ main_set_source (xd3_stream *stream, xd3_cmd cmd,
                  main_file *sfile, xd3_source *source)
 {
   int ret = 0;
-  usize_t i;
+  int i;
   xoff_t source_size = 0;
-  usize_t blksize;
 
   XD3_ASSERT (lru == NULL);
   XD3_ASSERT (stream->src == NULL);
   XD3_ASSERT (option_srcwinsz >= XD3_MINSRCWINSZ);
 
-  /* TODO: this code needs refactoring into FIFO, LRU, FAKE.  Yuck!
-   * This is simplified from 3.0z which had issues with sizing the
-   * source buffer memory allocation and the source blocksize. */
-
   /* LRU-specific */
   main_blklru_list_init (& lru_list);
 
   if (allow_fake_source)
     {
-      /* TODO: refactor
-       * TOOLS/recode-specific: Check "allow_fake_source" mode looks
-       * broken now. */
       sfile->mode = XO_READ;
       sfile->realname = sfile->filename;
       sfile->nread = 0;
     }
   else
     {
-      /* Either a regular file (possibly compressed) or a FIFO
-       * (possibly compressed). */
+      /* Either a regular file or a FIFO. Both are possibly compressed. */
       if ((ret = main_file_open (sfile, sfile->filename, XO_READ)))
         {
           return ret;
@@ -152,22 +141,24 @@ main_set_source (xd3_stream *stream, xd3_cmd cmd,
       return ret;
     }
 
-  /* Main calls main_getblk_func() once before xd3_set_source().  This
-   * is the point at which external decompression may begin.  Set the
-   * system for a single block. */
-  lru_size = 1;
-  lru[0].blkno = (xoff_t) -1;
-  blksize = option_srcwinsz;
-  main_blklru_list_push_back (& lru_list, & lru[0]);
-  XD3_ASSERT (blksize != 0);
+  /* Setup LRU blocks. */
+  for (i = MAX_LRU_SIZE - 1; i >= 0; --i)
+    {
+      lru[i].blk = lru[0].blk + (option_srcwinsz / MAX_LRU_SIZE) * i;
+      lru[i].blkno = (xoff_t) -1;
+      lru[i].size = 0;
+      main_blklru_list_push_back (& lru_list, & lru[i]);
+    }
 
   /* Initialize xd3_source. */
-  source->blksize  = blksize;
   source->name     = sfile->filename;
   source->ioh      = sfile;
   source->curblkno = (xoff_t) -1;
   source->curblk   = NULL;
   source->max_winsize = option_srcwinsz;
+  source->blksize = option_srcwinsz;
+
+  lru_size = 1;
 
   if ((ret = main_getblk_func (stream, source, 0)) != 0)
     {
@@ -177,44 +168,26 @@ main_set_source (xd3_stream *stream, xd3_cmd cmd,
       return ret;
     }
 
-  source->onblk = lru[0].size;  /* xd3 sets onblk */
-
-  /* If the file is smaller than a block, size is known. */
-  if (!sfile->size_known && source->onblk < blksize)
+  /* If the file is smaller than the buffer, size is known. */
+  if (source->onblk < source->blksize)
     {
       source_size = source->onblk;
       sfile->size_known = 1;
     }
-
-  /* If the size is not known or is greater than the buffer size, we
-   * split the buffer across MAX_LRU_SIZE blocks (already allocated in
-   * "lru"). */
-  if (!sfile->size_known || source_size > option_srcwinsz)
+  else
     {
-      /* Modify block 0, change blocksize. */
-      blksize = option_srcwinsz / MAX_LRU_SIZE;
-      source->blksize = blksize;
-      source->onblk = blksize;  /* xd3 sets onblk */
-      /* Note: source->max_winsize is unchanged. */
-      lru[0].size = blksize;
+      /* Split the buffer */
+      source->blksize = option_srcwinsz / MAX_LRU_SIZE;
+      source->onblk = source->blksize;
       lru_size = MAX_LRU_SIZE;
 
-      /* Setup rest of blocks. */
-      for (i = 1; i < lru_size; i += 1)
+      for (i = 0; i < (int) MAX_LRU_SIZE; ++i)
         {
-          lru[i].blk = lru[0].blk + (blksize * i);
           lru[i].blkno = i;
-          lru[i].size = blksize;
-          main_blklru_list_push_back (& lru_list, & lru[i]);
+          lru[i].size = source->blksize;
         }
     }
 
-  if (! sfile->size_known)
-    {
-      /* If the size is not know, we must use FIFO discipline. */
-      do_src_fifo = 1;
-    }
-
   /* Call the appropriate set_source method, handle errors, print
    * verbose message, etc. */
   if (sfile->size_known)
@@ -223,6 +196,8 @@ main_set_source (xd3_stream *stream, xd3_cmd cmd,
     }
   else
     {
+      /* If the size is not known, we must use FIFO discipline. */
+      do_src_fifo = 1;
       ret = xd3_set_source (stream, source);
     }
 
@@ -233,7 +208,6 @@ main_set_source (xd3_stream *stream, xd3_cmd cmd,
     }
 
   XD3_ASSERT (stream->src == source);
-  XD3_ASSERT (source->blksize == blksize);
 
   if (option_verbose)
     {
@@ -264,7 +238,7 @@ main_set_source (xd3_stream *stream, xd3_cmd cmd,
       XPR(NT "source %s %s blksize %s window %s%s%s\n",
           sfile->filename,
           srcszbuf.buf,
-          main_format_bcnt (blksize, &blkszbuf),
+          main_format_bcnt (source->blksize, &blkszbuf),
           main_format_bcnt (option_srcwinsz, &winszbuf),
           nbufs.buf,
           do_src_fifo ? " (FIFO)" : "");
@@ -333,6 +307,14 @@ main_getblk_lru (xd3_source *source, xoff_t blkno,
   return 0;
 }
 
+/* Opens / seeks the source file and:
+ * (a) determines the source stream blocksize
+ * (b) determines the source file size (if possible)
+ * (c) calls main_read_primary_input (i.e., start external decompression)
+ *
+ * The main difficulty concerns an optimization: if the entire source
+ * fits inside the buffer, we decide to use a single block.
+ */
 static int
 main_read_seek_source (xd3_stream *stream,
                        xd3_source *source,
@@ -439,7 +421,7 @@ main_read_seek_source (xd3_stream *stream,
           sfile->source_position += nread;
           blru->size = nread;
 
-          IF_DEBUG1 (DP(RINT "[getblk] skip blkno %"Q"u size %u\n",
+          IF_DEBUG1 (DP(RINT "[getblk] skip blkno %"Q"u size %"Q"u\n",
                         skip_blkno, blru->size));
 
           XD3_ASSERT (sfile->source_position <= pos);
@@ -467,7 +449,6 @@ main_getblk_func (xd3_stream *stream,
   main_file *sfile = (main_file*) source->ioh;
   main_blklru *blru;
   int is_new;
-  int did_seek = 0;
   size_t nread = 0;
 
   if (allow_fake_source)
@@ -505,19 +486,14 @@ main_getblk_func (xd3_stream *stream,
           return ret;
         }
 
-      /* Indicates that another call to main_getblk_lru() may be
-       * needed */
-      did_seek = 1;
+      if ((ret = main_getblk_lru (source, blkno, & blru, & is_new)))
+        {
+          return ret;
+        }
     }
 
   XD3_ASSERT (sfile->source_position == pos);
 
-  if (did_seek &&
-      (ret = main_getblk_lru (source, blkno, & blru, & is_new)))
-    {
-      return ret;
-    }
-
   if ((ret = main_read_primary_input (sfile,
                                       (uint8_t*) blru->blk,
                                       source->blksize,
diff --git a/xdelta3/xdelta3-decode.h b/xdelta3/xdelta3-decode.h
index c0cb578..f867055 100644
--- a/xdelta3/xdelta3-decode.h
+++ b/xdelta3/xdelta3-decode.h
@@ -454,7 +454,7 @@ xd3_decode_output_halfinst (xd3_stream *stream, xd3_hinst *inst)
                     (blkoff + take > source->onblk))
                   {
                     IF_DEBUG1 (XPR(NT "[srcfile] short at blkno %"Q"u onblk "
-                                   "%u blksize %u blkoff %u take %u\n",
+                                   "%"Q"u blksize %u blkoff %u take %u\n",
                                    block,
                                    source->onblk,
                                    blksize,
@@ -1096,7 +1096,7 @@ xd3_decode_input (xd3_stream *stream)
                        "decode cpyoff %"Q"u "
                        "cpyblkno %"Q"u "
                        "cpyblkoff %u "
-                       "blksize %u\n",
+                       "blksize %"Q"u\n",
                        stream->dec_cpyoff,
                        src->cpyoff_blocks,
                        src->cpyoff_blkoff,
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h
index 126271f..3ae24cd 100644
--- a/xdelta3/xdelta3-main.h
+++ b/xdelta3/xdelta3-main.h
@@ -296,9 +296,6 @@ static main_extcomp extcomp_types[] =
   { "gzip",     "-c",   "gzip",       "-dc",   "G", "\037\213",     2, 0 },
   { "compress", "-c",   "uncompress", "-c",    "Z", "\037\235",     2, 0 },
 
-  /* TODO: add commandline support for magic-less formats */
-  /*{ "lzma", "-c",   "lzma", "-dc",   "M", "]\000", 2, 0 },*/
-
   /* Xz is lzma with a magic number http://tukaani.org/xz/ */
   { "xz", "-c", "xz", "-dc", "Y", "\xfd\x37\x7a\x58\x5a\x00", 2, 0 },
 };
@@ -3776,7 +3773,7 @@ int main (int argc, char **argv)
         case 'B': {
           xoff_t bsize;
           if ((ret = main_atoux (my_optarg, & bsize,
-                                 XD3_MINSRCWINSZ, 0, 'B')))
+                                 XD3_MINSRCWINSZ, XD3_MAXSRCWINSZ, 'B')))
             {
               goto exit;
             }
diff --git a/xdelta3/xdelta3.c b/xdelta3/xdelta3.c
index 608622a..c22c763 100644
--- a/xdelta3/xdelta3.c
+++ b/xdelta3/xdelta3.c
@@ -1529,10 +1529,10 @@ xd3_swap_usize_t (usize_t* p1, usize_t* p2)
 
 /* It's not constant time, but it computes the log. */
 static int
-xd3_check_pow2 (xoff_t value, usize_t *logof)
+xd3_check_pow2 (xoff_t value, int *logof)
 {
   xoff_t x = 1;
-  usize_t nolog;
+  int nolog;
   if (logof == NULL) {
     logof = &nolog;
   }
@@ -2653,7 +2653,7 @@ int
 xd3_set_source (xd3_stream *stream,
                 xd3_source *src)
 {
-  usize_t shiftby;
+  int shiftby;
 
   stream->src = src;
   src->srclen  = 0;
@@ -2665,7 +2665,7 @@ xd3_set_source (xd3_stream *stream,
     {
       src->blksize = xd3_pow2_roundup(src->blksize);
       xd3_check_pow2 (src->blksize, &shiftby);
-      IF_DEBUG1 (DP(RINT "raising src_blksz to %u\n", src->blksize));
+      IF_DEBUG1 (DP(RINT "raising src_blksz to %"Q"u\n", src->blksize));
     }
 
   src->shiftby = shiftby;
@@ -2674,7 +2674,7 @@ xd3_set_source (xd3_stream *stream,
   if (xd3_check_pow2 (src->max_winsize, NULL) != 0)
     {
       src->max_winsize = xd3_xoff_roundup(src->max_winsize);
-      IF_DEBUG1 (DP(RINT "raising src_maxsize to %u\n", src->blksize));
+      IF_DEBUG1 (DP(RINT "raising src_maxsize to %"Q"u\n", src->blksize));
     }
   src->max_winsize = max(src->max_winsize, XD3_ALLOCSIZE);
 
diff --git a/xdelta3/xdelta3.h b/xdelta3/xdelta3.h
index 686e05c..66def15 100644
--- a/xdelta3/xdelta3.h
+++ b/xdelta3/xdelta3.h
@@ -718,7 +718,7 @@ struct _xd3_config
 struct _xd3_source
 {
   /* you set */
-  usize_t             blksize;       /* block size */
+  xoff_t              blksize;       /* block size */
   const char         *name;          /* its name, for debug/print
                                         purposes */
   void               *ioh;           /* opaque handle */
@@ -727,14 +727,14 @@ struct _xd3_source
   /* getblk sets */
   xoff_t              curblkno;      /* current block number: client
                                         sets after getblk request */
-  usize_t             onblk;         /* number of bytes on current
+  xoff_t              onblk;         /* number of bytes on current
                                         block: client sets,  must be >= 0
                                         and <= blksize */
   const uint8_t      *curblk;        /* current block array: client
                                         sets after getblk request */
 
   /* xd3 sets */
-  usize_t             srclen;        /* length of this source window */
+  usize_t              srclen;        /* length of this source window */
   xoff_t              srcbase;       /* offset of this source window
                                         in the source itself */
   int                 shiftby;       /* for power-of-two blocksizes */