4 files changed, 3993 insertions, 3957 deletions

diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h
index 79f9617..eb6ca9c 100755
--- a/xdelta3/xdelta3-main.h
+++ b/xdelta3/xdelta3-main.h
@@ -1,2937 +1,2959 @@
-/* xdelta 3 - delta compression tools and library
- * Copyright (C) 2001 and onward.  Joshua P. MacDonald
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-/* This is all the extra stuff you need for convenience to users in a command line
- * application.  It contains these major components:
- *
- * 1. VCDIFF tools
- * 2. external compression support (this is POSIX-specific).
- * 3. a general read/write loop that handles all of the Xdelta decode/encode/VCDIFF-print
- *    functions
- * 4. command-line interpreter
- * 5. an Xdelta application header which stores default filename, external compression settings
- * 6. output/error printing
- * 7. basic file support and OS interface
- */
-
-/* TODO list:
- * 1. do exact gzip-like filename, stdout handling.  make a .xz extension, refuse
- *    to encode to stdout without -cf, etc.
- * 2. Allow the user to add a comment string to the app header without disturbing the default
- *    behavior.
- */
-
-/* On error handling and printing:
- *
- * The xdelta library sets stream->msg to indicate what condition caused an internal
- * failure, but many failures originate here and are printed here.  The return convention
- * is 0 for success, as throughout Xdelta code, but special attention is required here for
- * the operating system calls with different error handling.  See the main_file_* routines.
- * All errors in this file have a message printed at the time of occurance.  Since some of
- * these calls occur within calls to the library, the error may end up being printed again
- * with a more general error message.
- */
-
-/******************************************************************************************/
-
-#ifndef XD3_POSIX
-#define XD3_POSIX 0
-#endif
-#ifndef XD3_STDIO
-#define XD3_STDIO 0
-#endif
-
-/* XPRINTX (used by main) prefixes an "xdelta3: " to the output. */
-#define XPR fprintf
-#define NT stderr, "xdelta3: "
-
-#define VC fprintf
-#define UT vcout,
-
-/* If none are set, default to posix. */
-#if (XD3_POSIX + XD3_STDIO) == 0
-#undef XD3_POSIX
-#define XD3_POSIX 1
-#endif
-
-/* Handle externally-compressed inputs. */
-#ifndef EXTERNAL_COMPRESSION
-#define EXTERNAL_COMPRESSION 1
-#endif
-
-#define PRINTHDR_SPECIAL -4378291
-
-/* The number of soft-config variables.  */
-#define XD3_SOFTCFG_VARCNT 1
-
-/* this is used as in XPR(NT XD3_LIB_ERRMSG (stream, ret)) to print an error message
- * from the library. */
-#define XD3_LIB_ERRMSG(stream, ret) "%s: %s\n", xd3_errstring (stream), xd3_strerror (ret)
-
-#include <stdio.h>  /* fprintf */
-
-#if XD3_POSIX
-#include <unistd.h> /* close, read, write... */
-#include <sys/types.h>
-#include <fcntl.h>
-#endif
-
-#ifndef _WIN32
-#include <unistd.h> /* lots */
-#include <sys/time.h> /* gettimeofday() */
-#include <sys/stat.h> /* stat() and fstat() */
-#else
-#endif
-
-/******************************************************************************************
- ENUMS and TYPES
- ******************************************************************************************/
-
-/* These flags (mainly pertaining to main_read() operations) are set in the
- * main_file->flags variable.  All are related to with external decompression support.
- *
- * RD_FIRST causes the external decompression check when the input is first read.
- *
- * RD_NONEXTERNAL disables external decompression for reading a compressed input, in the
- * case of Xdelta inputs.  Note: Xdelta is supported as an external compression type,
- * which makes is the reason for this flag.  An example to justify this is: to create a
- * delta between two files that are VCDIFF-compressed.  Two external Xdelta decoders are
- * run to supply decompressed source and target inputs to the Xdelta encoder. */
-typedef enum
-{
-  RD_FIRST        = (1 << 0),
-  RD_NONEXTERNAL  = (1 << 1),
-  RD_EXTERNAL_V1  = (1 << 2),
-} xd3_read_flags;
-
-/* main_file->mode values */
-typedef enum
-{
-  XO_READ  = 0,
-  XO_WRITE = 1,
-} main_file_modes;
-
-/* Main commands.  For example, CMD_PRINTHDR is the "xdelta printhdr" command. */
-typedef enum
-{
-  CMD_NONE = 0,
-  CMD_PRINTHDR,
-  CMD_PRINTHDRS,
-  CMD_PRINTDELTA,
-#if XD3_ENCODER
-  CMD_ENCODE,
-#endif
-  CMD_DECODE,
-  CMD_TEST,
-  CMD_CONFIG,
-} xd3_cmd;
-
-#if XD3_ENCODER
-#define CMD_DEFAULT CMD_ENCODE
-#define IS_ENCODE(cmd) (cmd == CMD_ENCODE)
-#else
-#define CMD_DEFAULT CMD_DECODE
-#define IS_ENCODE(cmd) (0)
-#endif
-
-typedef struct _main_file        main_file;
-typedef struct _main_extcomp     main_extcomp;
-typedef struct _main_blklru      main_blklru;
-typedef struct _main_blklru_list main_blklru_list;
-
-/* The main_file object supports abstract system calls like open, close, read, write, seek,
- * stat.  The program uses these to represent both seekable files and non-seekable files.
- * Source files must be seekable, but the target input and any output file do not require
- * seekability.
- */
-struct _main_file
-{
-#if XD3_STDIO
-  FILE               *file;
-#elif XD3_POSIX
-  int                 file;
-#endif
-
-  int                 mode;          /* XO_READ and XO_WRITE */
-  const char         *filename;      /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */
-  char               *filename_copy; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */
-  const char         *realname;      /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */
-  const main_extcomp *compressor; /* External compression struct. */
-  int                 flags;      /* RD_FIRST, RD_NONEXTERNAL, ... */
-  xoff_t              nread;      /* for input position */
-  xoff_t              nwrite;     /* for output position */
-};
-
-/* Various strings and magic values used to detect and call external compression.  See
- * below for examples. */
-struct _main_extcomp
-{
-  const char    *recomp_cmdname;
-  const char    *recomp_options;
-
-  const char    *decomp_cmdname;
-  const char    *decomp_options;
-
-  const char    *ident;
-  const char    *magic;
-  int            magic_size;
-  int            flags;
-};
-
-/* This file implements a small LRU of source blocks.  For encoding purposes,
- * we prevent paging in blocks we've already scanned in the source (return
- * XD3_NOTAVAIL). */
-struct _main_blklru_list
-{
-  main_blklru_list  *next;
-  main_blklru_list  *prev;
-};
-
-struct _main_blklru
-{
-  uint8_t         *blk;
-  xoff_t           blkno;
-  main_blklru_list  link;
-};
-
-/* ... represented as a list (no cache index). */
-XD3_MAKELIST(main_blklru_list,main_blklru,link);
-
-/* Program options: various command line flags and options. */
-static int         option_stdout             = 0;
-static int         option_force              = 0;
-static int         option_verbose            = 0;
-static int         option_quiet              = 0;
-static int         option_level              = 6;
-static int         option_use_appheader      = 1;
-static uint8_t*    option_appheader          = NULL;
-static int         option_use_secondary      = /* until-standardized, leave this off */ 0;
-static char*       option_secondary          = NULL;
-static int         option_use_checksum       = 1;
-static int         option_use_altcodetable   = 0;
-static char*       option_smatch_config      = NULL;
-static int         option_no_compress        = 0;
-static int         option_no_output          = 0; /* go through the motions, but do not open or write output */
-static const char *option_source_filename    = NULL;
-static const char *option_xdelta1            = "xdelta1";
-
-static usize_t     option_winsize            = XD3_DEFAULT_WINSIZE;
-static usize_t     option_srcwinsz           = XD3_DEFAULT_SRCWINSZ;
-static usize_t     option_memsize            = XD3_DEFAULT_MEMSIZE;
-
-/* This controls the number of times main repeats itself, only for profiling. */
-static int option_profile_cnt = 0;
-
-/* These variables are supressed to avoid their use w/o support.  main() warns
- * appropriately. */
-#if EXTERNAL_COMPRESSION
-static int         option_decompress_inputs  = 1;
-static int         option_recompress_outputs = 1;
-#endif
-
-/* This is for comparing "printdelta" output without attention to
- * copy-instruction modes, useful for reverse engineering. */
-#if VCDIFF_TOOLS
-static int         option_print_cpymode = 1;
-#endif
-
-/* Static variables */
-IF_DEBUG(static int main_mallocs = 0;)
-
-static char*          program_name = NULL;
-static uint8_t*       appheader_used = NULL;
-static uint8_t*       main_bdata = NULL;
-
-/* The LRU: obviously this is shared by all callers. */
-static int               lru_size = 0;
-static main_blklru      *lru = NULL;  /* array of lru_size elts */
-static main_blklru_list  lru_list;
-static main_blklru_list  lru_free;
-static int               do_not_lru = 0;  /* set to avoid lru, instead discard oldest */
-
-static int lru_hits   = 0;
-static int lru_misses = 0;
-static int lru_filled = 0;
-
-/* Hacks for VCDIFF tools */
-static int allow_fake_source = 0;
-
-/* This array of compressor types is compiled even if EXTERNAL_COMPRESSION is false just so
- * the program knows the mapping of IDENT->NAME. */
-static main_extcomp extcomp_types[] =
-{
-  /* The entry for xdelta3/1 must be 0/1 because the program_names are set there. */
-  { "xdelta3",  "-cfq",  "xdelta3",    "-dcfq",  "X", "\xd6\xc3\xc4", 3, RD_NONEXTERNAL },
-  { "xdelta1",  "delta", "xdelta1",    "patch", "1", "%XD",          3, RD_EXTERNAL_V1 },
-  { "bzip2",    "-cf",   "bzip2",      "-dcf",   "B", "BZh",          3, 0 },
-  { "gzip",     "-cf",   "gzip",       "-dcf",   "G", "\037\213",     2, 0 },
-  { "compress", "-cf",   "uncompress", "-cf",    "Z", "\037\235",     2, 0 },
-};
-
-static void main_get_appheader (xd3_stream *stream, main_file *ifile,
-                                main_file *output, main_file *sfile);
-
-static int main_help (void);
-
-static int
-main_version (void)
-{
-  /* $Format: "  P(RINT \"VERSION=3.$Xdelta3Version$\\n\");" $ */
-  P(RINT "VERSION=3.0h_pre0\n");
-  return EXIT_SUCCESS;
-}
-
-static int
-main_config (void)
-{
-  main_version ();
-
-  P(RINT "EXTERNAL_COMPRESSION=%d\n", EXTERNAL_COMPRESSION);
-  P(RINT "GENERIC_ENCODE_TABLES=%d\n", GENERIC_ENCODE_TABLES);
-  P(RINT "GENERIC_ENCODE_TABLES_COMPUTE=%d\n", GENERIC_ENCODE_TABLES_COMPUTE);
-  P(RINT "REGRESSION_TEST=%d\n", REGRESSION_TEST);
-  P(RINT "SECONDARY_DJW=%d\n", SECONDARY_DJW);
-  P(RINT "SECONDARY_FGK=%d\n", SECONDARY_FGK);
-  P(RINT "VCDIFF_TOOLS=%d\n", VCDIFF_TOOLS);
-  P(RINT "XD3_ALLOCSIZE=%d\n", XD3_ALLOCSIZE);
-  P(RINT "XD3_DEBUG=%d\n", XD3_DEBUG);
-  P(RINT "XD3_DEFAULT_CKSUM_SIZE=%d\n", XD3_DEFAULT_CKSUM_ADVANCE);
-  P(RINT "XD3_DEFAULT_IOPT_SIZE=%d\n", XD3_DEFAULT_IOPT_SIZE);
-  P(RINT "XD3_DEFAULT_MEMSIZE=%d\n", XD3_DEFAULT_MEMSIZE);
-  P(RINT "XD3_DEFAULT_SPREVSZ=%d\n", XD3_DEFAULT_SPREVSZ);
-  P(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ);
-  P(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE);
-  P(RINT "XD3_ENCODER=%d\n", XD3_ENCODER);
-  P(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE);
-  P(RINT "XD3_NODECOMPRESSSIZE=%d\n", XD3_NODECOMPRESSSIZE);
-  P(RINT "XD3_POSIX=%d\n", XD3_POSIX);
-  P(RINT "XD3_USE_LARGEFILE64=%d\n", XD3_USE_LARGEFILE64);
-
-  return EXIT_SUCCESS;
-}
-
-static void*
-main_malloc1 (usize_t size)
-{
-  void* r = malloc (size);
-  if (r == NULL) { XPR(NT "malloc: %s\n", xd3_strerror (ENOMEM)); }
-  else if (option_verbose > 2) { XPR(NT "malloc: %u: %p\n", size, r); }
-  return r;
-}
-
-static void*
-main_malloc (usize_t size)
-{
-  void *r = main_malloc1 (size);
-   if (r) { IF_DEBUG (main_mallocs += 1); }
-  return r;
-}
-
-static void*
-main_alloc (void   *opaque,
-            usize_t  items,
-            usize_t  size)
-{
-  return main_malloc1 (items * size);
-}
-
-static void
-main_free1 (void *opaque, void *ptr)
-{
-  if (option_verbose > 2) { XPR(NT "free: %p\n", ptr); }
-  free (ptr);
-}
-
-static void
-main_free (void *ptr)
-{
-  if (ptr)
-    {
-      IF_DEBUG (main_mallocs -= 1);
-      main_free1 (NULL, ptr);
-    }
-}
-
-/* This ensures that (ret = errno) always indicates failure, in case errno was
- * accidentally not set.  If this prints there's a bug somewhere. */
-static int
-get_errno (void)
-{
-  if (errno == 0)
-    {
-      XPR(NT "you found a bug: expected errno != 0\n");
-      errno = XD3_INTERNAL;
-    }
-  return errno;
-}
-
-static long
-get_millisecs_now (void)
-{
-  struct timeval tv;
-
-  gettimeofday (& tv, NULL);
-
-  return (tv.tv_sec) * 1000L + (tv.tv_usec) / 1000;
-}
-
-/* Always >= 1 millisec, right? */
-static long
-get_millisecs_since (void)
-{
-  double millis;
-  struct timeval tv;
-  /* static holds the first timeval */
-  static struct timeval init;
- 
-  gettimeofday (& tv, NULL);
-
-  millis = (tv.tv_sec - init.tv_sec) * 1e6;
-  millis += (tv.tv_usec - init.tv_usec);
-  millis /= 1000;
-
-  init = tv;
-
-  return max ((long)millis, 1L);
-}
-
-static char*
-main_format_bcnt (xoff_t r, char *buf)
-{
-  static const char* fmts[] = { "B", "KB", "MB", "GB" };
-  int i;
-
-  for (i = 0; i < SIZEOF_ARRAY(fmts); i += 1)
-    {
-      if (r < 10 * 1e3 || i == -1 + SIZEOF_ARRAY(fmts))
-        {
-          sprintf (buf, "%"Q"u %s", r, fmts[i]);
-          break;
-        }
-      r /= 1000;
-    }
-  return buf;
-}
-
-static char*
-main_format_rate (xoff_t bytes, long millis, char *buf)
-{
-  xoff_t r = 1.0 * bytes / (1.0 * millis / 1000.0);
-  static char lbuf[32];
-
-  main_format_bcnt (r, lbuf);
-  sprintf (buf, "%s/sec", lbuf);
-  return buf;
-}
-
-static char*
-main_format_millis (long millis, char *buf)
-{
-  if (millis < 1000)       { sprintf (buf, "%lu ms", millis); }
-  else if (millis < 10000) { sprintf (buf, "%.1f sec", millis / 1000.0); }  
-  else                     { sprintf (buf, "%lu sec", millis / 1000L); }  
-  return buf;
-}
-
-/* A safe version of strtol for xoff_t. */
-static int
-main_strtoxoff (const char* s, xoff_t *xo, char which)
-{
-  char *e;
-  xoff_t x;
-
-  XD3_ASSERT(s && *s != 0);
-
-  {
-    /* Should check LONG_MIN, LONG_MAX, LLONG_MIN, LLONG_MAX? */
-#if SIZEOF_XOFF_T == 4
-    long xx = strtol (s, &e, 0);
-#else
-    long long xx = strtoll (s, &e, 0);
-#endif
-
-    if (xx < 0)
-      {
-        XPR(NT "-%c: negative integer: %s\n", which, s);
-        return EXIT_FAILURE;
-      }
-
-    x = xx;
-  }
-
-  if (*e != 0)
-    {
-      XPR(NT "-%c: invalid integer: %s\n", which, s);
-      return EXIT_FAILURE;
-    }
-
-  (*xo) = x;
-  return 0;
-}
-
-static int
-main_atou (const char* arg, usize_t *xo, usize_t low, char which)
-{
-  xoff_t x;
-  int ret;
-
-  if ((ret = main_strtoxoff (arg, & x, which))) { return ret; }
-
-  if (x > USIZE_T_MAX || x < low)
-    {
-      XPR(NT "-%c: minimum value: %u", which, low);
-      return EXIT_FAILURE;
-    }
-  (*xo) = x;
-  return 0;
-}
-
-/******************************************************************************************
- FILE BASICS
- ******************************************************************************************/
-
-/* With all the variation in file system-call semantics, arguments, return values and
- * error-handling for the POSIX and STDIO file APIs, the insides of these functions make
- * me sick, which is why these wrappers exist. */
-
-#define XOPEN_OPNAME (xfile->mode == XO_READ ? "read" : "write")
-#define XOPEN_STDIO  (xfile->mode == XO_READ ? "rb" : "wb")
-#define XOPEN_POSIX  (xfile->mode == XO_READ ? O_RDONLY : O_WRONLY | O_CREAT | O_TRUNC)
-#define XOPEN_MODE   (xfile->mode == XO_READ ? 0 : 0666)
-
-#define XF_ERROR(op, name, ret) XPR(NT "file %s failed: %s: %s: %s\n", (op), XOPEN_OPNAME, (name), xd3_strerror (ret))
-
-#if XD3_STDIO
-#define XFNO(f) fileno(f->file)
-#define XSTDOUT_XF(f) { (f)->file = stdout; (f)->filename = "/dev/stdout"; }
-#define XSTDERR_XF(f) { (f)->file = stderr; (f)->filename = "/dev/stderr"; }
-#define XSTDIN_XF(f)  { (f)->file = stdin;  (f)->filename = "/dev/stdin"; }
-
-#elif XD3_POSIX
-#define XFNO(f) f->file
-#define XSTDOUT_XF(f) { (f)->file = STDOUT_FILENO; (f)->filename = "/dev/stdout"; }
-#define XSTDERR_XF(f) { (f)->file = STDERR_FILENO; (f)->filename = "/dev/stderr"; }
-#define XSTDIN_XF(f)  { (f)->file = STDIN_FILENO;  (f)->filename = "/dev/stdin"; }
-#endif
-
-static void
-main_file_init (main_file *xfile)
-{
-  memset (xfile, 0, sizeof (*xfile));
-
-#if XD3_POSIX
-  xfile->file = -1;
-#endif
-}
-
-static void
-main_file_cleanup (main_file *xfile)
-{
-  if (xfile->filename_copy) {
-    main_free(xfile->filename_copy);
-  }
-}
-
-static int
-main_file_isopen (main_file *xfile)
-{
-#if XD3_STDIO
-  return xfile->file != NULL;
-
-#elif XD3_POSIX
-  return xfile->file != -1;
-#endif
-}
-
-static int
-main_file_close (main_file *xfile)
-{
-  int ret = 0;
-
-  if (! main_file_isopen (xfile))
-    {
-      return 0;
-    }
-
-#if XD3_STDIO
-  ret = fclose (xfile->file);
-  xfile->file = NULL;
-
-#elif XD3_POSIX
-  ret = close (xfile->file);
-  xfile->file = -1;
-#endif
-
-  if (ret != 0) { XF_ERROR ("close", xfile->filename, ret = get_errno ()); }
-  return ret;
-}
-
-static int
-main_file_open (main_file *xfile, const char* name, int mode)
-{
-  int ret = 0;
-
-  xfile->mode = mode;
-
-  XD3_ASSERT (! main_file_isopen (xfile));
-
-#if XD3_STDIO
-  xfile->file = fopen (name, XOPEN_STDIO);
-
-  ret = (xfile->file == NULL) ? get_errno () : 0;
-
-#elif XD3_POSIX
-  if ((ret = open (name, XOPEN_POSIX, XOPEN_MODE)) < 0)
-    {
-      ret = get_errno ();
-    }
-  else
-    {
-      xfile->file = ret;
-      ret = 0;
-    }
-#endif
-  if (ret) { XF_ERROR ("open", name, ret); }
-  else     { xfile->realname = name; xfile->nread = 0; }
-  return ret;
-}
-
-static int
-main_file_stat (main_file *xfile, xoff_t *size, int err_ifnoseek)
-{
-  int ret;
-  struct stat sbuf;
-
-  XD3_ASSERT (main_file_isopen (xfile));
-
-  if (fstat (XFNO (xfile), & sbuf) < 0)
-    {
-      ret = get_errno ();
-      if (err_ifnoseek) { XF_ERROR ("stat", xfile->filename, ret); }
-      return ret;
-    }
-
-  if (! S_ISREG (sbuf.st_mode))
-    {
-      if (err_ifnoseek) { XPR(NT "source file must be seekable: %s\n", xfile->filename); }
-      return ESPIPE;
-    }
-
-  (*size) = sbuf.st_size;
-  return 0;
-}
-
-static int
-main_file_exists (main_file *xfile)
-{
-  struct stat sbuf;
-  return stat (xfile->filename, & sbuf) == 0 && S_ISREG (sbuf.st_mode);
-}
-
-#if (XD3_POSIX || EXTERNAL_COMPRESSION)
-/* POSIX-generic code takes a function pointer to read() or write().  This calls the
- * function repeatedly until the buffer is full or EOF.  The NREAD parameter is not
- * set for write, NULL is passed.  Return is signed, < 0 indicate errors, otherwise
- * byte count. */
-typedef int (xd3_posix_func) (int fd, uint8_t *buf, usize_t size);
-
-static int
-xd3_posix_io (int fd, uint8_t *buf, usize_t size, xd3_posix_func *func, usize_t *nread)
-{
-  int ret;
-  usize_t nproc = 0;
-
-  while (nproc < size)
-    {
-      int result = (*func) (fd, buf + nproc, size - nproc);
-      
-      if (result < 0)
-        {
-          ret = get_errno ();
-          if (ret != EAGAIN && ret != EINTR)
-            {
-              return ret;
-            }
-          result = 0;
-        }
-
-      if (nread != NULL && result == 0) { break; }
-
-      nproc += result;
-    }
-  if (nread != NULL) { (*nread) = nproc; }
-  return 0;
-}
-#endif
-
-/* POSIX is unbuffered, while STDIO is buffered.  main_file_read() should always be called
- * on blocks. */
-static int
-main_file_read (main_file   *ifile,
-               uint8_t    *buf,
-               usize_t      size,
-               usize_t     *nread,
-               const char *msg)
-{
-  int ret = 0;
-
-#if XD3_STDIO
-  usize_t result;
-
-  result = fread (buf, 1, size, ifile->file);
-
-  if (result < size && ferror (ifile->file))
-    {
-      ret = get_errno ();
-    }
-  else
-    {
-      *nread = result;
-    }
-
-#elif XD3_POSIX
-  ret = xd3_posix_io (ifile->file, buf, size, (xd3_posix_func*) &read, nread);
-#endif
-
-  if (ret)
-    {
-      XPR(NT "%s: %s: %s\n", msg, ifile->filename, xd3_strerror (ret));
-    }
-  else
-    {
-      if (option_verbose > 2) { XPR(NT "main read: %s: %u\n", ifile->filename, (*nread)); }
-      ifile->nread += (*nread);
-    }
-
-  return ret;
-}
-
-static int
-main_file_write (main_file *ofile, uint8_t *buf, usize_t size, const char *msg)
-{
-  int ret = 0;
-
-#if XD3_STDIO
-  usize_t result;
-
-  result = fwrite (buf, 1, size, ofile->file);
-
-  if (result != size) { ret = get_errno (); }
-
-#elif XD3_POSIX
-  ret = xd3_posix_io (ofile->file, buf, size, (xd3_posix_func*) &write, NULL);
-#endif
-
-  if (ret)
-    {
-      XPR(NT "%s: %s: %s\n", msg, ofile->filename, xd3_strerror (ret));
-    }
-  else
-    {
-      if (option_verbose > 2) { XPR(NT "main write: %s: %u\n", ofile->filename, size); }
-      ofile->nwrite += size;
-    }
-
-  return ret;
-}
-
-static int
-main_file_seek (main_file *xfile, xoff_t pos)
-{
-  int ret = 0;
-
-#if XD3_STDIO
-  if (fseek (xfile->file, pos, SEEK_SET) != 0) { ret = get_errno (); }
-#else
-  if (lseek (xfile->file, pos, SEEK_SET) != pos) { ret = get_errno (); }
-#endif
-
-  if (ret)
-    {
-      XPR(NT "seek failed: %s: %s\n", xfile->filename, xd3_strerror (ret));
-    }
-
-  return ret;
-}
-
-/******************************************************************************************
- VCDIFF TOOLS
- ******************************************************************************************/
-
-#if VCDIFF_TOOLS
-/* This function prints a single VCDIFF window, mainly for debugging purposes. */
-static int
-main_print_window (xd3_stream* stream, FILE *vcout)
-{
-  int ret;
-  usize_t size = 0;
-
-  VC(UT "  Offset Code Type1 Size1 @Addr1 + Type2 Size2 @Addr2\n");
-
-  while (stream->inst_sect.buf < stream->inst_sect.buf_max)
-    {
-      uint   code = stream->inst_sect.buf[0];
-
-      if ((ret = xd3_decode_instruction (stream))) { return ret; }
-
-      VC(UT "  %06"Q"u %03u  %s %3u", stream->dec_winstart + size, code,
-               xd3_rtype_to_string (stream->dec_current1.type, option_print_cpymode),
-               stream->dec_current1.size);
-
-      if (stream->dec_current1.type != XD3_NOOP)
-        {
-          size += stream->dec_current1.size;
-          if (stream->dec_current1.type >= XD3_CPY)
-            {
-              VC(UT " @%-6u", stream->dec_current1.addr);
-            }
-          else
-            {
-              VC(UT "        ");
-            }
-        }
-
-      if (stream->dec_current2.type != XD3_NOOP)
-        {
-          size += stream->dec_current2.size;
-          VC(UT "  %s %3u",
-                   xd3_rtype_to_string (stream->dec_current2.type, option_print_cpymode),
-                   stream->dec_current2.size);
-
-          if (stream->dec_current2.type >= XD3_CPY)
-            {
-              VC(UT " @%-6u", stream->dec_current2.addr);
-            }
-        }
-
-      VC(UT "\n");
-    }
-
-  if (stream->dec_tgtlen != size && (stream->flags & XD3_SKIP_WINDOW) == 0)
-    {
-      XPR(NT "target window size inconsistency");
-      return XD3_INTERNAL;
-    }
-
-  if (stream->dec_position != stream->dec_maxpos)
-    {
-      XPR(NT "target window position inconsistency");
-      return XD3_INTERNAL;
-    }
-
-  if (stream->addr_sect.buf != stream->addr_sect.buf_max)
-    {
-      XPR(NT "address section inconsistency");
-      return XD3_INTERNAL;
-    }
-
-  IF_DEBUG (VC(UT "SIZE=%u  TGTLEN=%u\n", size, stream->dec_tgtlen));
-
-  return 0;
-}
-
-static void
-main_print_vcdiff_file (main_file *file, const char *type, FILE *vcout)
-{
-  if (file->filename)   { VC(UT "XDELTA filename (%s):     %s\n", type, file->filename); }
-  if (file->compressor) { VC(UT "XDELTA ext comp (%s):     %s\n", type, file->compressor->recomp_cmdname); }  
-}
-
-/* This function prints a VCDIFF input, mainly for debugging purposes. */
-static int
-main_print_func (xd3_stream* stream, main_file *xfile)
-{
-  int ret;
-  FILE *vcout;
-#if XD3_POSIX
-  if (! (vcout = fdopen (dup(xfile->file), "w")))
-    {
-      ret = get_errno ();
-      XPR(NT "fdopen: %s: %s\n", xfile->filename, xd3_strerror (ret));
-      return ret;
-    }
-#elif XD3_STDIO
-  vcout = xfile->file;
-#endif
-  XD3_ASSERT (vcout);
-  if (stream->dec_winstart == 0)
-    {
-      VC(UT "VCDIFF version:               0\n");
-
-      VC(UT "VCDIFF header size:           %d\n", stream->dec_hdrsize);
-      VC(UT "VCDIFF header indicator:      ");
-      if ((stream->dec_hdr_ind & VCD_SECONDARY) != 0) VC(UT "VCD_SECONDARY ");
-      if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) VC(UT "VCD_CODETABLE ");
-      if ((stream->dec_hdr_ind & VCD_APPHEADER) != 0) VC(UT "VCD_APPHEADER ");
-      if (stream->dec_hdr_ind == 0) VC(UT "none");
-      VC(UT "\n");
-
-      IF_SEC(VC(UT "VCDIFF secondary compressor:  %s\n", stream->sec_type ? stream->sec_type->name : "none"));
-      IF_NSEC(VC(UT "VCDIFF secondary compressor: unsupported\n"));
-
-      if (stream->dec_hdr_ind & VCD_APPHEADER)
-        {
-          uint8_t *apphead;
-          usize_t appheadsz;
-          ret = xd3_get_appheader (stream, & apphead, & appheadsz);
-
-          if (ret == 0 && appheadsz > 0)
-            {
-              int sq = option_quiet;
-              main_file i, o, s;
-              XD3_ASSERT (apphead != NULL);
-              VC(UT "VCDIFF application header:    ");
-              fwrite (apphead, 1, appheadsz, vcout);
-              VC(UT "\n");
-
-              main_file_init (& i);
-              main_file_init (& o);
-              main_file_init (& s);
-              option_quiet = 1;
-              main_get_appheader (stream, &i, & o, & s);
-              option_quiet = sq;
-              main_print_vcdiff_file (& o, "output", vcout);
-              main_print_vcdiff_file (& s, "source", vcout);
-              main_file_cleanup (& i);
-              main_file_cleanup (& o);
-              main_file_cleanup (& s);
-            }
-        }
-    }
-  else
-    {
-      VC(UT "\n");
-    }
-
-  VC(UT "VCDIFF window number:         %"Q"u\n", stream->current_window);
-  VC(UT "VCDIFF window indicator:      ");
-  if ((stream->dec_win_ind & VCD_SOURCE) != 0) VC(UT "VCD_SOURCE ");
-  if ((stream->dec_win_ind & VCD_TARGET) != 0) VC(UT "VCD_TARGET ");
-  if ((stream->dec_win_ind & VCD_ADLER32) != 0) VC(UT "VCD_ADLER32 ");
-  if (stream->dec_win_ind == 0) VC(UT "none");
-  VC(UT "\n");
-
-  if ((stream->dec_win_ind & VCD_ADLER32) != 0)
-    {
-      VC(UT "VCDIFF adler32 checksum:      %08X\n", stream->dec_adler32);
-    }
-
-  if (stream->dec_del_ind != 0)
-    {
-      VC(UT "VCDIFF delta indicator:       ");
-      if ((stream->dec_del_ind & VCD_DATACOMP) != 0) VC(UT "VCD_DATACOMP ");
-      if ((stream->dec_del_ind & VCD_INSTCOMP) != 0) VC(UT "VCD_INSTCOMP ");
-      if ((stream->dec_del_ind & VCD_ADDRCOMP) != 0) VC(UT "VCD_ADDRCOMP ");
-      if (stream->dec_del_ind == 0) VC(UT "none");
-      VC(UT "\n");
-    }
-
-  if (stream->dec_winstart != 0)
-    {
-      VC(UT "VCDIFF window at offset:      %"Q"u\n", stream->dec_winstart);
-    }
-
-  if (SRCORTGT (stream->dec_win_ind))
-    {
-      VC(UT "VCDIFF copy window length:    %u\n", stream->dec_cpylen);
-      VC(UT "VCDIFF copy window offset:    %"Q"u\n", stream->dec_cpyoff);
-    }
-
-  VC(UT "VCDIFF delta encoding length: %u\n", stream->dec_enclen);
-  VC(UT "VCDIFF target window length:  %u\n", stream->dec_tgtlen);
-
-  VC(UT "VCDIFF data section length:   %u\n", stream->data_sect.size);
-  VC(UT "VCDIFF inst section length:   %u\n", stream->inst_sect.size);
-  VC(UT "VCDIFF addr section length:   %u\n", stream->addr_sect.size);
-
-  ret = 0; 
-  if ((stream->flags & XD3_JUST_HDR) != 0)
-    {
-      /* Print a header -- finished! */
-      ret = PRINTHDR_SPECIAL;
-    }
-  else if ((stream->flags & XD3_SKIP_WINDOW) == 0)
-    {
-      ret = main_print_window (stream, vcout);
-    }
-
-  fclose (vcout);
-  return ret;
-}
-#endif /* VCDIFF_TOOLS */
-
-/******************************************************************************************
- Input decompression, output recompression
- ******************************************************************************************/
-
-#if EXTERNAL_COMPRESSION
-/* This is tricky POSIX-specific code with lots of fork(), pipe(), dup(), waitpid(), and
- * exec() business.  Most of this code originated in PRCS1, which did automatic
- * package-file decompression.  It works with both XD3_POSIX and XD3_STDIO file
- * disciplines.
- *
- * To automatically detect compressed inputs requires a child process to reconstruct the
- * input stream, which was advanced in order to detect compression, because it may not be
- * seekable.  In other words, the main program reads part of the input stream, and if it
- * detects a compressed input it then forks a pipe copier process, which copies the
- * first-read block out of the main-program's memory, then streams the remaining
- * compressed input into the input-decompression pipe.
- */
-
-#include <unistd.h>
-#include <sys/stat.h>
-#include <sys/wait.h>
-
-/* Remember which pipe FD is which. */
-#define PIPE_READ_FD  0
-#define PIPE_WRITE_FD 1
-
-static pid_t ext_subprocs[2];
-static char* ext_tmpfile = NULL;
-
-/* Like write(), but makes repeated calls to empty the buffer. */
-static int
-main_pipe_write (int outfd, const uint8_t *exist_buf, usize_t remain)
-{
-  int ret;
-
-  if ((ret = xd3_posix_io (outfd, (uint8_t*) exist_buf, remain, (xd3_posix_func*) &write, NULL)))
-    {
-      XPR(NT "pipe write failed: %s", xd3_strerror (ret));
-      return ret;
-    }
-
-  return 0;
-}
-
-/* A simple error-reporting waitpid interface. */
-static int
-main_waitpid_check(pid_t pid)
-{
-  int status;
-  int ret = 0;
-
-  if (waitpid (pid, & status, 0) < 0)
-    {
-      ret = get_errno ();
-      XPR(NT "compression subprocess: wait: %s\n", xd3_strerror (ret));
-    }
-  else if (! WIFEXITED (status))
-    {
-      ret = ECHILD;
-      XPR(NT "compression subprocess: signal %d\n",
-         WIFSIGNALED (status) ? WTERMSIG (status) : WSTOPSIG (status));
-    }
-  else if (WEXITSTATUS (status) != 0)
-    {
-      ret = ECHILD;
-      XPR(NT "compression subprocess: exit %d\n", WEXITSTATUS (status));
-    }
-
-  return ret;
-}
-
-/* Wait for any existing child processes to check for abnormal exit. */
-static int
-main_external_compression_finish (void)
-{
-  int i;
-  int ret;
-
-  for (i = 0; i < 2; i += 1)
-    {
-      if (! ext_subprocs[i]) { continue; }
-
-      if ((ret = main_waitpid_check (ext_subprocs[i])))
-        {
-          return ret;
-        }
-    }
-
-  return 0;
-}
-
-/* This runs as a forked process of main_input_decompress_setup() to copy input to the
- * decompression process.  First, the available input is copied out of the existing
- * buffer, then the buffer is reused to continue reading from the compressed input
- * file. */
-static int
-main_pipe_copier (uint8_t    *pipe_buf,
-                  usize_t      pipe_bufsize,
-                  usize_t      nread,
-                  main_file   *ifile,
-                  int         outfd)
-{
-  int ret;
-
-  for (;;)
-    {
-      if (nread > 0 && (ret = main_pipe_write (outfd, pipe_buf, nread)))
-        {
-          return ret;
-        }
-
-      if (nread < pipe_bufsize)
-        {
-          break;
-        }
-
-      if ((ret = main_file_read (ifile, pipe_buf, pipe_bufsize, & nread, "pipe read failed")) < 0)
-        {
-          return ret;
-        }
-    }
-
-  return 0;
-}
-
-/* This function is called after we have read some amount of data from the input file and
- * detected a compressed input.  Here we start a decompression subprocess by forking
- * twice.  The first process runs the decompression command, the second process copies
- * data to the input of the first. */
-static int
-main_input_decompress_setup (const main_extcomp     *decomp,
-                             main_file              *ifile,
-                             uint8_t               *input_buf,
-                             usize_t                 input_bufsize,
-                             uint8_t               *pipe_buf,
-                             usize_t                 pipe_bufsize,
-                             usize_t                 pipe_avail,
-                             usize_t                *nread)
-{
-  int outpipefd[2], inpipefd[2];  /* The two pipes: input and output file descriptors. */
-  int input_fd = -1;              /* The resulting input_fd (output of decompression). */
-  pid_t decomp_id, copier_id;     /* The two subprocs. */
-  int ret;
-
-  outpipefd[0] = outpipefd[1] = -1;
-  inpipefd[0]  = inpipefd[1]  = -1;
-
-  if (pipe (outpipefd) || pipe (inpipefd))
-    {
-      XPR(NT "pipe failed: %s\n", xd3_strerror (ret = get_errno ()));
-      goto pipe_cleanup;
-    }
-
-  if ((decomp_id = fork ()) < 0)
-    {
-      XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ()));
-      goto pipe_cleanup;
-    }
-
-  /* The first child runs the decompression process: */
-  if (decomp_id == 0)
-    {
-      /* Setup pipes: write to the outpipe, read from the inpipe. */
-      if (dup2 (outpipefd[PIPE_WRITE_FD], STDOUT_FILENO) < 0 ||
-          dup2 (inpipefd[PIPE_READ_FD], STDIN_FILENO) < 0 ||
-          close (outpipefd[PIPE_READ_FD]) ||
-          close (outpipefd[PIPE_WRITE_FD]) ||
-          close (inpipefd[PIPE_READ_FD]) ||
-          close (inpipefd[PIPE_WRITE_FD]) ||
-          execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, decomp->decomp_options, NULL))
-        {
-          XPR(NT "child process %s failed to execute: %s\n", decomp->decomp_cmdname, xd3_strerror (get_errno ()));
-        }
-
-      _exit (127);
-    }
-
-  ext_subprocs[0] = decomp_id;
-
-  if ((copier_id = fork ()) < 0)
-    {
-      XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ()));
-      goto pipe_cleanup;
-    }
-
-  /* The second child runs the copier process: */
-  if (copier_id == 0)
-    {
-      int exitval = 0;
-
-      if (close (inpipefd[PIPE_READ_FD]) ||
-          main_pipe_copier (pipe_buf, pipe_bufsize, pipe_avail, ifile, inpipefd[PIPE_WRITE_FD]) ||
-          close (inpipefd[PIPE_WRITE_FD]))
-        {
-          XPR(NT "child copier process failed: %s\n", xd3_strerror (get_errno ()));
-          exitval = 1;
-        }
-
-      _exit (exitval);
-    }
-
-  ext_subprocs[1] = copier_id;
-
-  /* The parent closes both pipes after duplicating the output of compression. */
-  input_fd = dup (outpipefd[PIPE_READ_FD]);
-
-  if (input_fd < 0 ||
-      main_file_close (ifile) ||
-      close (outpipefd[PIPE_READ_FD]) ||
-      close (outpipefd[PIPE_WRITE_FD]) ||
-      close (inpipefd[PIPE_READ_FD]) ||
-      close (inpipefd[PIPE_WRITE_FD]))
-    {
-      XPR(NT "dup/close failed: %s\n", xd3_strerror (ret = get_errno ()));
-      goto pipe_cleanup;
-    }
-
-#if XD3_STDIO
-  /* Note: fdopen() acquires the fd, closes it when finished. */
-  if ((ifile->file = fdopen (input_fd, "r")) == NULL)
-    {
-      XPR(NT "fdopen failed: %s\n", xd3_strerror (ret = get_errno ()));
-      goto pipe_cleanup;
-    }
-
-#elif XD3_POSIX
-  ifile->file = input_fd;
-#endif
-
-  ifile->compressor = decomp;
-
-  /* Now the input file is decompressed. */
-  return main_file_read (ifile, input_buf, input_bufsize, nread, "input decompression failed");
-
- pipe_cleanup:
-  close (input_fd);
-  close (outpipefd[PIPE_READ_FD]);
-  close (outpipefd[PIPE_WRITE_FD]);
-  close (inpipefd[PIPE_READ_FD]);
-  close (inpipefd[PIPE_WRITE_FD]);
-  return ret;
-}
-
-
-/* This routine is called when the first buffer of input data is read by the main program
- * (unless input decompression is disabled by command-line option).  If it recognizes the
- * magic number of a known input type it invokes decompression.
- *
- * Skips decompression if the decompression type or the file type is RD_NONEXTERNAL.
- *
- * Behaves exactly like main_file_read, otherwise.
- *
- * This function uses a separate buffer to read the first small block of input.  If a
- * compressed input is detected, the separate buffer is passed to the pipe copier.  This
- * avoids using the same size buffer in both cases. */
-static int
-main_decompress_input_check (main_file   *ifile,
-                            uint8_t    *input_buf,
-                            usize_t      input_size,
-                            usize_t     *nread)
-{
-  int i;
-  int ret;
-  uint8_t check_buf[XD3_ALLOCSIZE];
-  usize_t  check_nread;
-
-  if ((ret = main_file_read (ifile, check_buf, min (input_size, XD3_ALLOCSIZE), & check_nread, "input read failed")))
-    {
-      return ret;
-    }
-
-  for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)
-    {
-      const main_extcomp *decomp = & extcomp_types[i];
-
-      if ((check_nread > decomp->magic_size) &&
-          /* The following expr skips decompression if we are trying to read a VCDIFF
-           * input and that is the magic number. */
-          !((decomp->flags & RD_NONEXTERNAL) && (ifile->flags & RD_NONEXTERNAL)) &&
-          memcmp (check_buf, decomp->magic, decomp->magic_size) == 0)
-        {
-          if (! option_quiet)
-            {
-              XPR(NT "%s | %s %s\n",
-                 ifile->filename,
-                 decomp->decomp_cmdname,
-                 decomp->decomp_options);
-            }
-
-          return main_input_decompress_setup (decomp, ifile,
-                                              input_buf, input_size,
-                                              check_buf, XD3_ALLOCSIZE,
-                                              check_nread, nread);
-        }
-    }
-
-  /* Now read the rest of the input block. */
-  (*nread) = 0;
-
-  if (check_nread == XD3_ALLOCSIZE)
-    {
-      ret = main_file_read (ifile, input_buf + XD3_ALLOCSIZE,
-                            input_size - XD3_ALLOCSIZE, nread,
-                            "input read failed");
-    }
-
-  memcpy (input_buf, check_buf, check_nread);
-
-  (*nread) += check_nread;
-
-  return 0;
-}
-
-/* This is called when the source file needs to be decompressed.  We fork/exec a
- * decompression command with the proper input and output to a temporary file. */
-static int
-main_decompress_source (main_file *sfile, xd3_source *source)
-{
-  const main_extcomp *decomp = sfile->compressor;
-  pid_t decomp_id;  /* One subproc. */
-  int   input_fd  = -1;
-  int   output_fd = -1;
-  int   ret;
-  char *tmpname = NULL;
-  char *tmpdir  = getenv ("TMPDIR");
-  static const char tmpl[] = "/xd3src.XXXXXX";
-
-  /* Make a template for mkstmp() */
-  if (tmpdir == NULL) { tmpdir = "/tmp"; }
-  if ((tmpname = main_malloc (strlen (tmpdir) + sizeof (tmpl) + 1)) == NULL) { return ENOMEM; }
-  sprintf (tmpname, "%s%s", tmpdir, tmpl);
-
-  XD3_ASSERT (ext_tmpfile == NULL);
-  ext_tmpfile = tmpname;
-
-  /* Open the output FD. */
-  if ((output_fd = mkstemp (tmpname)) < 0)
-    {
-      XPR(NT "mkstemp failed: %s: %s", tmpname, xd3_strerror (ret = get_errno ()));
-      goto cleanup;
-    }
-
-  /* Copy the input FD, reset file position. */
-  XD3_ASSERT (main_file_isopen (sfile));
-#if XD3_STDIO
-  if ((input_fd = dup (fileno (sfile->file))) < 0)
-    {
-      XPR(NT "dup failed: %s", xd3_strerror (ret = get_errno ()));
-      goto cleanup;
-    }
-  main_file_close (sfile);
-  sfile->file = NULL;
-#elif XD3_POSIX
-  input_fd = sfile->file;
-  sfile->file = -1;
-#endif
-
-  if ((ret = lseek (input_fd, SEEK_SET, 0)) != 0)
-    {
-      XPR(NT "lseek failed: : %s", xd3_strerror (ret = get_errno ()));
-      goto cleanup;
-    }
-
-  if ((decomp_id = fork ()) < 0)
-    {
-      XPR(NT "fork failed: %s", xd3_strerror (ret = get_errno ()));
-      goto cleanup;
-    }
-
-  /* The child runs the decompression process: */
-  if (decomp_id == 0)
-    {
-      /* Setup pipes: write to the output file, read from the pipe. */
-      if (dup2 (input_fd, STDIN_FILENO) < 0 ||
-          dup2 (output_fd, STDOUT_FILENO) < 0 ||
-          execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, decomp->decomp_options, NULL))
-        {
-          XPR(NT "child process %s failed to execute: %s\n",
-                   decomp->decomp_cmdname, xd3_strerror (get_errno ()));
-        }
-
-      _exit (127);
-    }
-
-  close (input_fd);
-  close (output_fd);
-  input_fd  = -1;
-  output_fd = -1;
-
-  /* Then wait for completion. */
-  if ((ret = main_waitpid_check (decomp_id)))
-    {
-      goto cleanup;
-    }
-
-  /* Open/stat the decompressed source file. */
-  if ((ret = main_file_open (sfile, tmpname, XO_READ))) { goto cleanup; }
-  if ((ret = main_file_stat (sfile, & source->size, 1))) { goto cleanup; }
-  return 0;
-
- cleanup:
-  close (input_fd);
-  close (output_fd);
-  if (tmpname) { free (tmpname); }
-  ext_tmpfile = NULL;
-  return ret;
-}
-
-/* Initiate re-compression of the output stream.  This is easier than input decompression
- * because we know beforehand that the stream will be compressed, whereas the input has
- * already been read when we decide it should be decompressed.  Thus, it only requires one
- * subprocess and one pipe. */
-static int
-main_recompress_output (main_file *ofile)
-{
-  pid_t recomp_id;  /* One subproc. */
-  int   pipefd[2];  /* One pipe. */
-  int   output_fd = -1;
-  int   ret;
-  const main_extcomp *recomp = ofile->compressor;
-
-  pipefd[0] = pipefd[1] = -1;
-
-  if (pipe (pipefd))
-    {
-      XPR(NT "pipe failed: %s\n", xd3_strerror (ret = get_errno ()));
-      goto pipe_cleanup;
-    }
-
-  if ((recomp_id = fork ()) < 0)
-    {
-      XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ()));
-      goto pipe_cleanup;
-    }
-
-  /* The child runs the recompression process: */
-  if (recomp_id == 0)
-    {
-      /* Setup pipes: write to the output file, read from the pipe. */
-      if (dup2 (XFNO (ofile), STDOUT_FILENO) < 0 ||
-          dup2 (pipefd[PIPE_READ_FD], STDIN_FILENO) < 0 ||
-          close (pipefd[PIPE_READ_FD]) ||
-          close (pipefd[PIPE_WRITE_FD]) ||
-          execlp (recomp->recomp_cmdname, recomp->recomp_cmdname, recomp->recomp_options, NULL))
-        {
-          XPR(NT "child process %s failed to execute: %s\n", recomp->recomp_cmdname, xd3_strerror (get_errno ()));
-        }
-
-      _exit (127);
-    }
-
-  ext_subprocs[0] = recomp_id;
-
-  /* The parent closes both pipes after duplicating the output-fd for writing to the
-   * compression pipe. */
-  output_fd = dup (pipefd[PIPE_WRITE_FD]);
-
-  if (output_fd < 0 ||
-      main_file_close (ofile) ||
-      close (pipefd[PIPE_READ_FD]) ||
-      close (pipefd[PIPE_WRITE_FD]))
-    {
-      XPR(NT "close failed: %s\n", xd3_strerror (ret = get_errno ()));
-      goto pipe_cleanup;
-    }
-
-#if XD3_STDIO
-  /* Note: fdopen() acquires the fd, closes it when finished. */
-  if ((ofile->file = fdopen (output_fd, "w")) == NULL)
-    {
-      XPR(NT "fdopen failed: %s\n", xd3_strerror (ret = get_errno ()));
-      goto pipe_cleanup;
-    }
-
-#elif XD3_POSIX
-  ofile->file = output_fd;
-#endif
-
-  /* Now the output file will be compressed. */
-  return 0;
-
- pipe_cleanup:
-  close (output_fd);
-  close (pipefd[PIPE_READ_FD]);
-  close (pipefd[PIPE_WRITE_FD]);
-  return ret;
-}
-#endif /* EXTERNAL_COMPRESSION */
-
-/* Identify the compressor that was used based on its ident string, which is passed in the
- * application header. */
-static const main_extcomp*
-main_ident_compressor (const char *ident)
-{
-  int i;
-
-  for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)
-    {
-      if (strcmp (extcomp_types[i].ident, ident) == 0)
-        {
-          return & extcomp_types[i];
-        }
-    }
-
-  return NULL;
-}
-
-/* Return the main_extcomp record to use for this identifier, if possible. */
-static const main_extcomp*
-main_get_compressor (const char *ident)
-{
-  const main_extcomp *ext = main_ident_compressor (ident);
-
-  if (ext == NULL)
-    {
-      if (! option_quiet)
-        {
-          XPR(NT "warning: cannot recompress output: "
-                   "unrecognized external compression ID: %s\n", ident);
-        }
-      return NULL;
-    }
-  else if (! EXTERNAL_COMPRESSION)
-    {
-      if (! option_quiet)
-        {
-          XPR(NT "warning: external support not compiled: "
-                   "original input was compressed: %s\n", ext->recomp_cmdname);
-        }
-      return NULL;
-    }
-  else
-    {
-      return ext;
-    }
-}
-
-/******************************************************************************************
- APPLICATION HEADER
- ******************************************************************************************/
-
-#if XD3_ENCODER
-static const char*
-main_apphead_string (const char* x)
-{
-  const char *y;
-
-  if (x == NULL) { return ""; }
-
-  if (strcmp (x, "/dev/stdin") == 0 ||
-      strcmp (x, "/dev/stdout") == 0 ||
-      strcmp (x, "/dev/stderr") == 0) { return "-"; }
-
-  // TODO: this is not portable
-  return (y = strrchr (x, '/')) == NULL ? x : y + 1;
-}
-
-static int
-main_set_appheader (xd3_stream *stream, main_file *input, main_file *sfile)
-{
-  /* The user may disable the application header.  Once the appheader is set, this
-   * disables setting it again. */
-  if (appheader_used || ! option_use_appheader) { return 0; }
-
-  /* The user may specify the application header, otherwise format the default header. */
-  if (option_appheader)
-    {
-      appheader_used = option_appheader;
-    }
-  else
-    {
-      const char *iname;
-      const char *icomp;
-      const char *sname;
-      const char *scomp;
-      int len;
-
-      iname = main_apphead_string (input->filename);
-      icomp = (input->compressor == NULL) ? "" : input->compressor->ident;
-      len = strlen (iname) + strlen (icomp) + 2;
-
-      if (sfile->filename != NULL)
-        {
-          sname = main_apphead_string (sfile->filename);
-          scomp = (sfile->compressor == NULL) ? "" : sfile->compressor->ident;
-          len += strlen (sname) + strlen (scomp) + 2;
-        }
-      else
-        {
-          sname = scomp = "";
-        }
-
-      if ((appheader_used = main_malloc (len)) == NULL)
-        {
-          return ENOMEM;
-        }
-
-      if (sfile->filename == NULL)
-        {
-          sprintf ((char*)appheader_used, "%s/%s", iname, icomp);
-        }
-      else
-        {
-          sprintf ((char*)appheader_used, "%s/%s/%s/%s", iname, icomp, sname, scomp);
-        }
-    }
-
-  xd3_set_appheader (stream, appheader_used, strlen ((char*)appheader_used));
-
-  return 0;
-}
-#endif
-
-static void
-main_get_appheader_params (main_file *file, char **parsed, int output, const char *type,
-                           main_file *other)
-{
-  /* Set the filename if it was not specified.  If output, option_stdout (-c) overrides. */
-  if (file->filename == NULL && ! (output && option_stdout) && strcmp (parsed[0], "-") != 0)
-    {
-      file->filename = parsed[0];
-
-      if (other->filename != NULL) {
-        /* Take directory from the other file, if it has one. */
-        char *last_slash = strrchr(other->filename, '/');
-
-        if (last_slash != NULL) {
-          int dlen = last_slash - other->filename;
-          file->filename_copy = main_malloc(dlen + 2 + strlen(file->filename));
-
-          strncpy(file->filename_copy, other->filename, dlen);
-          file->filename_copy[dlen] = '/';
-          strcpy(file->filename_copy + dlen + 1, parsed[0]);
-
-          file->filename = file->filename_copy;
-        }
-      }
-
-      if (! option_quiet)
-        {
-          XPR(NT "using default %s filename: %s\n", type, file->filename);
-        }
-    }
-
-  /* Set the compressor, initiate de/recompression later. */
-  if (file->compressor == NULL && *parsed[1] != 0)
-    {
-      file->compressor = main_get_compressor (parsed[1]);
-    }
-}
-
-static void
-main_get_appheader (xd3_stream *stream, main_file *ifile, main_file *output, main_file *sfile)
-{
-  uint8_t *apphead;
-  usize_t appheadsz;
-  int ret;
-
-  /* The user may disable the application header.  Once the appheader is set, this
-   * disables setting it again. */
-  if (! option_use_appheader) { return; }
-
-  ret = xd3_get_appheader (stream, & apphead, & appheadsz);
-
-  /* Ignore failure, it only means we haven't received a header yet. */
-  if (ret != 0) { return; }
-
-  if (appheadsz > 0)
-    {
-      char *start = (char*)apphead;
-      char *slash;
-      int   place = 0;
-      char *parsed[4];
-
-      memset (parsed, 0, sizeof (parsed));
-
-      while ((slash = strchr (start, '/')) != NULL)
-        {
-          *slash = 0;
-          parsed[place++] = start;
-          start = slash + 1;
-        }
-
-      parsed[place++] = start;
-
-      /* First take the output parameters. */
-      if (place == 2 || place == 4)
-        {
-          main_get_appheader_params (output, parsed, 1, "output", ifile);
-        }
-
-      /* Then take the source parameters. */
-      if (place == 4)
-        {
-          main_get_appheader_params (sfile, parsed+2, 0, "source", ifile);
-        }
-    }
-
-  option_use_appheader = 0;
-  return;
-}
-
-/******************************************************************************************
- Main I/O routines
- ******************************************************************************************/
-
-/* This function acts like the above except it may also try to recognize a compressed
- * input when the first buffer of data is read.  The EXTERNAL_COMPRESSION code is called
- * to search for magic numbers. */
-static int
-main_read_primary_input (main_file   *ifile,
-                         uint8_t    *buf,
-                         usize_t      size,
-                         usize_t     *nread)
-{
-#if EXTERNAL_COMPRESSION
-  if (option_decompress_inputs && ifile->flags & RD_FIRST)
-    {
-      ifile->flags &= ~RD_FIRST;
-
-      return main_decompress_input_check (ifile, buf, size, nread);
-    }
-#endif
-
-  return main_file_read (ifile, buf, size, nread, "input read failed");
-}
-
-/* This function simply writes the stream output buffer, if there is any.  This is used
- * for both encode and decode commands.  (The VCDIFF tools use main_print_func()). */
-static int
-main_write_output (xd3_stream* stream, main_file *ofile)
-{
-  int ret;
-
-  if (stream->avail_out > 0 && (ret = main_file_write (ofile, stream->next_out, stream->avail_out, "write failed")))
-    {
-      return ret;
-    }
-
-  return 0;
-}
-
-/* Open the main output file, sets a default file name, initiate recompression.  This
- * function is expected to fprint any error messages. */
-static int
-main_open_output (xd3_stream *stream, main_file *ofile)
-{
-  int ret;
-
-  if (ofile->filename == NULL)
-    {
-      XSTDOUT_XF (ofile);
-
-      if (option_verbose > 1) { XPR(NT "using standard output: %s\n", ofile->filename); }
-    }
-  else
-    {
-      /* Stat the file to check for overwrite. */
-      if (option_force == 0 && main_file_exists (ofile))
-        {
-          XPR(NT "to overwrite output file specify -f: %s\n", ofile->filename);
-          return EEXIST;
-        }
-
-      if ((ret = main_file_open (ofile, ofile->filename, XO_WRITE)))
-        {
-          return ret;
-        }
-
-      if (option_verbose > 1) { XPR(NT "open output: %s\n", ofile->filename); }
-    }
-
-#if EXTERNAL_COMPRESSION
-  /* Do output recompression. */
-  if (ofile->compressor != NULL && option_recompress_outputs == 1)
-    {
-      if (! option_quiet)
-        {
-          XPR(NT "%s %s | %s\n",
-             ofile->compressor->recomp_cmdname,
-             ofile->compressor->recomp_options,
-             ofile->filename);
-        }
-
-      if ((ret = main_recompress_output (ofile)))
-        {
-          return ret;
-        }
-    }
-#endif
-
-  return 0;
-}
-
-/* This is called at different times for encoding and decoding.  The encoder calls it
- * immediately, the decoder delays until the application header is received. */
-static int
-main_set_source (xd3_stream *stream, int cmd, main_file *sfile, xd3_source *source)
-{
-  int ret, i;
-
-  /* Open it, check for seekability, set required xd3_source fields. */
-  if (allow_fake_source)
-    {
-      sfile->mode = XO_READ;
-      sfile->realname = sfile->filename;
-      sfile->nread = 0;
-      source->size = UINT64_MAX;
-    }
-  else if ((ret = main_file_open (sfile, sfile->filename, XO_READ)) ||
-           (ret = main_file_stat (sfile, & source->size, 1)))
-    {
-      return ret;
-    }
-
-  source->name     = sfile->filename;
-  source->ioh      = sfile;
-  source->curblkno = (xoff_t) -1;
-  source->curblk   = NULL;
-
-  /* Source block LRU init. */
-  main_blklru_list_init (& lru_list);
-  main_blklru_list_init (& lru_free);
-
-  option_srcwinsz = min(source->size, (xoff_t) option_srcwinsz);
-
-  if (option_verbose > 1) { XPR(NT "source window size: %u\n", option_srcwinsz); }
-  if (option_verbose > 1) { XPR(NT "source block size: %u\n", source->blksize); }
-  if (option_verbose > 1) { XPR(NT "memory size: %u\n", option_memsize); }
-
-  lru_size = (option_srcwinsz / source->blksize);
-  lru_size = max(1, lru_size);
-
-  if ((lru = main_malloc (sizeof (main_blklru) * lru_size)) == NULL)
-    {
-      return ENOMEM;
-    }
-
-  for (i = 0; i < lru_size; i += 1)
-    {
-      lru[i].blkno = (xoff_t) -1;
-
-      if ((lru[i].blk = main_malloc (source->blksize)) == NULL)
-        {
-          return ENOMEM;
-        }
-
-      main_blklru_list_push_back (& lru_free, & lru[i]);
-    }
-
-#if EXTERNAL_COMPRESSION
-  if (option_decompress_inputs)
-    {
-      if (IS_ENCODE (cmd))
-        {
-          usize_t nread;
-
-          source->curblk = lru[0].blk;
-
-          /* If encoding, read the first block now to check for decompression. */
-          if ((ret = main_file_read (sfile, (uint8_t*) source->curblk, source->blksize, & nread, "source read failed")))
-            {
-              return ret;
-            }
-
-          /* Check known magic numbers. */
-          for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)
-            {
-              const main_extcomp *decomp = & extcomp_types[i];
-
-              if ((nread > decomp->magic_size) && memcmp (source->curblk, decomp->magic, decomp->magic_size) == 0)
-                {
-                  sfile->compressor = decomp;
-                  break;
-                }
-            }
-
-          /* If no decompression, the current buffer is now a valid source->curblock. */
-          if (sfile->compressor == NULL)
-            {
-              main_blklru_list_remove (& lru[0]);
-              main_blklru_list_push_back (& lru_list, & lru[0]);
-
-              lru[0].blkno     = 0;
-              source->curblkno = 0;
-              source->onblk    = nread;
-
-              if (option_verbose > 2)
-                {
-                  XPR(NT "source block 0 read (not compressed)\n");
-                }
-            }
-        }
-
-      /* In either the encoder or decoder, start decompression. */
-      if (sfile->compressor)
-        {
-          xoff_t osize = source->size;
-
-          if (osize > XD3_NODECOMPRESSSIZE)
-            {
-              XPR(NT "source file too large for external decompression: %s: %"Q"u\n",
-                       sfile->filename, osize);
-              return XD3_INTERNAL;
-            }
-
-          if ((ret = main_decompress_source (sfile, source)))
-            {
-              return ret;
-            }
-
-          if (! option_quiet)
-            {
-              char s1[32], s2[32];
-              XPR(NT "%s | %s %s => %s %.1f%% [ %s , %s ]\n",
-                 sfile->filename,
-                 sfile->compressor->decomp_cmdname,
-                 sfile->compressor->decomp_options,
-                 sfile->realname,
-                 100.0 * source->size / osize,
-                 main_format_bcnt (osize, s1),
-                 main_format_bcnt (source->size, s2));
-            }
-        }
-    }
-#endif
-
-  if (option_verbose > 1) { XPR(NT "source file: %s: %"Q"u bytes\n", sfile->realname, source->size); }
-
-  if ((ret = xd3_set_source (stream, source)))
-    {
-      XPR(NT XD3_LIB_ERRMSG (stream, ret));
-      return EXIT_FAILURE;
-    }
-
-  return 0;
-}
-
-/******************************************************************************************
- Source routines
- ******************************************************************************************/
-
-/* This is the callback for reading a block of source.  This function is blocking and it
- * implements a small LRU.
- *
- * Note that it is possible for main_input() to handle getblk requests in a non-blocking
- * manner.  If the callback is NULL then the caller of xd3_*_input() must handle the
- * XD3_GETSRCBLK return value and fill the source in the same way.  See xd3_getblk for
- * details.  To see an example of non-blocking getblk, see xdelta-test.h. */
-static int
-main_getblk_func (xd3_stream *stream,
-                  xd3_source *source,
-                  xoff_t      blkno)
-{
-  xoff_t      pos   = blkno * source->blksize;
-  main_file   *sfile = (main_file*) source->ioh;
-  main_blklru *blru  = NULL;
-  usize_t      onblk = xd3_bytes_on_srcblk (source, blkno);
-  usize_t      nread;
-  int         ret;
-  int         i;
-
-  if (allow_fake_source)
-    {
-      source->curblkno = blkno;
-      source->onblk    = onblk;
-      source->curblk   = lru[0].blk;
-      return 0;
-    }
-
-  if (do_not_lru)
-    {
-      /* Direct lookup assumes sequential scan w/o skipping blocks. */
-      int idx = blkno % lru_size;
-      if (lru[idx].blkno == blkno)
-        {
-          source->curblkno = blkno;
-          source->onblk    = onblk;
-          source->curblk   = lru[idx].blk;
-          lru_hits += 1;
-          return 0;
-        }
-
-      if (lru[idx].blkno != -1LL &&
-          lru[idx].blkno != blkno - lru_size)
-        {
-          return XD3_TOOFARBACK;
-        }
-    }
-  else
-    {
-      /* Sequential search through LRU. */
-      for (i = 0; i < lru_size; i += 1)
-        {
-          if (lru[i].blkno == blkno)
-            {
-              main_blklru_list_remove (& lru[i]);
-              main_blklru_list_push_back (& lru_list, & lru[i]);
-
-              source->curblkno = blkno;
-              source->onblk    = onblk;
-              source->curblk   = lru[i].blk;
-              lru_hits += 1;
-              return 0;
-            }
-        }
-    }
-
-  if (! main_blklru_list_empty (& lru_free))
-    {
-      blru = main_blklru_list_pop_front (& lru_free);
-    }
-  else if (! main_blklru_list_empty (& lru_list))
-    {
-      if (do_not_lru) {
-        blru = & lru[blkno % lru_size];
-        main_blklru_list_remove(blru);
-      } else {
-        blru = main_blklru_list_pop_front (& lru_list);
-      }
-      lru_misses += 1;
-    }
-
-  lru_filled += 1;
-
-  if ((ret = main_file_seek (sfile, pos)))
-    {
-      return ret;
-    }
-
-  if ((ret = main_file_read (sfile, (uint8_t*) blru->blk, source->blksize,
-                             & nread, "source read failed")))
-    {
-      return ret;
-    }
-
-  if (nread != onblk)
-    {
-      XPR(NT "source file size change: %s\n", sfile->filename);
-      return XD3_INTERNAL;
-    }
-
-  main_blklru_list_push_back (& lru_list, blru);
-
-  if (option_verbose > 2)
-    {
-      if (blru->blkno != -1LL)
-        {
-          XPR(NT "source block %"Q"u ejects %"Q"u (lru_hits=%u, lru_misses=%u, lru_filled=%u)\n",
-              blkno, blru->blkno, lru_hits, lru_misses, lru_filled);
-        }
-      else
-        {
-          XPR(NT "source block %"Q"u read (lru_hits=%u, lru_misses=%u, lru_filled=%u)\n",
-                                          blkno, lru_hits, lru_misses, lru_filled);
-        }
-    }
-
-  blru->blkno      = blkno;
-  source->curblk   = blru->blk;
-  source->curblkno = blkno;
-  source->onblk    = onblk;
-
-  return 0;
-}
-
-/******************************************************************************************
- Main routines
- ******************************************************************************************/
-
-/* This is a generic input function.  It calls the xd3_encode_input or xd3_decode_input
- * functions and makes calls to the various input handling routines above, which
- * coordinate external decompression.
- */
-static int
-main_input (xd3_cmd     cmd,
-            main_file   *ifile,
-            main_file   *ofile,
-            main_file   *sfile)
-{
-  int        ret;
-  xd3_stream stream;
-  usize_t    nread;
-  int        stream_flags = 0;
-  xd3_config config;
-  xd3_source source;
-  xoff_t     last_total_in = 0;
-  xoff_t     last_total_out = 0;
-  long       start_time;
-
-  int (*input_func) (xd3_stream*);
-  int (*output_func) (xd3_stream*, main_file *);
-
-  memset (& source, 0, sizeof (source));
-  memset (& config, 0, sizeof (config));
-
-  config.alloc = main_alloc;
-  config.freef = main_free1;
-  config.sec_data.ngroups = 1;
-  config.sec_addr.ngroups = 1;
-  config.sec_inst.ngroups = 1;
-
-  /* main_input setup. */
-  switch ((int) cmd)
-    {
-#if VCDIFF_TOOLS
-           if (1) { case CMD_PRINTHDR:   stream_flags = XD3_JUST_HDR; }
-      else if (1) { case CMD_PRINTHDRS:  stream_flags = XD3_SKIP_WINDOW; }
-      else        { case CMD_PRINTDELTA: stream_flags = XD3_SKIP_EMIT; }
-      ifile->flags |= RD_NONEXTERNAL;
-      input_func    = xd3_decode_input;
-      output_func   = main_print_func;
-      stream_flags |= XD3_ADLER32_NOVER;
-      break;
-#endif
-#if XD3_ENCODER
-    case CMD_ENCODE:
-      input_func  = xd3_encode_input;
-      output_func = main_write_output;
-
-      if (option_use_checksum) { stream_flags |= XD3_ADLER32; }
-      if (option_use_secondary)
-        {
-          /* The default secondary compressor is DJW, if it's compiled, being used, etc. */
-          if (option_secondary == NULL)
-            {
-              if (SECONDARY_DJW) { stream_flags |= XD3_SEC_DJW; }
-            }
-          else
-            {
-              if (strcmp (option_secondary, "fgk") == 0 && SECONDARY_FGK)
-                {
-                  stream_flags |= XD3_SEC_FGK;
-                }
-              else if (strcmp (option_secondary, "djw") == 0 && SECONDARY_DJW)
-                {
-                  stream_flags |= XD3_SEC_DJW;
-                }
-              else
-                {
-                  XPR(NT "unrecognized secondary compressor type: %s\n", option_secondary);
-                  return EXIT_FAILURE;
-                }
-            }
-        }
-      if (option_no_compress)      { stream_flags |= XD3_NOCOMPRESS; }
-      if (option_use_altcodetable) { stream_flags |= XD3_ALT_CODE_TABLE; }
-      if (option_smatch_config)
-        {
-          char *s = option_smatch_config, *e;
-          int values[XD3_SOFTCFG_VARCNT];
-          int got;
-
-          config.smatch_cfg = XD3_SMATCH_SOFT;
-
-          for (got = 0; got < XD3_SOFTCFG_VARCNT; got += 1, s = e + 1)
-            {
-              values[got] = strtol (s, &e, 10);
-
-              if ((values[got] < 0) ||
-                  (e == s) ||
-                  (got < XD3_SOFTCFG_VARCNT-1 && *e == 0) ||
-                  (got == XD3_SOFTCFG_VARCNT-1 && *e != 0))
-                {
-                  XPR(NT "invalid string match specifier (-C)\n");
-                  return EXIT_FAILURE;
-                }
-            }
-
-          config.smatcher_soft.large_look    = values[0];
-          config.smatcher_soft.large_step    = values[1];
-          config.smatcher_soft.small_look    = values[2];
-          config.smatcher_soft.small_chain   = values[3];
-          config.smatcher_soft.small_lchain  = values[4];
-          config.smatcher_soft.ssmatch       = values[5];
-          config.smatcher_soft.try_lazy      = values[6];
-          config.smatcher_soft.max_lazy      = values[7];
-          config.smatcher_soft.long_enough   = values[8];
-          config.smatcher_soft.promote       = values[9];
-        }
-      else if (option_level < 5) { config.smatch_cfg = XD3_SMATCH_FAST; }
-      else                       { config.smatch_cfg = XD3_SMATCH_SLOW; }
-      break;
-#endif
-    case CMD_DECODE:
-      if (option_use_checksum == 0) { stream_flags |= XD3_ADLER32_NOVER; }
-      stream_flags  = 0;
-      ifile->flags |= RD_NONEXTERNAL;
-      input_func    = xd3_decode_input;
-      output_func   = main_write_output;
-      break;
-    default:
-      XPR(NT "internal error\n");
-      return EXIT_FAILURE;
-    }
-
-  start_time = get_millisecs_now ();
-
-  xoff_t input_size = 0;
-  if (main_file_stat (ifile, & input_size, 0) == 0)
-    {
-      option_winsize = min (input_size, (xoff_t) option_winsize);
-    }
-
-  option_srcwinsz = max(option_srcwinsz, XD3_ALLOCSIZE);
-  option_winsize = max(option_winsize, XD3_ALLOCSIZE);
-
-  source.blksize = (option_srcwinsz / 32) & ~(XD3_ALLOCSIZE-1);
-  source.blksize = max(XD3_DEFAULT_WINSIZE, source.blksize);
-
-  config.srcwin_maxsz = option_srcwinsz;
-  config.winsize = option_winsize;
-  config.memsize = option_memsize;
-  config.getblk = main_getblk_func;
-  config.flags = stream_flags;
-
-  if (option_verbose > 1)
-    {
-      XPR(NT "input buffer size: %u\n", option_winsize);
-    }
-
-  if ((main_bdata = main_malloc (option_winsize)) == NULL)
-    {
-      return EXIT_FAILURE;
-    }
-
-  if ((ret = xd3_config_stream (& stream, & config)))
-    {
-      XPR(NT XD3_LIB_ERRMSG (& stream, ret));
-      return EXIT_FAILURE;
-    }
-
-  if (IS_ENCODE (cmd))
-    {
-      /* When encoding, open the source file, possibly decompress it.  The decoder delays
-       * this step until XD3_GOTHEADER. */
-      if (sfile->filename != NULL && (ret = main_set_source (& stream, cmd, sfile, & source)))
-        {
-          return EXIT_FAILURE;
-        }
-    }
-
-  /* This times each window. */
-  get_millisecs_since ();
-
-  /* Main input loop. */
-  do
-    {
-      xoff_t input_offset;
-      xoff_t input_remain;
-      usize_t try_read;
-
-      input_offset = ifile->nread;
-      /*XD3_ASSERT (input_offset <= option_last_offset);*/
-
-      input_remain = /*option_last_offset*/ XOFF_T_MAX - input_offset;
-
-      try_read = (usize_t) min ((xoff_t) config.winsize, input_remain);
-
-      if ((ret = main_read_primary_input (ifile, main_bdata, try_read, & nread)))
-        {
-          return EXIT_FAILURE;
-        }
-
-      /* If we've reached EOF tell the stream to flush. */
-      if (nread < try_read)
-        {
-          stream_flags |= XD3_FLUSH;
-          xd3_set_flags (& stream, stream_flags);
-        }
-
-#if XD3_ENCODER
-      /* After the first main_read_primary_input completes, we know all the information
-       * needed to encode the application header. */
-      if (cmd == CMD_ENCODE && (ret = main_set_appheader (& stream, ifile, sfile)))
-        {
-          return EXIT_FAILURE;
-        }
-#endif
-      xd3_avail_input (& stream, main_bdata, nread);
-
-      /* If we read zero bytes after encoding at least one window... */
-      if (nread == 0 && stream.current_window > 0) {
-        break;
-      }
-
-    again:
-      ret = input_func (& stream);
-      /*if (option_verbose > 1) { XPR(NT XD3_LIB_ERRMSG (& stream, ret)); }*/
-
-      switch (ret)
-        {
-        case XD3_INPUT:
-          continue;
-
-        case XD3_GOTHEADER:
-          {
-            XD3_ASSERT (stream.current_window == 0);
-
-            /* Need to process the appheader as soon as possible.  It may contain a
-             * suggested default filename/decompression routine for the ofile, and it may
-             * contain default/decompression routine for the sources. */
-            if (cmd == CMD_DECODE)
-              {
-                int have_src = sfile->filename != NULL;
-                int need_src = xd3_decoder_needs_source (& stream);
-                int recv_src;
-
-                /* May need to set the sfile->filename if none was given. */
-                main_get_appheader (& stream, ifile, ofile, sfile);
-
-                recv_src = sfile->filename != NULL;
-
-                /* Check if the user expected a source to be required although it was not. */
-                if (have_src && ! need_src && ! option_quiet)
-                  {
-                    XPR(NT "warning: output window %"Q"u does not copy source\n", stream.current_window);
-                  }
-
-                /* Check if we have no source name and need one. */
-                /* TODO: this doesn't fire due to cpyblocks_ calculation check */
-                if (need_src && ! recv_src)
-                  {
-                    XPR(NT "input requires a source file, use -s\n");
-                    return EXIT_FAILURE;
-                  }
-
-                /* Now open the source file. */
-                if (need_src && (ret = main_set_source (& stream, cmd, sfile, & source)))
-                  {
-                    return EXIT_FAILURE;
-                  }
-              }
-            else if (cmd == CMD_PRINTHDR ||
-                     cmd == CMD_PRINTHDRS ||
-                     cmd == CMD_PRINTDELTA)
-              {
-                if (xd3_decoder_needs_source (& stream) && sfile->filename == NULL)
-                  {
-                    allow_fake_source = 1; 
-                    sfile->filename = "<placeholder>";
-                    main_set_source (& stream, cmd, sfile, & source);
-                  }
-              }
-          }
-        /* FALLTHROUGH */
-        case XD3_WINSTART:
-          {
-            /* Set or unset XD3_SKIP_WINDOW. */
-            /*if (stream.current_window < option_first_window || stream.current_window > option_last_window)
-              { stream_flags |= XD3_SKIP_WINDOW; }
-            else
-              { stream_flags &= ~XD3_SKIP_WINDOW; }*/
-
-            xd3_set_flags (& stream, stream_flags);
-            goto again;
-          }
-
-        case XD3_OUTPUT:
-          {
-            if (option_no_output == 0/* &&
-                stream.current_window >= option_first_window &&
-                stream.current_window <= option_last_window*/)
-              {
-                /* Defer opening the output file until the stream produces its first
-                 * output for both encoder and decoder, this way we delay long enough for
-                 * the decoder to receive the application header.  (Or longer if there are
-                 * skipped windows, but I can't think of any reason not to delay open.) */
-                
-                if (! main_file_isopen (ofile) && (ret = main_open_output (& stream, ofile)) != 0)
-                  {
-                    return EXIT_FAILURE;
-                  }
-                if ((ret = output_func (& stream, ofile)) && (ret != PRINTHDR_SPECIAL))
-                  {
-                    return EXIT_FAILURE;
-                  }
-                if (ret == PRINTHDR_SPECIAL)
-                  {
-                    xd3_abort_stream (& stream);
-                    ret = EXIT_SUCCESS;
-                    goto done;
-                  }
-                ret = 0;
-              }
-
-            xd3_consume_output (& stream);
-            goto again;
-          }
-
-        case XD3_WINFINISH:
-          {
-            if (IS_ENCODE (cmd) || cmd == CMD_DECODE)
-              {
-                int used_source = xd3_encoder_used_source (& stream);
-
-                if (! option_quiet && IS_ENCODE (cmd) && main_file_isopen (sfile) && ! used_source)
-                  {
-                    XPR(NT "warning: input position %"Q"u no source copies\n",
-                        stream.current_window * option_winsize);
-                  }
-
-                if (option_verbose)
-                  {
-                    char rrateavg[32], wrateavg[32], tm[32];
-                    char rdb[32], wdb[32];
-                    char trdb[32], twdb[32];
-                    long millis = get_millisecs_since ();
-                    usize_t this_read = stream.total_in - last_total_in;
-                    usize_t this_write = stream.total_out - last_total_out;
-                    last_total_in = stream.total_in;
-                    last_total_out = stream.total_out;
-
-                    if (option_verbose > 1)
-                      {
-                        XPR(NT "%"Q"u: in %s (%s): out %s (%s): total in %s: out %s: %s\n",
-                            stream.current_window,
-                            main_format_bcnt (this_read, rdb),
-                            main_format_rate (this_read, millis, rrateavg),
-                            main_format_bcnt (this_write, wdb),
-                            main_format_rate (this_write, millis, wrateavg),
-                            main_format_bcnt (stream.total_in, trdb),
-                            main_format_bcnt (stream.total_out, twdb),
-                            main_format_millis (millis, tm));
-                      }
-                    else
-                      {
-                        XPR(NT "%"Q"u: in %s: out %s: total in %s: out %s: %s\n",
-                            stream.current_window,
-                            main_format_bcnt (this_read, rdb),
-                            main_format_bcnt (this_write, wdb),
-                            main_format_bcnt (stream.total_in, trdb),
-                            main_format_bcnt (stream.total_out, twdb),
-                            main_format_millis (millis, tm));
-                      }
-                  }
-              }
-            goto again;
-          }
-
-        default:
-          /* input_func() error */
-          XPR(NT XD3_LIB_ERRMSG (& stream, ret));
-          return EXIT_FAILURE;
-        }
-    }
-  while (nread == config.winsize);
-done:
-  /* Close the inputs. (ifile must be open, sfile may be open) */
-  main_file_close (ifile);
-  main_file_close (sfile);
-
-#if EXTERNAL_COMPRESSION
-  if ((ret = main_external_compression_finish ()))
-    {
-      XPR(NT "external compression commands failed\n");
-      return EXIT_FAILURE;
-    }
-#endif
-
-  /* If output file is not open yet because of delayed-open, it means we never encountered
-   * a window in the delta, but it could have had a VCDIFF header?  TODO: solve this
-   * elsewhere.  For now, it prints "nothing to output" below, but the check doesn't
-   * happen in case of option_no_output. */
-  if (! option_no_output)
-    {
-      if (! main_file_isopen (ofile))
-        {
-          XPR(NT "nothing to output: %s\n", ifile->filename);
-          return EXIT_FAILURE;
-        }
-
-      /* Have to close the output before calling main_external_compression_finish, or else it hangs. */
-      if (main_file_close (ofile) != 0)
-        {
-          return EXIT_FAILURE;
-        }
-    }
-
-  if ((ret = xd3_close_stream (& stream)))
-    {
-      XPR(NT XD3_LIB_ERRMSG (& stream, ret));
-      return EXIT_FAILURE;
-    }
-
-  xd3_free_stream (& stream);
-
-  if (option_verbose)
-    {
-      char tm[32];
-      long end_time = get_millisecs_now ();
-      XPR(NT "command finished in %s\n", main_format_millis (end_time - start_time, tm));
-      XPR(NT "input bytes:  %"Q"u\n",  ifile->nread);
-      XPR(NT "output bytes: %"Q"u\n", ofile->nwrite);
-    }
-
-  return EXIT_SUCCESS;
-}
-
-/* free memory before exit, reset single-use variables. */
-static void
-main_cleanup (void)
-{
-  int i;
-  
-  if (option_appheader) { appheader_used = NULL; }
-
-  main_free (appheader_used);
-  main_free (main_bdata);
-
-#if EXTERNAL_COMPRESSION
-  main_free (ext_tmpfile);
-#endif
-
-  for (i = 0; lru && i < lru_size; i += 1)
-    {
-      main_free (lru[i].blk);
-    }
-
-  main_free (lru);
-
-  lru_hits = 0;
-  lru_misses = 0;
-  lru_filled = 0;
-
-  XD3_ASSERT (main_mallocs == 0);
-}
-
-int
-#if PYTHON_MODULE
-xd3_main_cmdline (int argc, char **argv)
-#else
-main (int argc, char **argv)
-#endif
-{
-  xd3_cmd cmd;
-  main_file ifile;
-  main_file ofile;
-  main_file sfile;
-  static char *flags = "0123456789cdefhnqvDJNORTVs:B:C:E:F:L:O:P:M:W:A::S::";
-  int my_optind;
-  char *my_optarg;
-  char *my_optstr;
-  char *sfilename;
-  int orig_argc = argc;
-  char **orig_argv = argv;
-  int ret;
-
-  main_file_init (& ifile);
-  main_file_init (& ofile);
-  main_file_init (& sfile);
-
- go:  /* Go. */
-  cmd = CMD_NONE;
-  sfilename = NULL;
-  my_optind = 1;
-  argv = orig_argv;
-  argc = orig_argc;
-  program_name = argv[0];
-  extcomp_types[0].recomp_cmdname = program_name;
-  extcomp_types[0].decomp_cmdname = program_name;
- takearg:
-  my_optarg = NULL;
-  my_optstr = argv[my_optind];
-  /* This doesn't use getopt() because it makes trouble for -P & python which reenter
-   * main() and thus care about freeing all memory.  I never had much trust for getopt
-   * anyway, it's too opaque.  This implements a fairly standard non-long-option getopt
-   * with support for named operations (e.g., "xdelta3 [encode|decode|printhdr...] < in >
-   * out").  I'll probably add long options at some point. */
-  if (my_optstr)
-    {
-      if (*my_optstr == '-')    { my_optstr += 1; }
-      else if (cmd == CMD_NONE) { goto nonflag; }
-      else                      { my_optstr = NULL; }
-    }
-  while (my_optstr)
-    {
-      char *s;
-      my_optarg = NULL;
-      if ((ret = *my_optstr++) == 0) { my_optind += 1; goto takearg; }
-
-      /* Option handling: first check for one ':' following the option in flags, then
-       * check for two.  The syntax allows:
-       *
-       * 1. -Afoo                   defines optarg="foo"
-       * 2. -A foo                  defines optarg="foo"
-       * 3. -A ""                   defines optarg="" (allows optional empty-string)
-       * 4. -A [EOA or -moreargs]   error (mandatory case)
-       * 5. -A [EOA -moreargs]      defines optarg=NULL (optional case)
-       * 6. -A=foo                  defines optarg="foo"
-       * 7. -A=                     defines optarg="" (mandatory case)
-       * 8. -A=                     defines optarg=NULL (optional case)
-       *
-       * See tests in test_command_line_arguments().
-       */
-      s = strchr (flags, ret);
-      if (s && s[1] && s[1] == ':')
-        {
-          int eqcase = 0;
-          int option = s[2] && s[2] == ':';
-
-          /* Case 1, set optarg to the remaining characters. */
-          my_optarg = my_optstr;
-          my_optstr = "";
-
-          /* Case 2-5 */
-          if (*my_optarg == 0)
-            {
-              /* Condition 4-5 */
-              int have_arg = my_optind < (argc - 1) && *argv[my_optind+1] != '-';
-
-              if (! have_arg)
-                {
-                  if (! option)
-                  {
-                    /* Case 4 */
-                    XPR(NT "-%c: requires an argument\n", ret);
-                    ret = EXIT_FAILURE;
-                    goto cleanup;
-                  }
-                  /* Case 5. */
-                  my_optarg = NULL;
-                }
-              else
-                {
-                  /* Case 2-3. */
-                  my_optarg = argv[++my_optind];
-                }
-            }
-          /* Case 6-8. */
-          else if (*my_optarg == '=')
-            {
-              /* Remove the = in all cases. */
-              my_optarg += 1;
-              eqcase = 1;
-
-              if (option && *my_optarg == 0)
-                {
-                  /* Case 8. */
-                  my_optarg = NULL;
-                }
-            }
-        }
-
-      switch (ret)
-        {
-        /* case: if no '-' was found, maybe check for a command name. */
-        nonflag:
-               if (strcmp (my_optstr, "decode") == 0) { cmd = CMD_DECODE; }
-          else if (strcmp (my_optstr, "encode") == 0)
-            {
-#if XD3_ENCODER
-              cmd = CMD_ENCODE;
-#else
-              XPR(NT "encoder support not compiled\n");
-              return EXIT_FAILURE;
-#endif
-            }
-          else if (strcmp (my_optstr, "config") == 0) { cmd = CMD_CONFIG; }
-#if REGRESSION_TEST
-          else if (strcmp (my_optstr, "test") == 0) { cmd = CMD_TEST; }
-#endif
-#if VCDIFF_TOOLS
-          else if (strcmp (my_optstr, "printhdr") == 0) { cmd = CMD_PRINTHDR; }
-          else if (strcmp (my_optstr, "printhdrs") == 0) { cmd = CMD_PRINTHDRS; }
-          else if (strcmp (my_optstr, "printdelta") == 0) { cmd = CMD_PRINTDELTA; }
-#endif
-
-          /* If no option was found and still no command, let the default command be
-           * encode.  The remaining args are treated as filenames. */
-          if (cmd == CMD_NONE)
-            {
-              cmd = CMD_DEFAULT;
-              my_optstr = NULL;
-              break;
-            }
-          else
-            {
-              /* But if we find a command name, continue the getopt loop. */
-              my_optind += 1;
-              goto takearg;
-            }
-
-          /* gzip-like options */
-        case '0': case '1': case '2': case '3': case '4':
-        case '5': case '6': case '7': case '8': case '9':
-          option_level = ret - '0';
-          break;
-        case 'f': option_force = 1; break;
-        case 'v': option_verbose += 1; option_quiet = 0; break;
-        case 'q': option_quiet = 1; option_verbose = 0; break;
-        case 'c': option_stdout = 1; break;
-        case 'd':
-          if (cmd == CMD_NONE) { cmd = CMD_DECODE; }
-          else { ret = main_help (); goto exit; }
-          break;
-        case 'e':
-#if XD3_ENCODER
-          if (cmd == CMD_NONE) { cmd = CMD_ENCODE; }
-          else { ret = main_help (); goto exit; }
-          break;
-#else
-          XPR(NT "encoder support not compiled\n");
-          return EXIT_FAILURE;
-#endif    
-        case 'P':
-          /* only set profile count once, since... */
-          if (option_profile_cnt == 0)
-            {
-              if ((ret = main_atou(my_optarg, (usize_t*) & option_profile_cnt, 0, 'P'))) { goto exit; }
-
-              if (option_profile_cnt <= 0)
-                {
-                  ret = EXIT_SUCCESS;
-                  goto exit;
-                }
-            }
-          break;
-
-        case 'n': option_use_checksum = 0; break;
-        case 'N': option_no_compress = 1; break;
-        case 'T': option_use_altcodetable = 1; break;
-        case 'C': option_smatch_config = my_optarg; break;
-        case 'J': option_no_output = 1; break;
-        case 'O': option_xdelta1 = my_optarg; break;
-        case 'S': if (my_optarg == NULL) { option_use_secondary = 0; }
-                  else { option_use_secondary = 1; option_secondary = my_optarg; } break;
-        case 'A': if (my_optarg == NULL) { option_use_appheader = 0; }
-                  else { option_appheader = (uint8_t*) my_optarg; } break;
-        case 'B': if ((ret = main_atou (my_optarg, & option_srcwinsz, XD3_ALLOCSIZE, 'B')))
-          {
-            goto exit;
-          }
-          break;
-        case 'W': if ((ret = main_atou (my_optarg, & option_winsize, XD3_ALLOCSIZE, 'W')))
-          {
-            goto exit;
-          }
-          break;
-        case 'M': if ((ret = main_atou (my_optarg, & option_memsize, XD3_ALLOCSIZE, 'M')))
-          {
-            goto exit;
-          }
-          break;
-        case 'D':
-#if EXTERNAL_COMPRESSION == 0
-          if (! option_quiet)
-            {
-              XPR(NT "warning: -D option ignored, "
-                       "external compression support was not compiled\n");
-            }
-#else
-          option_decompress_inputs  = 0;
-#endif
-          break;
-        case 'R':
-#if EXTERNAL_COMPRESSION == 0
-          if (! option_quiet)
-            {
-              XPR(NT "warning: -R option ignored, "
-                       "external compression support was not compiled\n");
-            }
-#else
-          option_recompress_outputs = 0;
-#endif
-          break;
-        case 's':
-          if (sfilename != NULL)
-            {
-              XPR(NT "specify only one source file\n");
-              goto cleanup;
-            }
-
-          sfilename = my_optarg;
-          break;
-
-        case 'V':
-          ret = main_version (); goto exit;
-        default:
-          ret = main_help (); goto exit;
-        }
-    }
-
-  option_source_filename = sfilename;
-
-  /* In case there were no arguments, set the default command. */
-  if (cmd == CMD_NONE) { cmd = CMD_DEFAULT; }
-
-  argc -= my_optind;
-  argv += my_optind;
-
-  /* There may be up to two more arguments. */
-  if (argc > 2)
-    {
-      XPR(NT "too many filenames: %s ...\n", argv[2]);
-      ret = EXIT_FAILURE;
-      goto cleanup;
-    }
-
-  extcomp_types[1].recomp_cmdname = option_xdelta1;
-  extcomp_types[1].decomp_cmdname = option_xdelta1;
-  
-  if (option_verbose > 1)
-    {
-      int l = 1;
-      int i;
-      char buf[1024];
-      for (i = 0; i < orig_argc; i += 1)
-        {
-          l += strlen (orig_argv[i]) + 1;
-        }
-      buf[0] = 0;
-      for (i = 0; i < orig_argc; i += 1)
-        {
-          strcat (buf, orig_argv[i]);
-          strcat (buf, " ");
-        }
-      XPR(NT "command line: %s\n", buf);
-    }      
-
-  ifile.flags    = RD_FIRST;
-  sfile.flags    = RD_FIRST;
-  sfile.filename = option_source_filename;
-
-  /* The infile takes the next argument, if there is one.  But if not, infile is set to
-   * stdin. */
-  if (argc > 0)
-    {
-      ifile.filename = argv[0];
-
-      if ((ret = main_file_open (& ifile, ifile.filename, XO_READ)))
-        {
-          goto cleanup;   
-        }
-    }
-  else
-    {
-      XSTDIN_XF (& ifile);
-    }
-
-  /* The ofile takes the following argument, if there is one.  But if not, it is left NULL
-   * until the application header is processed.  It will be set in main_open_output. */
-  if (argc > 1)
-    {
-      /* Check for conflicting arguments. */
-      if (option_stdout && ! option_quiet)
-        {
-          XPR(NT "warning: -c option overrides output filename: %s\n", argv[1]);
-        }
-
-      if (! option_stdout) { ofile.filename = argv[1]; }
-    }
-
-  switch (cmd)
-    {
-    case CMD_PRINTHDR:
-    case CMD_PRINTHDRS:
-    case CMD_PRINTDELTA:
-#if XD3_ENCODER
-    case CMD_ENCODE:
-      if (cmd == CMD_ENCODE)
-        {
-          do_not_lru = 1;
-        }
-#endif
-    case CMD_DECODE:
-      ret = main_input (cmd, & ifile, & ofile, & sfile);
-      break;
-
-#if REGRESSION_TEST
-    case CMD_TEST:
-      ret = xd3_selftest ();
-      break;
-#endif
-
-    case CMD_CONFIG:
-      ret = main_config ();
-      break;
-
-    default:
-      ret = main_help ();
-      break;
-    }
-
-#if EXTERNAL_COMPRESSION
-  if (ext_tmpfile != NULL) { unlink (ext_tmpfile); }
-#endif
-
-  if (0)
-    {
-    cleanup:
-      ret = EXIT_FAILURE;
-    exit:
-      (void)0;
-    }
-
-  main_file_cleanup (& ifile);
-  main_file_cleanup (& ofile);
-  main_file_cleanup (& sfile);
-
-  main_cleanup ();
-
-  if (--option_profile_cnt > 0 && ret == EXIT_SUCCESS) { goto go; }
-
-  return ret;
-}
-
-static int
-main_help (void)
-{
-  /* TODO: update www/xdelta3-cmdline.html */ 
-  main_version ();
-  P(RINT "usage: xdelta3 [command/options] [input [output]]\n");
-  P(RINT "special command names:\n");
-  P(RINT "    config      prints xdelta3 configuration\n");
-  P(RINT "    decode      decompress the input\n");
-  P(RINT "    encode      compress the input%s\n", XD3_ENCODER ? "" : " [Not compiled]");
-#if REGRESSION_TEST
-  P(RINT "    test        run the builtin tests\n");
-#endif
-#if VCDIFF_TOOLS
-  P(RINT "special commands for VCDIFF inputs:\n");
-  P(RINT "    printdelta  print information about the entire delta\n");
-  P(RINT "    printhdr    print information about the first window\n");
-  P(RINT "    printhdrs   print information about all windows\n");
-#endif
-  P(RINT "standard options:\n");
-  P(RINT "   -0 .. -9     compression level\n");
-  P(RINT "   -c           use stdout\n");
-  P(RINT "   -d           decompress\n");
-  P(RINT "   -e           compress%s\n", XD3_ENCODER ? "" : " [Not compiled]");
-  P(RINT "   -f           force overwrite\n");
-  P(RINT "   -h           show help\n");
-  P(RINT "   -q           be quiet\n");
-  P(RINT "   -v           be verbose (max 2)\n");
-  P(RINT "   -V           show version\n");
-
-  P(RINT "memory options:\n");
-  P(RINT "   -B blksize   source file block size\n");
-  P(RINT "   -M memsize   memory budget for hash tables\n");
-  P(RINT "   -W winsize   input window buffer size\n");
-
-  P(RINT "compression options:\n");
-  P(RINT "   -s source    source file to copy from (if any)\n");
-  P(RINT "   -S [djw|fgk] enable/disable secondary compression\n");
-  P(RINT "   -N           disable small string-matching compression\n");
-  P(RINT "   -D           disable external decompression (encode/decode)\n");
-  P(RINT "   -R           disable external recompression (decode)\n");
-
-#if XD3_DEBUG > 0
-  P(RINT "developer options:\n");
-  P(RINT "   -A [apphead] disable/provide application header\n");
-  P(RINT "   -C           soft config (see xdelta3-cfgs.h)\n");
-  P(RINT "   -J           disable output (check/compute only)\n");
-  P(RINT "   -P           repeat count (for profiling)\n");
-  P(RINT "   -T           use alternate code table\n");
-  P(RINT "   -n           disable checksum (encode/decode)\n");
-#endif
-  return EXIT_FAILURE;
-}
+/* xdelta 3 - delta compression tools and library

+ * Copyright (C) 2001 and onward.  Joshua P. MacDonald

+ *

+ *  This program is free software; you can redistribute it and/or modify

+ *  it under the terms of the GNU General Public License as published by

+ *  the Free Software Foundation; either version 2 of the License, or

+ *  (at your option) any later version.

+ *

+ *  This program is distributed in the hope that it will be useful,

+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of

+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ *  GNU General Public License for more details.

+ *

+ *  You should have received a copy of the GNU General Public License

+ *  along with this program; if not, write to the Free Software

+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

+ */

+

+/* This is all the extra stuff you need for convenience to users in a command line

+ * application.  It contains these major components:

+ *

+ * 1. VCDIFF tools

+ * 2. external compression support (this is POSIX-specific).

+ * 3. a general read/write loop that handles all of the Xdelta decode/encode/VCDIFF-print

+ *    functions

+ * 4. command-line interpreter

+ * 5. an Xdelta application header which stores default filename, external compression settings

+ * 6. output/error printing

+ * 7. basic file support and OS interface

+ */

+

+/* TODO list:

+ * 1. do exact gzip-like filename, stdout handling.  make a .xz extension, refuse

+ *    to encode to stdout without -cf, etc.

+ * 2. Allow the user to add a comment string to the app header without disturbing the default

+ *    behavior.

+ */

+

+/* On error handling and printing:

+ *

+ * The xdelta library sets stream->msg to indicate what condition caused an internal

+ * failure, but many failures originate here and are printed here.  The return convention

+ * is 0 for success, as throughout Xdelta code, but special attention is required here for

+ * the operating system calls with different error handling.  See the main_file_* routines.

+ * All errors in this file have a message printed at the time of occurance.  Since some of

+ * these calls occur within calls to the library, the error may end up being printed again

+ * with a more general error message.

+ */

+

+/******************************************************************************************/

+

+#ifndef XD3_POSIX

+#define XD3_POSIX 0

+#endif

+#ifndef XD3_STDIO

+#define XD3_STDIO 0

+#endif

+

+/* XPRINTX (used by main) prefixes an "xdelta3: " to the output. */

+#define XPR fprintf

+#define NT stderr, "xdelta3: "

+

+#define VC fprintf

+#define UT vcout,

+

+/* If none are set, default to posix. */

+#if (XD3_POSIX + XD3_STDIO) == 0

+#undef XD3_POSIX

+#define XD3_POSIX 1

+#endif

+

+/* Handle externally-compressed inputs. */

+#ifndef EXTERNAL_COMPRESSION

+#define EXTERNAL_COMPRESSION 1

+#endif

+

+#define PRINTHDR_SPECIAL -4378291

+

+/* The number of soft-config variables.  */

+#define XD3_SOFTCFG_VARCNT 1

+

+/* this is used as in XPR(NT XD3_LIB_ERRMSG (stream, ret)) to print an error message

+ * from the library. */

+#define XD3_LIB_ERRMSG(stream, ret) "%s: %s\n", xd3_errstring (stream), xd3_strerror (ret)

+

+#include <stdio.h>  /* fprintf */

+

+#if XD3_POSIX

+#include <unistd.h> /* close, read, write... */

+#include <sys/types.h>

+#include <fcntl.h>

+#endif

+

+#ifndef _WIN32

+#include <unistd.h> /* lots */

+#include <sys/time.h> /* gettimeofday() */

+#include <sys/stat.h> /* stat() and fstat() */

+#else

+#define strtoll _strtoi64

+#include <sys/types.h>

+#include <sys/stat.h>

+#ifndef WIFEXITED

+#   define WIFEXITED(stat)  (((*((int *) &(stat))) & 0xff) == 0)

+#endif

+#ifndef WEXITSTATUS

+#   define WEXITSTATUS(stat) (((*((int *) &(stat))) >> 8) & 0xff)

+#endif

+#ifndef S_ISREG

+#   ifdef S_IFREG

+#       define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)

+#   else

+#       define S_ISREG(m) 1

+#   endif

+#endif /* !S_ISREG */

+#endif

+

+/******************************************************************************************

+ ENUMS and TYPES

+ ******************************************************************************************/

+

+/* These flags (mainly pertaining to main_read() operations) are set in the

+ * main_file->flags variable.  All are related to with external decompression support.

+ *

+ * RD_FIRST causes the external decompression check when the input is first read.

+ *

+ * RD_NONEXTERNAL disables external decompression for reading a compressed input, in the

+ * case of Xdelta inputs.  Note: Xdelta is supported as an external compression type,

+ * which makes is the reason for this flag.  An example to justify this is: to create a

+ * delta between two files that are VCDIFF-compressed.  Two external Xdelta decoders are

+ * run to supply decompressed source and target inputs to the Xdelta encoder. */

+typedef enum

+{

+  RD_FIRST        = (1 << 0),

+  RD_NONEXTERNAL  = (1 << 1),

+  RD_EXTERNAL_V1  = (1 << 2),

+} xd3_read_flags;

+

+/* main_file->mode values */

+typedef enum

+{

+  XO_READ  = 0,

+  XO_WRITE = 1,

+} main_file_modes;

+

+/* Main commands.  For example, CMD_PRINTHDR is the "xdelta printhdr" command. */

+typedef enum

+{

+  CMD_NONE = 0,

+  CMD_PRINTHDR,

+  CMD_PRINTHDRS,

+  CMD_PRINTDELTA,

+#if XD3_ENCODER

+  CMD_ENCODE,

+#endif

+  CMD_DECODE,

+  CMD_TEST,

+  CMD_CONFIG,

+} xd3_cmd;

+

+#if XD3_ENCODER

+#define CMD_DEFAULT CMD_ENCODE

+#define IS_ENCODE(cmd) (cmd == CMD_ENCODE)

+#else

+#define CMD_DEFAULT CMD_DECODE

+#define IS_ENCODE(cmd) (0)

+#endif

+

+typedef struct _main_file        main_file;

+typedef struct _main_extcomp     main_extcomp;

+typedef struct _main_blklru      main_blklru;

+typedef struct _main_blklru_list main_blklru_list;

+

+/* The main_file object supports abstract system calls like open, close, read, write, seek,

+ * stat.  The program uses these to represent both seekable files and non-seekable files.

+ * Source files must be seekable, but the target input and any output file do not require

+ * seekability.

+ */

+struct _main_file

+{

+#if XD3_STDIO

+  FILE               *file;

+#elif XD3_POSIX

+  int                 file;

+#endif

+

+  int                 mode;          /* XO_READ and XO_WRITE */

+  const char         *filename;      /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */

+  char               *filename_copy; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */

+  const char         *realname;      /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */

+  const main_extcomp *compressor; /* External compression struct. */

+  int                 flags;      /* RD_FIRST, RD_NONEXTERNAL, ... */

+  xoff_t              nread;      /* for input position */

+  xoff_t              nwrite;     /* for output position */

+};

+

+/* Various strings and magic values used to detect and call external compression.  See

+ * below for examples. */

+struct _main_extcomp

+{

+  const char    *recomp_cmdname;

+  const char    *recomp_options;

+

+  const char    *decomp_cmdname;

+  const char    *decomp_options;

+

+  const char    *ident;

+  const char    *magic;

+  int            magic_size;

+  int            flags;

+};

+

+/* This file implements a small LRU of source blocks.  For encoding purposes,

+ * we prevent paging in blocks we've already scanned in the source (return

+ * XD3_NOTAVAIL). */

+struct _main_blklru_list

+{

+  main_blklru_list  *next;

+  main_blklru_list  *prev;

+};

+

+struct _main_blklru

+{

+  uint8_t         *blk;

+  xoff_t           blkno;

+  main_blklru_list  link;

+};

+

+/* ... represented as a list (no cache index). */

+XD3_MAKELIST(main_blklru_list,main_blklru,link);

+

+/* Program options: various command line flags and options. */

+static int         option_stdout             = 0;

+static int         option_force              = 0;

+static int         option_verbose            = 0;

+static int         option_quiet              = 0;

+static int         option_level              = 6;

+static int         option_use_appheader      = 1;

+static uint8_t*    option_appheader          = NULL;

+static int         option_use_secondary      = /* until-standardized, leave this off */ 0;

+static char*       option_secondary          = NULL;

+static int         option_use_checksum       = 1;

+static int         option_use_altcodetable   = 0;

+static char*       option_smatch_config      = NULL;

+static int         option_no_compress        = 0;

+static int         option_no_output          = 0; /* go through the motions, but do not open or write output */

+static const char *option_source_filename    = NULL;

+static const char *option_xdelta1            = "xdelta1";

+

+static usize_t     option_winsize            = XD3_DEFAULT_WINSIZE;

+static usize_t     option_srcwinsz           = XD3_DEFAULT_SRCWINSZ;

+static usize_t     option_memsize            = XD3_DEFAULT_MEMSIZE;

+

+/* This controls the number of times main repeats itself, only for profiling. */

+static int option_profile_cnt = 0;

+

+/* These variables are supressed to avoid their use w/o support.  main() warns

+ * appropriately. */

+#if EXTERNAL_COMPRESSION

+static int         option_decompress_inputs  = 1;

+static int         option_recompress_outputs = 1;

+#endif

+

+/* This is for comparing "printdelta" output without attention to

+ * copy-instruction modes, useful for reverse engineering. */

+#if VCDIFF_TOOLS

+static int         option_print_cpymode = 1;

+#endif

+

+/* Static variables */

+IF_DEBUG(static int main_mallocs = 0;)

+

+static char*          program_name = NULL;

+static uint8_t*       appheader_used = NULL;

+static uint8_t*       main_bdata = NULL;

+

+/* The LRU: obviously this is shared by all callers. */

+static int               lru_size = 0;

+static main_blklru      *lru = NULL;  /* array of lru_size elts */

+static main_blklru_list  lru_list;

+static main_blklru_list  lru_free;

+static int               do_not_lru = 0;  /* set to avoid lru, instead discard oldest */

+

+static int lru_hits   = 0;

+static int lru_misses = 0;

+static int lru_filled = 0;

+

+/* Hacks for VCDIFF tools */

+static int allow_fake_source = 0;

+

+/* This array of compressor types is compiled even if EXTERNAL_COMPRESSION is false just so

+ * the program knows the mapping of IDENT->NAME. */

+static main_extcomp extcomp_types[] =

+{

+  /* The entry for xdelta3/1 must be 0/1 because the program_names are set there. */

+  { "xdelta3",  "-cfq",  "xdelta3",    "-dcfq",  "X", "\xd6\xc3\xc4", 3, RD_NONEXTERNAL },

+  { "xdelta1",  "delta", "xdelta1",    "patch", "1", "%XD",          3, RD_EXTERNAL_V1 },

+  { "bzip2",    "-cf",   "bzip2",      "-dcf",   "B", "BZh",          3, 0 },

+  { "gzip",     "-cf",   "gzip",       "-dcf",   "G", "\037\213",     2, 0 },

+  { "compress", "-cf",   "uncompress", "-cf",    "Z", "\037\235",     2, 0 },

+};

+

+static void main_get_appheader (xd3_stream *stream, main_file *ifile,

+                                main_file *output, main_file *sfile);

+

+static int main_help (void);

+

+static int

+main_version (void)

+{

+  /* $Format: "  P(RINT \"VERSION=3.$Xdelta3Version$\\n\");" $ */

+  P(RINT "VERSION=3.0h_pre0\n");

+  return EXIT_SUCCESS;

+}

+

+static int

+main_config (void)

+{

+  main_version ();

+

+  P(RINT "EXTERNAL_COMPRESSION=%d\n", EXTERNAL_COMPRESSION);

+  P(RINT "GENERIC_ENCODE_TABLES=%d\n", GENERIC_ENCODE_TABLES);

+  P(RINT "GENERIC_ENCODE_TABLES_COMPUTE=%d\n", GENERIC_ENCODE_TABLES_COMPUTE);

+  P(RINT "REGRESSION_TEST=%d\n", REGRESSION_TEST);

+  P(RINT "SECONDARY_DJW=%d\n", SECONDARY_DJW);

+  P(RINT "SECONDARY_FGK=%d\n", SECONDARY_FGK);

+  P(RINT "VCDIFF_TOOLS=%d\n", VCDIFF_TOOLS);

+  P(RINT "XD3_ALLOCSIZE=%d\n", XD3_ALLOCSIZE);

+  P(RINT "XD3_DEBUG=%d\n", XD3_DEBUG);

+  P(RINT "XD3_DEFAULT_CKSUM_SIZE=%d\n", XD3_DEFAULT_CKSUM_ADVANCE);

+  P(RINT "XD3_DEFAULT_IOPT_SIZE=%d\n", XD3_DEFAULT_IOPT_SIZE);

+  P(RINT "XD3_DEFAULT_MEMSIZE=%d\n", XD3_DEFAULT_MEMSIZE);

+  P(RINT "XD3_DEFAULT_SPREVSZ=%d\n", XD3_DEFAULT_SPREVSZ);

+  P(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ);

+  P(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE);

+  P(RINT "XD3_ENCODER=%d\n", XD3_ENCODER);

+  P(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE);

+  P(RINT "XD3_NODECOMPRESSSIZE=%d\n", XD3_NODECOMPRESSSIZE);

+  P(RINT "XD3_POSIX=%d\n", XD3_POSIX);

+  P(RINT "XD3_USE_LARGEFILE64=%d\n", XD3_USE_LARGEFILE64);

+

+  return EXIT_SUCCESS;

+}

+

+static void*

+main_malloc1 (usize_t size)

+{

+  void* r = malloc (size);

+  if (r == NULL) { XPR(NT "malloc: %s\n", xd3_strerror (ENOMEM)); }

+  else if (option_verbose > 2) { XPR(NT "malloc: %u: %p\n", size, r); }

+  return r;

+}

+

+static void*

+main_malloc (usize_t size)

+{

+  void *r = main_malloc1 (size);

+   if (r) { IF_DEBUG (main_mallocs += 1); }

+  return r;

+}

+

+static void*

+main_alloc (void   *opaque,

+            usize_t  items,

+            usize_t  size)

+{

+  return main_malloc1 (items * size);

+}

+

+static void

+main_free1 (void *opaque, void *ptr)

+{

+  if (option_verbose > 2) { XPR(NT "free: %p\n", ptr); }

+  free (ptr);

+}

+

+static void

+main_free (void *ptr)

+{

+  if (ptr)

+    {

+      IF_DEBUG (main_mallocs -= 1);

+      main_free1 (NULL, ptr);

+    }

+}

+

+/* This ensures that (ret = errno) always indicates failure, in case errno was

+ * accidentally not set.  If this prints there's a bug somewhere. */

+static int

+get_errno (void)

+{

+  if (errno == 0)

+    {

+      XPR(NT "you found a bug: expected errno != 0\n");

+      errno = XD3_INTERNAL;

+    }

+  return errno;

+}

+

+

+

+static long

+get_millisecs_now (void)

+{

+#ifndef WIN32

+  struct timeval tv;

+

+  gettimeofday (& tv, NULL);

+

+  return (tv.tv_sec) * 1000L + (tv.tv_usec) / 1000;

+#else

+  // Found this in an example on www.codeproject.com

+  // It doesn't matter that the offset is Jan 1, 1601

+  // Result is the numbre of 100 nanosecond units

+  // 100ns * 10,000 = 1ms

+  SYSTEMTIME st;

+  FILETIME ft;

+  __int64 *pi = (__int64*)&ft;

+  GetLocalTime(&st);

+  SystemTimeToFileTime(&st, &ft);

+  return (long)((*pi) / 10000);

+#endif

+}

+

+/* Always >= 1 millisec, right? */

+static long

+get_millisecs_since (void)

+{

+  static long last = 0;

+  long now = get_millisecs_now();

+  long diff = now - last;

+  last = now;

+  return diff;

+}

+

+static char*

+main_format_bcnt (xoff_t r, char *buf)

+{

+  static const char* fmts[] = { "B", "KB", "MB", "GB" };

+  int i;

+

+  for (i = 0; i < SIZEOF_ARRAY(fmts); i += 1)

+    {

+      if (r < 10 * 1e3 || i == (-1 + (int)SIZEOF_ARRAY(fmts)))

+            {

+              sprintf (buf, "%"Q"u %s", r, fmts[i]);

+              break;

+            }

+      r /= 1000;

+    }

+  return buf;

+}

+

+static char*

+main_format_rate (xoff_t bytes, long millis, char *buf)

+{

+  xoff_t r = (xoff_t)(1.0 * bytes / (1.0 * millis / 1000.0));

+  static char lbuf[32];

+

+  main_format_bcnt (r, lbuf);

+  sprintf (buf, "%s/sec", lbuf);

+  return buf;

+}

+

+static char*

+main_format_millis (long millis, char *buf)

+{

+  if (millis < 1000)       { sprintf (buf, "%lu ms", millis); }

+  else if (millis < 10000) { sprintf (buf, "%.1f sec", millis / 1000.0); }  

+  else                     { sprintf (buf, "%lu sec", millis / 1000L); }  

+  return buf;

+}

+

+/* A safe version of strtol for xoff_t. */

+static int

+main_strtoxoff (const char* s, xoff_t *xo, char which)

+{

+  char *e;

+  xoff_t x;

+

+  XD3_ASSERT(s && *s != 0);

+

+  {

+    /* Should check LONG_MIN, LONG_MAX, LLONG_MIN, LLONG_MAX? */

+#if SIZEOF_XOFF_T == 4

+    long xx = strtol (s, &e, 0);

+#else

+    long long xx = strtoll (s, &e, 0);

+#endif

+

+    if (xx < 0)

+      {

+        XPR(NT "-%c: negative integer: %s\n", which, s);

+        return EXIT_FAILURE;

+      }

+

+    x = xx;

+  }

+

+  if (*e != 0)

+    {

+      XPR(NT "-%c: invalid integer: %s\n", which, s);

+      return EXIT_FAILURE;

+    }

+

+  (*xo) = x;

+  return 0;

+}

+

+static int

+main_atou (const char* arg, usize_t *xo, usize_t low, char which)

+{

+  xoff_t x;

+  int ret;

+

+  if ((ret = main_strtoxoff (arg, & x, which))) { return ret; }

+

+  if (x > USIZE_T_MAX || x < low)

+    {

+      XPR(NT "-%c: minimum value: %u", which, low);

+      return EXIT_FAILURE;

+    }

+  (*xo) = (usize_t)x;

+  return 0;

+}

+

+/******************************************************************************************

+ FILE BASICS

+ ******************************************************************************************/

+

+/* With all the variation in file system-call semantics, arguments, return values and

+ * error-handling for the POSIX and STDIO file APIs, the insides of these functions make

+ * me sick, which is why these wrappers exist. */

+

+#define XOPEN_OPNAME (xfile->mode == XO_READ ? "read" : "write")

+#define XOPEN_STDIO  (xfile->mode == XO_READ ? "rb" : "wb")

+#define XOPEN_POSIX  (xfile->mode == XO_READ ? O_RDONLY : O_WRONLY | O_CREAT | O_TRUNC)

+#define XOPEN_MODE   (xfile->mode == XO_READ ? 0 : 0666)

+

+#define XF_ERROR(op, name, ret) XPR(NT "file %s failed: %s: %s: %s\n", (op), XOPEN_OPNAME, (name), xd3_strerror (ret))

+

+#if XD3_STDIO

+#define XFNO(f) fileno(f->file)

+#define XSTDOUT_XF(f) { (f)->file = stdout; (f)->filename = "/dev/stdout"; }

+#define XSTDERR_XF(f) { (f)->file = stderr; (f)->filename = "/dev/stderr"; }

+#define XSTDIN_XF(f)  { (f)->file = stdin;  (f)->filename = "/dev/stdin"; }

+

+#elif XD3_POSIX

+#define XFNO(f) f->file

+#define XSTDOUT_XF(f) { (f)->file = STDOUT_FILENO; (f)->filename = "/dev/stdout"; }

+#define XSTDERR_XF(f) { (f)->file = STDERR_FILENO; (f)->filename = "/dev/stderr"; }

+#define XSTDIN_XF(f)  { (f)->file = STDIN_FILENO;  (f)->filename = "/dev/stdin"; }

+#endif

+

+static void

+main_file_init (main_file *xfile)

+{

+  memset (xfile, 0, sizeof (*xfile));

+

+#if XD3_POSIX

+  xfile->file = -1;

+#endif

+}

+

+static void

+main_file_cleanup (main_file *xfile)

+{

+  if (xfile->filename_copy) {

+    main_free(xfile->filename_copy);

+  }

+}

+

+static int

+main_file_isopen (main_file *xfile)

+{

+#if XD3_STDIO

+  return xfile->file != NULL;

+

+#elif XD3_POSIX

+  return xfile->file != -1;

+#endif

+}

+

+static int

+main_file_close (main_file *xfile)

+{

+  int ret = 0;

+

+  if (! main_file_isopen (xfile))

+    {

+      return 0;

+    }

+

+#if XD3_STDIO

+  ret = fclose (xfile->file);

+  xfile->file = NULL;

+

+#elif XD3_POSIX

+  ret = close (xfile->file);

+  xfile->file = -1;

+#endif

+

+  if (ret != 0) { XF_ERROR ("close", xfile->filename, ret = get_errno ()); }

+  return ret;

+}

+

+static int

+main_file_open (main_file *xfile, const char* name, int mode)

+{

+  int ret = 0;

+

+  xfile->mode = mode;

+

+  XD3_ASSERT (! main_file_isopen (xfile));

+

+#if XD3_STDIO

+  xfile->file = fopen (name, XOPEN_STDIO);

+

+  ret = (xfile->file == NULL) ? get_errno () : 0;

+

+#elif XD3_POSIX

+  if ((ret = open (name, XOPEN_POSIX, XOPEN_MODE)) < 0)

+    {

+      ret = get_errno ();

+    }

+  else

+    {

+      xfile->file = ret;

+      ret = 0;

+    }

+#endif

+  if (ret) { XF_ERROR ("open", name, ret); }

+  else     { xfile->realname = name; xfile->nread = 0; }

+  return ret;

+}

+

+static int

+main_file_stat (main_file *xfile, xoff_t *size, int err_ifnoseek)

+{

+  int ret;

+  struct stat sbuf;

+

+  XD3_ASSERT (main_file_isopen (xfile));

+

+  if (fstat (XFNO (xfile), & sbuf) < 0)

+    {

+      ret = get_errno ();

+      if (err_ifnoseek) { XF_ERROR ("stat", xfile->filename, ret); }

+      return ret;

+    }

+

+  if (! S_ISREG (sbuf.st_mode))

+    {

+      if (err_ifnoseek) { XPR(NT "source file must be seekable: %s\n", xfile->filename); }

+      return ESPIPE;

+    }

+

+  (*size) = sbuf.st_size;

+  return 0;

+}

+

+static int

+main_file_exists (main_file *xfile)

+{

+  struct stat sbuf;

+  return stat (xfile->filename, & sbuf) == 0 && S_ISREG (sbuf.st_mode);

+}

+

+#if (XD3_POSIX || EXTERNAL_COMPRESSION)

+/* POSIX-generic code takes a function pointer to read() or write().  This calls the

+ * function repeatedly until the buffer is full or EOF.  The NREAD parameter is not

+ * set for write, NULL is passed.  Return is signed, < 0 indicate errors, otherwise

+ * byte count. */

+typedef int (xd3_posix_func) (int fd, uint8_t *buf, usize_t size);

+

+static int

+xd3_posix_io (int fd, uint8_t *buf, usize_t size, xd3_posix_func *func, usize_t *nread)

+{

+  int ret;

+  usize_t nproc = 0;

+

+  while (nproc < size)

+    {

+      int result = (*func) (fd, buf + nproc, size - nproc);

+      

+      if (result < 0)

+        {

+          ret = get_errno ();

+          if (ret != EAGAIN && ret != EINTR)

+            {

+              return ret;

+            }

+          result = 0;

+        }

+

+      if (nread != NULL && result == 0) { break; }

+

+      nproc += result;

+    }

+  if (nread != NULL) { (*nread) = nproc; }

+  return 0;

+}

+#endif

+

+/* POSIX is unbuffered, while STDIO is buffered.  main_file_read() should always be called

+ * on blocks. */

+static int

+main_file_read (main_file   *ifile,

+               uint8_t    *buf,

+               usize_t      size,

+               usize_t     *nread,

+               const char *msg)

+{

+  int ret = 0;

+

+#if XD3_STDIO

+  usize_t result;

+

+  result = fread (buf, 1, size, ifile->file);

+

+  if (result < size && ferror (ifile->file))

+    {

+      ret = get_errno ();

+    }

+  else

+    {

+      *nread = result;

+    }

+

+#elif XD3_POSIX

+  ret = xd3_posix_io (ifile->file, buf, size, (xd3_posix_func*) &read, nread);

+#endif

+

+  if (ret)

+    {

+      XPR(NT "%s: %s: %s\n", msg, ifile->filename, xd3_strerror (ret));

+    }

+  else

+    {

+      if (option_verbose > 2) { XPR(NT "main read: %s: %u\n", ifile->filename, (*nread)); }

+      ifile->nread += (*nread);

+    }

+

+  return ret;

+}

+

+static int

+main_file_write (main_file *ofile, uint8_t *buf, usize_t size, const char *msg)

+{

+  int ret = 0;

+

+#if XD3_STDIO

+  usize_t result;

+

+  result = fwrite (buf, 1, size, ofile->file);

+

+  if (result != size) { ret = get_errno (); }

+

+#elif XD3_POSIX

+  ret = xd3_posix_io (ofile->file, buf, size, (xd3_posix_func*) &write, NULL);

+#endif

+

+  if (ret)

+    {

+      XPR(NT "%s: %s: %s\n", msg, ofile->filename, xd3_strerror (ret));

+    }

+  else

+    {

+      if (option_verbose > 2) { XPR(NT "main write: %s: %u\n", ofile->filename, size); }

+      ofile->nwrite += size;

+    }

+

+  return ret;

+}

+

+static int

+main_file_seek (main_file *xfile, xoff_t pos)

+{

+  int ret = 0;

+

+#if XD3_STDIO

+  if (fseek (xfile->file, pos, SEEK_SET) != 0) { ret = get_errno (); }

+#else

+  if (lseek (xfile->file, pos, SEEK_SET) != pos) { ret = get_errno (); }

+#endif

+

+  if (ret)

+    {

+      XPR(NT "seek failed: %s: %s\n", xfile->filename, xd3_strerror (ret));

+    }

+

+  return ret;

+}

+

+/******************************************************************************************

+ VCDIFF TOOLS

+ ******************************************************************************************/

+

+#if VCDIFF_TOOLS

+/* This function prints a single VCDIFF window, mainly for debugging purposes. */

+static int

+main_print_window (xd3_stream* stream, FILE *vcout)

+{

+  int ret;

+  usize_t size = 0;

+

+  VC(UT "  Offset Code Type1 Size1 @Addr1 + Type2 Size2 @Addr2\n");

+

+  while (stream->inst_sect.buf < stream->inst_sect.buf_max)

+    {

+      uint   code = stream->inst_sect.buf[0];

+

+      if ((ret = xd3_decode_instruction (stream))) { return ret; }

+

+      VC(UT "  %06"Q"u %03u  %s %3u", stream->dec_winstart + size, code,

+               xd3_rtype_to_string (stream->dec_current1.type, option_print_cpymode),

+               stream->dec_current1.size);

+

+      if (stream->dec_current1.type != XD3_NOOP)

+        {

+          size += stream->dec_current1.size;

+          if (stream->dec_current1.type >= XD3_CPY)

+            {

+              VC(UT " @%-6u", stream->dec_current1.addr);

+            }

+          else

+            {

+              VC(UT "        ");

+            }

+        }

+

+      if (stream->dec_current2.type != XD3_NOOP)

+        {

+          size += stream->dec_current2.size;

+          VC(UT "  %s %3u",

+                   xd3_rtype_to_string (stream->dec_current2.type, option_print_cpymode),

+                   stream->dec_current2.size);

+

+          if (stream->dec_current2.type >= XD3_CPY)

+            {

+              VC(UT " @%-6u", stream->dec_current2.addr);

+            }

+        }

+

+      VC(UT "\n");

+    }

+

+  if (stream->dec_tgtlen != size && (stream->flags & XD3_SKIP_WINDOW) == 0)

+    {

+      XPR(NT "target window size inconsistency");

+      return XD3_INTERNAL;

+    }

+

+  if (stream->dec_position != stream->dec_maxpos)

+    {

+      XPR(NT "target window position inconsistency");

+      return XD3_INTERNAL;

+    }

+

+  if (stream->addr_sect.buf != stream->addr_sect.buf_max)

+    {

+      XPR(NT "address section inconsistency");

+      return XD3_INTERNAL;

+    }

+

+  IF_DEBUG (VC(UT "SIZE=%u  TGTLEN=%u\n", size, stream->dec_tgtlen));

+

+  return 0;

+}

+

+static void

+main_print_vcdiff_file (main_file *file, const char *type, FILE *vcout)

+{

+  if (file->filename)   { VC(UT "XDELTA filename (%s):     %s\n", type, file->filename); }

+  if (file->compressor) { VC(UT "XDELTA ext comp (%s):     %s\n", type, file->compressor->recomp_cmdname); }  

+}

+

+/* This function prints a VCDIFF input, mainly for debugging purposes. */

+static int

+main_print_func (xd3_stream* stream, main_file *xfile)

+{

+  int ret;

+  FILE *vcout;

+#if XD3_POSIX

+  if (! (vcout = fdopen (dup(xfile->file), "w")))

+    {

+      ret = get_errno ();

+      XPR(NT "fdopen: %s: %s\n", xfile->filename, xd3_strerror (ret));

+      return ret;

+    }

+#elif XD3_STDIO

+  vcout = xfile->file;

+#endif

+  XD3_ASSERT (vcout);

+  if (stream->dec_winstart == 0)

+    {

+      VC(UT "VCDIFF version:               0\n");

+

+      VC(UT "VCDIFF header size:           %d\n", stream->dec_hdrsize);

+      VC(UT "VCDIFF header indicator:      ");

+      if ((stream->dec_hdr_ind & VCD_SECONDARY) != 0) VC(UT "VCD_SECONDARY ");

+      if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) VC(UT "VCD_CODETABLE ");

+      if ((stream->dec_hdr_ind & VCD_APPHEADER) != 0) VC(UT "VCD_APPHEADER ");

+      if (stream->dec_hdr_ind == 0) VC(UT "none");

+      VC(UT "\n");

+

+      IF_SEC(VC(UT "VCDIFF secondary compressor:  %s\n", stream->sec_type ? stream->sec_type->name : "none"));

+      IF_NSEC(VC(UT "VCDIFF secondary compressor: unsupported\n"));

+

+      if (stream->dec_hdr_ind & VCD_APPHEADER)

+        {

+          uint8_t *apphead;

+          usize_t appheadsz;

+          ret = xd3_get_appheader (stream, & apphead, & appheadsz);

+

+          if (ret == 0 && appheadsz > 0)

+            {

+              int sq = option_quiet;

+              main_file i, o, s;

+              XD3_ASSERT (apphead != NULL);

+              VC(UT "VCDIFF application header:    ");

+              fwrite (apphead, 1, appheadsz, vcout);

+              VC(UT "\n");

+

+              main_file_init (& i);

+              main_file_init (& o);

+              main_file_init (& s);

+              option_quiet = 1;

+              main_get_appheader (stream, &i, & o, & s);

+              option_quiet = sq;

+              main_print_vcdiff_file (& o, "output", vcout);

+              main_print_vcdiff_file (& s, "source", vcout);

+              main_file_cleanup (& i);

+              main_file_cleanup (& o);

+              main_file_cleanup (& s);

+            }

+        }

+    }

+  else

+    {

+      VC(UT "\n");

+    }

+

+  VC(UT "VCDIFF window number:         %"Q"u\n", stream->current_window);

+  VC(UT "VCDIFF window indicator:      ");

+  if ((stream->dec_win_ind & VCD_SOURCE) != 0) VC(UT "VCD_SOURCE ");

+  if ((stream->dec_win_ind & VCD_TARGET) != 0) VC(UT "VCD_TARGET ");

+  if ((stream->dec_win_ind & VCD_ADLER32) != 0) VC(UT "VCD_ADLER32 ");

+  if (stream->dec_win_ind == 0) VC(UT "none");

+  VC(UT "\n");

+

+  if ((stream->dec_win_ind & VCD_ADLER32) != 0)

+    {

+      VC(UT "VCDIFF adler32 checksum:      %08X\n", stream->dec_adler32);

+    }

+

+  if (stream->dec_del_ind != 0)

+    {

+      VC(UT "VCDIFF delta indicator:       ");

+      if ((stream->dec_del_ind & VCD_DATACOMP) != 0) VC(UT "VCD_DATACOMP ");

+      if ((stream->dec_del_ind & VCD_INSTCOMP) != 0) VC(UT "VCD_INSTCOMP ");

+      if ((stream->dec_del_ind & VCD_ADDRCOMP) != 0) VC(UT "VCD_ADDRCOMP ");

+      if (stream->dec_del_ind == 0) VC(UT "none");

+      VC(UT "\n");

+    }

+

+  if (stream->dec_winstart != 0)

+    {

+      VC(UT "VCDIFF window at offset:      %"Q"u\n", stream->dec_winstart);

+    }

+

+  if (SRCORTGT (stream->dec_win_ind))

+    {

+      VC(UT "VCDIFF copy window length:    %u\n", stream->dec_cpylen);

+      VC(UT "VCDIFF copy window offset:    %"Q"u\n", stream->dec_cpyoff);

+    }

+

+  VC(UT "VCDIFF delta encoding length: %u\n", stream->dec_enclen);

+  VC(UT "VCDIFF target window length:  %u\n", stream->dec_tgtlen);

+

+  VC(UT "VCDIFF data section length:   %u\n", stream->data_sect.size);

+  VC(UT "VCDIFF inst section length:   %u\n", stream->inst_sect.size);

+  VC(UT "VCDIFF addr section length:   %u\n", stream->addr_sect.size);

+

+  ret = 0; 

+  if ((stream->flags & XD3_JUST_HDR) != 0)

+    {

+      /* Print a header -- finished! */

+      ret = PRINTHDR_SPECIAL;

+    }

+  else if ((stream->flags & XD3_SKIP_WINDOW) == 0)

+    {

+      ret = main_print_window (stream, vcout);

+    }

+

+  fclose (vcout);

+  return ret;

+}

+#endif /* VCDIFF_TOOLS */

+

+/******************************************************************************************

+ Input decompression, output recompression

+ ******************************************************************************************/

+

+#if EXTERNAL_COMPRESSION

+/* This is tricky POSIX-specific code with lots of fork(), pipe(), dup(), waitpid(), and

+ * exec() business.  Most of this code originated in PRCS1, which did automatic

+ * package-file decompression.  It works with both XD3_POSIX and XD3_STDIO file

+ * disciplines.

+ *

+ * To automatically detect compressed inputs requires a child process to reconstruct the

+ * input stream, which was advanced in order to detect compression, because it may not be

+ * seekable.  In other words, the main program reads part of the input stream, and if it

+ * detects a compressed input it then forks a pipe copier process, which copies the

+ * first-read block out of the main-program's memory, then streams the remaining

+ * compressed input into the input-decompression pipe.

+ */

+

+#include <unistd.h>

+#include <sys/stat.h>

+#include <sys/wait.h>

+

+/* Remember which pipe FD is which. */

+#define PIPE_READ_FD  0

+#define PIPE_WRITE_FD 1

+

+static pid_t ext_subprocs[2];

+static char* ext_tmpfile = NULL;

+

+/* Like write(), but makes repeated calls to empty the buffer. */

+static int

+main_pipe_write (int outfd, const uint8_t *exist_buf, usize_t remain)

+{

+  int ret;

+

+  if ((ret = xd3_posix_io (outfd, (uint8_t*) exist_buf, remain, (xd3_posix_func*) &write, NULL)))

+    {

+      XPR(NT "pipe write failed: %s", xd3_strerror (ret));

+      return ret;

+    }

+

+  return 0;

+}

+

+/* A simple error-reporting waitpid interface. */

+static int

+main_waitpid_check(pid_t pid)

+{

+  int status;

+  int ret = 0;

+

+  if (waitpid (pid, & status, 0) < 0)

+    {

+      ret = get_errno ();

+      XPR(NT "compression subprocess: wait: %s\n", xd3_strerror (ret));

+    }

+  else if (! WIFEXITED (status))

+    {

+      ret = ECHILD;

+      XPR(NT "compression subprocess: signal %d\n",

+         WIFSIGNALED (status) ? WTERMSIG (status) : WSTOPSIG (status));

+    }

+  else if (WEXITSTATUS (status) != 0)

+    {

+      ret = ECHILD;

+      XPR(NT "compression subprocess: exit %d\n", WEXITSTATUS (status));

+    }

+

+  return ret;

+}

+

+/* Wait for any existing child processes to check for abnormal exit. */

+static int

+main_external_compression_finish (void)

+{

+  int i;

+  int ret;

+

+  for (i = 0; i < 2; i += 1)

+    {

+      if (! ext_subprocs[i]) { continue; }

+

+      if ((ret = main_waitpid_check (ext_subprocs[i])))

+        {

+          return ret;

+        }

+    }

+

+  return 0;

+}

+

+/* This runs as a forked process of main_input_decompress_setup() to copy input to the

+ * decompression process.  First, the available input is copied out of the existing

+ * buffer, then the buffer is reused to continue reading from the compressed input

+ * file. */

+static int

+main_pipe_copier (uint8_t    *pipe_buf,

+                  usize_t      pipe_bufsize,

+                  usize_t      nread,

+                  main_file   *ifile,

+                  int         outfd)

+{

+  int ret;

+

+  for (;;)

+    {

+      if (nread > 0 && (ret = main_pipe_write (outfd, pipe_buf, nread)))

+        {

+          return ret;

+        }

+

+      if (nread < pipe_bufsize)

+        {

+          break;

+        }

+

+      if ((ret = main_file_read (ifile, pipe_buf, pipe_bufsize, & nread, "pipe read failed")) < 0)

+        {

+          return ret;

+        }

+    }

+

+  return 0;

+}

+

+/* This function is called after we have read some amount of data from the input file and

+ * detected a compressed input.  Here we start a decompression subprocess by forking

+ * twice.  The first process runs the decompression command, the second process copies

+ * data to the input of the first. */

+static int

+main_input_decompress_setup (const main_extcomp     *decomp,

+                             main_file              *ifile,

+                             uint8_t               *input_buf,

+                             usize_t                 input_bufsize,

+                             uint8_t               *pipe_buf,

+                             usize_t                 pipe_bufsize,

+                             usize_t                 pipe_avail,

+                             usize_t                *nread)

+{

+  int outpipefd[2], inpipefd[2];  /* The two pipes: input and output file descriptors. */

+  int input_fd = -1;              /* The resulting input_fd (output of decompression). */

+  pid_t decomp_id, copier_id;     /* The two subprocs. */

+  int ret;

+

+  outpipefd[0] = outpipefd[1] = -1;

+  inpipefd[0]  = inpipefd[1]  = -1;

+

+  if (pipe (outpipefd) || pipe (inpipefd))

+    {

+      XPR(NT "pipe failed: %s\n", xd3_strerror (ret = get_errno ()));

+      goto pipe_cleanup;

+    }

+

+  if ((decomp_id = fork ()) < 0)

+    {

+      XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ()));

+      goto pipe_cleanup;

+    }

+

+  /* The first child runs the decompression process: */

+  if (decomp_id == 0)

+    {

+      /* Setup pipes: write to the outpipe, read from the inpipe. */

+      if (dup2 (outpipefd[PIPE_WRITE_FD], STDOUT_FILENO) < 0 ||

+          dup2 (inpipefd[PIPE_READ_FD], STDIN_FILENO) < 0 ||

+          close (outpipefd[PIPE_READ_FD]) ||

+          close (outpipefd[PIPE_WRITE_FD]) ||

+          close (inpipefd[PIPE_READ_FD]) ||

+          close (inpipefd[PIPE_WRITE_FD]) ||

+          execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, decomp->decomp_options, NULL))

+        {

+          XPR(NT "child process %s failed to execute: %s\n", decomp->decomp_cmdname, xd3_strerror (get_errno ()));

+        }

+

+      _exit (127);

+    }

+

+  ext_subprocs[0] = decomp_id;

+

+  if ((copier_id = fork ()) < 0)

+    {

+      XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ()));

+      goto pipe_cleanup;

+    }

+

+  /* The second child runs the copier process: */

+  if (copier_id == 0)

+    {

+      int exitval = 0;

+

+      if (close (inpipefd[PIPE_READ_FD]) ||

+          main_pipe_copier (pipe_buf, pipe_bufsize, pipe_avail, ifile, inpipefd[PIPE_WRITE_FD]) ||

+          close (inpipefd[PIPE_WRITE_FD]))

+        {

+          XPR(NT "child copier process failed: %s\n", xd3_strerror (get_errno ()));

+          exitval = 1;

+        }

+

+      _exit (exitval);

+    }

+

+  ext_subprocs[1] = copier_id;

+

+  /* The parent closes both pipes after duplicating the output of compression. */

+  input_fd = dup (outpipefd[PIPE_READ_FD]);

+

+  if (input_fd < 0 ||

+      main_file_close (ifile) ||

+      close (outpipefd[PIPE_READ_FD]) ||

+      close (outpipefd[PIPE_WRITE_FD]) ||

+      close (inpipefd[PIPE_READ_FD]) ||

+      close (inpipefd[PIPE_WRITE_FD]))

+    {

+      XPR(NT "dup/close failed: %s\n", xd3_strerror (ret = get_errno ()));

+      goto pipe_cleanup;

+    }

+

+#if XD3_STDIO

+  /* Note: fdopen() acquires the fd, closes it when finished. */

+  if ((ifile->file = fdopen (input_fd, "r")) == NULL)

+    {

+      XPR(NT "fdopen failed: %s\n", xd3_strerror (ret = get_errno ()));

+      goto pipe_cleanup;

+    }

+

+#elif XD3_POSIX

+  ifile->file = input_fd;

+#endif

+

+  ifile->compressor = decomp;

+

+  /* Now the input file is decompressed. */

+  return main_file_read (ifile, input_buf, input_bufsize, nread, "input decompression failed");

+

+ pipe_cleanup:

+  close (input_fd);

+  close (outpipefd[PIPE_READ_FD]);

+  close (outpipefd[PIPE_WRITE_FD]);

+  close (inpipefd[PIPE_READ_FD]);

+  close (inpipefd[PIPE_WRITE_FD]);

+  return ret;

+}

+

+

+/* This routine is called when the first buffer of input data is read by the main program

+ * (unless input decompression is disabled by command-line option).  If it recognizes the

+ * magic number of a known input type it invokes decompression.

+ *

+ * Skips decompression if the decompression type or the file type is RD_NONEXTERNAL.

+ *

+ * Behaves exactly like main_file_read, otherwise.

+ *

+ * This function uses a separate buffer to read the first small block of input.  If a

+ * compressed input is detected, the separate buffer is passed to the pipe copier.  This

+ * avoids using the same size buffer in both cases. */

+static int

+main_decompress_input_check (main_file   *ifile,

+                            uint8_t    *input_buf,

+                            usize_t      input_size,

+                            usize_t     *nread)

+{

+  int i;

+  int ret;

+  uint8_t check_buf[XD3_ALLOCSIZE];

+  usize_t  check_nread;

+

+  if ((ret = main_file_read (ifile, check_buf, min (input_size, XD3_ALLOCSIZE), & check_nread, "input read failed")))

+    {

+      return ret;

+    }

+

+  for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)

+    {

+      const main_extcomp *decomp = & extcomp_types[i];

+

+      if ((check_nread > decomp->magic_size) &&

+          /* The following expr skips decompression if we are trying to read a VCDIFF

+           * input and that is the magic number. */

+          !((decomp->flags & RD_NONEXTERNAL) && (ifile->flags & RD_NONEXTERNAL)) &&

+          memcmp (check_buf, decomp->magic, decomp->magic_size) == 0)

+        {

+          if (! option_quiet)

+            {

+              XPR(NT "%s | %s %s\n",

+                 ifile->filename,

+                 decomp->decomp_cmdname,

+                 decomp->decomp_options);

+            }

+

+          return main_input_decompress_setup (decomp, ifile,

+                                              input_buf, input_size,

+                                              check_buf, XD3_ALLOCSIZE,

+                                              check_nread, nread);

+        }

+    }

+

+  /* Now read the rest of the input block. */

+  (*nread) = 0;

+

+  if (check_nread == XD3_ALLOCSIZE)

+    {

+      ret = main_file_read (ifile, input_buf + XD3_ALLOCSIZE,

+                            input_size - XD3_ALLOCSIZE, nread,

+                            "input read failed");

+    }

+

+  memcpy (input_buf, check_buf, check_nread);

+

+  (*nread) += check_nread;

+

+  return 0;

+}

+

+/* This is called when the source file needs to be decompressed.  We fork/exec a

+ * decompression command with the proper input and output to a temporary file. */

+static int

+main_decompress_source (main_file *sfile, xd3_source *source)

+{

+  const main_extcomp *decomp = sfile->compressor;

+  pid_t decomp_id;  /* One subproc. */

+  int   input_fd  = -1;

+  int   output_fd = -1;

+  int   ret;

+  char *tmpname = NULL;

+  char *tmpdir  = getenv ("TMPDIR");

+  static const char tmpl[] = "/xd3src.XXXXXX";

+

+  /* Make a template for mkstmp() */

+  if (tmpdir == NULL) { tmpdir = "/tmp"; }

+  if ((tmpname = main_malloc (strlen (tmpdir) + sizeof (tmpl) + 1)) == NULL) { return ENOMEM; }

+  sprintf (tmpname, "%s%s", tmpdir, tmpl);

+

+  XD3_ASSERT (ext_tmpfile == NULL);

+  ext_tmpfile = tmpname;

+

+  /* Open the output FD. */

+  if ((output_fd = mkstemp (tmpname)) < 0)

+    {

+      XPR(NT "mkstemp failed: %s: %s", tmpname, xd3_strerror (ret = get_errno ()));

+      goto cleanup;

+    }

+

+  /* Copy the input FD, reset file position. */

+  XD3_ASSERT (main_file_isopen (sfile));

+#if XD3_STDIO

+  if ((input_fd = dup (fileno (sfile->file))) < 0)

+    {

+      XPR(NT "dup failed: %s", xd3_strerror (ret = get_errno ()));

+      goto cleanup;

+    }

+  main_file_close (sfile);

+  sfile->file = NULL;

+#elif XD3_POSIX

+  input_fd = sfile->file;

+  sfile->file = -1;

+#endif

+

+  if ((ret = lseek (input_fd, SEEK_SET, 0)) != 0)

+    {

+      XPR(NT "lseek failed: : %s", xd3_strerror (ret = get_errno ()));

+      goto cleanup;

+    }

+

+  if ((decomp_id = fork ()) < 0)

+    {

+      XPR(NT "fork failed: %s", xd3_strerror (ret = get_errno ()));

+      goto cleanup;

+    }

+

+  /* The child runs the decompression process: */

+  if (decomp_id == 0)

+    {

+      /* Setup pipes: write to the output file, read from the pipe. */

+      if (dup2 (input_fd, STDIN_FILENO) < 0 ||

+          dup2 (output_fd, STDOUT_FILENO) < 0 ||

+          execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, decomp->decomp_options, NULL))

+        {

+          XPR(NT "child process %s failed to execute: %s\n",

+                   decomp->decomp_cmdname, xd3_strerror (get_errno ()));

+        }

+

+      _exit (127);

+    }

+

+  close (input_fd);

+  close (output_fd);

+  input_fd  = -1;

+  output_fd = -1;

+

+  /* Then wait for completion. */

+  if ((ret = main_waitpid_check (decomp_id)))

+    {

+      goto cleanup;

+    }

+

+  /* Open/stat the decompressed source file. */

+  if ((ret = main_file_open (sfile, tmpname, XO_READ))) { goto cleanup; }

+  if ((ret = main_file_stat (sfile, & source->size, 1))) { goto cleanup; }

+  return 0;

+

+ cleanup:

+  close (input_fd);

+  close (output_fd);

+  if (tmpname) { free (tmpname); }

+  ext_tmpfile = NULL;

+  return ret;

+}

+

+/* Initiate re-compression of the output stream.  This is easier than input decompression

+ * because we know beforehand that the stream will be compressed, whereas the input has

+ * already been read when we decide it should be decompressed.  Thus, it only requires one

+ * subprocess and one pipe. */

+static int

+main_recompress_output (main_file *ofile)

+{

+  pid_t recomp_id;  /* One subproc. */

+  int   pipefd[2];  /* One pipe. */

+  int   output_fd = -1;

+  int   ret;

+  const main_extcomp *recomp = ofile->compressor;

+

+  pipefd[0] = pipefd[1] = -1;

+

+  if (pipe (pipefd))

+    {

+      XPR(NT "pipe failed: %s\n", xd3_strerror (ret = get_errno ()));

+      goto pipe_cleanup;

+    }

+

+  if ((recomp_id = fork ()) < 0)

+    {

+      XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ()));

+      goto pipe_cleanup;

+    }

+

+  /* The child runs the recompression process: */

+  if (recomp_id == 0)

+    {

+      /* Setup pipes: write to the output file, read from the pipe. */

+      if (dup2 (XFNO (ofile), STDOUT_FILENO) < 0 ||

+          dup2 (pipefd[PIPE_READ_FD], STDIN_FILENO) < 0 ||

+          close (pipefd[PIPE_READ_FD]) ||

+          close (pipefd[PIPE_WRITE_FD]) ||

+          execlp (recomp->recomp_cmdname, recomp->recomp_cmdname, recomp->recomp_options, NULL))

+        {

+          XPR(NT "child process %s failed to execute: %s\n", recomp->recomp_cmdname, xd3_strerror (get_errno ()));

+        }

+

+      _exit (127);

+    }

+

+  ext_subprocs[0] = recomp_id;

+

+  /* The parent closes both pipes after duplicating the output-fd for writing to the

+   * compression pipe. */

+  output_fd = dup (pipefd[PIPE_WRITE_FD]);

+

+  if (output_fd < 0 ||

+      main_file_close (ofile) ||

+      close (pipefd[PIPE_READ_FD]) ||

+      close (pipefd[PIPE_WRITE_FD]))

+    {

+      XPR(NT "close failed: %s\n", xd3_strerror (ret = get_errno ()));

+      goto pipe_cleanup;

+    }

+

+#if XD3_STDIO

+  /* Note: fdopen() acquires the fd, closes it when finished. */

+  if ((ofile->file = fdopen (output_fd, "w")) == NULL)

+    {

+      XPR(NT "fdopen failed: %s\n", xd3_strerror (ret = get_errno ()));

+      goto pipe_cleanup;

+    }

+

+#elif XD3_POSIX

+  ofile->file = output_fd;

+#endif

+

+  /* Now the output file will be compressed. */

+  return 0;

+

+ pipe_cleanup:

+  close (output_fd);

+  close (pipefd[PIPE_READ_FD]);

+  close (pipefd[PIPE_WRITE_FD]);

+  return ret;

+}

+#endif /* EXTERNAL_COMPRESSION */

+

+/* Identify the compressor that was used based on its ident string, which is passed in the

+ * application header. */

+static const main_extcomp*

+main_ident_compressor (const char *ident)

+{

+  int i;

+

+  for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)

+    {

+      if (strcmp (extcomp_types[i].ident, ident) == 0)

+        {

+          return & extcomp_types[i];

+        }

+    }

+

+  return NULL;

+}

+

+/* Return the main_extcomp record to use for this identifier, if possible. */

+static const main_extcomp*

+main_get_compressor (const char *ident)

+{

+  const main_extcomp *ext = main_ident_compressor (ident);

+

+  if (ext == NULL)

+    {

+      if (! option_quiet)

+        {

+          XPR(NT "warning: cannot recompress output: "

+                   "unrecognized external compression ID: %s\n", ident);

+        }

+      return NULL;

+    }

+  else if (! EXTERNAL_COMPRESSION)

+    {

+      if (! option_quiet)

+        {

+          XPR(NT "warning: external support not compiled: "

+                   "original input was compressed: %s\n", ext->recomp_cmdname);

+        }

+      return NULL;

+    }

+  else

+    {

+      return ext;

+    }

+}

+

+/******************************************************************************************

+ APPLICATION HEADER

+ ******************************************************************************************/

+

+#if XD3_ENCODER

+static const char*

+main_apphead_string (const char* x)

+{

+  const char *y;

+

+  if (x == NULL) { return ""; }

+

+  if (strcmp (x, "/dev/stdin") == 0 ||

+      strcmp (x, "/dev/stdout") == 0 ||

+      strcmp (x, "/dev/stderr") == 0) { return "-"; }

+

+  // TODO: this is not portable

+  return (y = strrchr (x, '/')) == NULL ? x : y + 1;

+}

+

+static int

+main_set_appheader (xd3_stream *stream, main_file *input, main_file *sfile)

+{

+  /* The user may disable the application header.  Once the appheader is set, this

+   * disables setting it again. */

+  if (appheader_used || ! option_use_appheader) { return 0; }

+

+  /* The user may specify the application header, otherwise format the default header. */

+  if (option_appheader)

+    {

+      appheader_used = option_appheader;

+    }

+  else

+    {

+      const char *iname;

+      const char *icomp;

+      const char *sname;

+      const char *scomp;

+      int len;

+

+      iname = main_apphead_string (input->filename);

+      icomp = (input->compressor == NULL) ? "" : input->compressor->ident;

+      len = strlen (iname) + strlen (icomp) + 2;

+

+      if (sfile->filename != NULL)

+        {

+          sname = main_apphead_string (sfile->filename);

+          scomp = (sfile->compressor == NULL) ? "" : sfile->compressor->ident;

+          len += strlen (sname) + strlen (scomp) + 2;

+        }

+      else

+        {

+          sname = scomp = "";

+        }

+

+      if ((appheader_used = main_malloc (len)) == NULL)

+        {

+          return ENOMEM;

+        }

+

+      if (sfile->filename == NULL)

+        {

+          sprintf ((char*)appheader_used, "%s/%s", iname, icomp);

+        }

+      else

+        {

+          sprintf ((char*)appheader_used, "%s/%s/%s/%s", iname, icomp, sname, scomp);

+        }

+    }

+

+  xd3_set_appheader (stream, appheader_used, strlen ((char*)appheader_used));

+

+  return 0;

+}

+#endif

+

+static void

+main_get_appheader_params (main_file *file, char **parsed, int output, const char *type,

+                           main_file *other)

+{

+  /* Set the filename if it was not specified.  If output, option_stdout (-c) overrides. */

+  if (file->filename == NULL && ! (output && option_stdout) && strcmp (parsed[0], "-") != 0)

+    {

+      file->filename = parsed[0];

+

+      if (other->filename != NULL) {

+        /* Take directory from the other file, if it has one. */

+        char *last_slash = strrchr(other->filename, '/');

+

+        if (last_slash != NULL) {

+          int dlen = last_slash - other->filename;

+          file->filename_copy = main_malloc(dlen + 2 + strlen(file->filename));

+

+          strncpy(file->filename_copy, other->filename, dlen);

+          file->filename_copy[dlen] = '/';

+          strcpy(file->filename_copy + dlen + 1, parsed[0]);

+

+          file->filename = file->filename_copy;

+        }

+      }

+

+      if (! option_quiet)

+        {

+          XPR(NT "using default %s filename: %s\n", type, file->filename);

+        }

+    }

+

+  /* Set the compressor, initiate de/recompression later. */

+  if (file->compressor == NULL && *parsed[1] != 0)

+    {

+      file->compressor = main_get_compressor (parsed[1]);

+    }

+}

+

+static void

+main_get_appheader (xd3_stream *stream, main_file *ifile, main_file *output, main_file *sfile)

+{

+  uint8_t *apphead;

+  usize_t appheadsz;

+  int ret;

+

+  /* The user may disable the application header.  Once the appheader is set, this

+   * disables setting it again. */

+  if (! option_use_appheader) { return; }

+

+  ret = xd3_get_appheader (stream, & apphead, & appheadsz);

+

+  /* Ignore failure, it only means we haven't received a header yet. */

+  if (ret != 0) { return; }

+

+  if (appheadsz > 0)

+    {

+      char *start = (char*)apphead;

+      char *slash;

+      int   place = 0;

+      char *parsed[4];

+

+      memset (parsed, 0, sizeof (parsed));

+

+      while ((slash = strchr (start, '/')) != NULL)

+        {

+          *slash = 0;

+          parsed[place++] = start;

+          start = slash + 1;

+        }

+

+      parsed[place++] = start;

+

+      /* First take the output parameters. */

+      if (place == 2 || place == 4)

+        {

+          main_get_appheader_params (output, parsed, 1, "output", ifile);

+        }

+

+      /* Then take the source parameters. */

+      if (place == 4)

+        {

+          main_get_appheader_params (sfile, parsed+2, 0, "source", ifile);

+        }

+    }

+

+  option_use_appheader = 0;

+  return;

+}

+

+/******************************************************************************************

+ Main I/O routines

+ ******************************************************************************************/

+

+/* This function acts like the above except it may also try to recognize a compressed

+ * input when the first buffer of data is read.  The EXTERNAL_COMPRESSION code is called

+ * to search for magic numbers. */

+static int

+main_read_primary_input (main_file   *ifile,

+                         uint8_t    *buf,

+                         usize_t      size,

+                         usize_t     *nread)

+{

+#if EXTERNAL_COMPRESSION

+  if (option_decompress_inputs && ifile->flags & RD_FIRST)

+    {

+      ifile->flags &= ~RD_FIRST;

+

+      return main_decompress_input_check (ifile, buf, size, nread);

+    }

+#endif

+

+  return main_file_read (ifile, buf, size, nread, "input read failed");

+}

+

+/* This function simply writes the stream output buffer, if there is any.  This is used

+ * for both encode and decode commands.  (The VCDIFF tools use main_print_func()). */

+static int

+main_write_output (xd3_stream* stream, main_file *ofile)

+{

+  int ret;

+

+  if (stream->avail_out > 0 && (ret = main_file_write (ofile, stream->next_out, stream->avail_out, "write failed")))

+    {

+      return ret;

+    }

+

+  return 0;

+}

+

+/* Open the main output file, sets a default file name, initiate recompression.  This

+ * function is expected to fprint any error messages. */

+static int

+main_open_output (xd3_stream *stream, main_file *ofile)

+{

+  int ret;

+

+  if (ofile->filename == NULL)

+    {

+      XSTDOUT_XF (ofile);

+

+      if (option_verbose > 1) { XPR(NT "using standard output: %s\n", ofile->filename); }

+    }

+  else

+    {

+      /* Stat the file to check for overwrite. */

+      if (option_force == 0 && main_file_exists (ofile))

+        {

+          XPR(NT "to overwrite output file specify -f: %s\n", ofile->filename);

+          return EEXIST;

+        }

+

+      if ((ret = main_file_open (ofile, ofile->filename, XO_WRITE)))

+        {

+          return ret;

+        }

+

+      if (option_verbose > 1) { XPR(NT "open output: %s\n", ofile->filename); }

+    }

+

+#if EXTERNAL_COMPRESSION

+  /* Do output recompression. */

+  if (ofile->compressor != NULL && option_recompress_outputs == 1)

+    {

+      if (! option_quiet)

+        {

+          XPR(NT "%s %s | %s\n",

+             ofile->compressor->recomp_cmdname,

+             ofile->compressor->recomp_options,

+             ofile->filename);

+        }

+

+      if ((ret = main_recompress_output (ofile)))

+        {

+          return ret;

+        }

+    }

+#endif

+

+  return 0;

+}

+

+/* This is called at different times for encoding and decoding.  The encoder calls it

+ * immediately, the decoder delays until the application header is received. */

+static int

+main_set_source (xd3_stream *stream, int cmd, main_file *sfile, xd3_source *source)

+{

+  int ret, i;

+

+  /* Open it, check for seekability, set required xd3_source fields. */

+  if (allow_fake_source)

+    {

+      sfile->mode = XO_READ;

+      sfile->realname = sfile->filename;

+      sfile->nread = 0;

+      source->size = UINT64_MAX;

+    }

+  else if ((ret = main_file_open (sfile, sfile->filename, XO_READ)) ||

+           (ret = main_file_stat (sfile, & source->size, 1)))

+    {

+      return ret;

+    }

+

+  source->name     = sfile->filename;

+  source->ioh      = sfile;

+  source->curblkno = (xoff_t) -1;

+  source->curblk   = NULL;

+

+  /* Source block LRU init. */

+  main_blklru_list_init (& lru_list);

+  main_blklru_list_init (& lru_free);

+

+  option_srcwinsz = min(source->size, (xoff_t) option_srcwinsz);

+

+  if (option_verbose > 1) { XPR(NT "source window size: %u\n", option_srcwinsz); }

+  if (option_verbose > 1) { XPR(NT "source block size: %u\n", source->blksize); }

+  if (option_verbose > 1) { XPR(NT "memory size: %u\n", option_memsize); }

+

+  lru_size = (option_srcwinsz / source->blksize);

+  lru_size = max(1, lru_size);

+

+  if ((lru = main_malloc (sizeof (main_blklru) * lru_size)) == NULL)

+    {

+      return ENOMEM;

+    }

+

+  for (i = 0; i < lru_size; i += 1)

+    {

+      lru[i].blkno = (xoff_t) -1;

+

+      if ((lru[i].blk = main_malloc (source->blksize)) == NULL)

+        {

+          return ENOMEM;

+        }

+

+      main_blklru_list_push_back (& lru_free, & lru[i]);

+    }

+

+#if EXTERNAL_COMPRESSION

+  if (option_decompress_inputs)

+    {

+      if (IS_ENCODE (cmd))

+        {

+          usize_t nread;

+

+          source->curblk = lru[0].blk;

+

+          /* If encoding, read the first block now to check for decompression. */

+          if ((ret = main_file_read (sfile, (uint8_t*) source->curblk, source->blksize, & nread, "source read failed")))

+            {

+              return ret;

+            }

+

+          /* Check known magic numbers. */

+          for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)

+            {

+              const main_extcomp *decomp = & extcomp_types[i];

+

+              if ((nread > decomp->magic_size) && memcmp (source->curblk, decomp->magic, decomp->magic_size) == 0)

+                {

+                  sfile->compressor = decomp;

+                  break;

+                }

+            }

+

+          /* If no decompression, the current buffer is now a valid source->curblock. */

+          if (sfile->compressor == NULL)

+            {

+              main_blklru_list_remove (& lru[0]);

+              main_blklru_list_push_back (& lru_list, & lru[0]);

+

+              lru[0].blkno     = 0;

+              source->curblkno = 0;

+              source->onblk    = nread;

+

+              if (option_verbose > 2)

+                {

+                  XPR(NT "source block 0 read (not compressed)\n");

+                }

+            }

+        }

+

+      /* In either the encoder or decoder, start decompression. */

+      if (sfile->compressor)

+        {

+          xoff_t osize = source->size;

+

+          if (osize > XD3_NODECOMPRESSSIZE)

+            {

+              XPR(NT "source file too large for external decompression: %s: %"Q"u\n",

+                       sfile->filename, osize);

+              return XD3_INTERNAL;

+            }

+

+          if ((ret = main_decompress_source (sfile, source)))

+            {

+              return ret;

+            }

+

+          if (! option_quiet)

+            {

+              char s1[32], s2[32];

+              XPR(NT "%s | %s %s => %s %.1f%% [ %s , %s ]\n",

+                 sfile->filename,

+                 sfile->compressor->decomp_cmdname,

+                 sfile->compressor->decomp_options,

+                 sfile->realname,

+                 100.0 * source->size / osize,

+                 main_format_bcnt (osize, s1),

+                 main_format_bcnt (source->size, s2));

+            }

+        }

+    }

+#endif

+

+  if (option_verbose > 1) { XPR(NT "source file: %s: %"Q"u bytes\n", sfile->realname, source->size); }

+

+  if ((ret = xd3_set_source (stream, source)))

+    {

+      XPR(NT XD3_LIB_ERRMSG (stream, ret));

+      return EXIT_FAILURE;

+    }

+

+  return 0;

+}

+

+/******************************************************************************************

+ Source routines

+ ******************************************************************************************/

+

+/* This is the callback for reading a block of source.  This function is blocking and it

+ * implements a small LRU.

+ *

+ * Note that it is possible for main_input() to handle getblk requests in a non-blocking

+ * manner.  If the callback is NULL then the caller of xd3_*_input() must handle the

+ * XD3_GETSRCBLK return value and fill the source in the same way.  See xd3_getblk for

+ * details.  To see an example of non-blocking getblk, see xdelta-test.h. */

+static int

+main_getblk_func (xd3_stream *stream,

+                  xd3_source *source,

+                  xoff_t      blkno)

+{

+  xoff_t      pos   = blkno * source->blksize;

+  main_file   *sfile = (main_file*) source->ioh;

+  main_blklru *blru  = NULL;

+  usize_t      onblk = xd3_bytes_on_srcblk (source, blkno);

+  usize_t      nread;

+  int         ret;

+  int         i;

+

+  if (allow_fake_source)

+    {

+      source->curblkno = blkno;

+      source->onblk    = onblk;

+      source->curblk   = lru[0].blk;

+      return 0;

+    }

+

+  if (do_not_lru)

+    {

+      /* Direct lookup assumes sequential scan w/o skipping blocks. */

+      int idx = blkno % lru_size;

+      if (lru[idx].blkno == blkno)

+        {

+          source->curblkno = blkno;

+          source->onblk    = onblk;

+          source->curblk   = lru[idx].blk;

+          lru_hits += 1;

+          return 0;

+        }

+

+      if (lru[idx].blkno != -1LL &&

+          lru[idx].blkno != blkno - lru_size)

+        {

+          return XD3_TOOFARBACK;

+        }

+    }

+  else

+    {

+      /* Sequential search through LRU. */

+      for (i = 0; i < lru_size; i += 1)

+        {

+          if (lru[i].blkno == blkno)

+            {

+              main_blklru_list_remove (& lru[i]);

+              main_blklru_list_push_back (& lru_list, & lru[i]);

+

+              source->curblkno = blkno;

+              source->onblk    = onblk;

+              source->curblk   = lru[i].blk;

+              lru_hits += 1;

+              return 0;

+            }

+        }

+    }

+

+  if (! main_blklru_list_empty (& lru_free))

+    {

+      blru = main_blklru_list_pop_front (& lru_free);

+    }

+  else if (! main_blklru_list_empty (& lru_list))

+    {

+      if (do_not_lru) {

+        blru = & lru[blkno % lru_size];

+        main_blklru_list_remove(blru);

+      } else {

+        blru = main_blklru_list_pop_front (& lru_list);

+      }

+      lru_misses += 1;

+    }

+

+  lru_filled += 1;

+

+  if ((ret = main_file_seek (sfile, pos)))

+    {

+      return ret;

+    }

+

+  if ((ret = main_file_read (sfile, (uint8_t*) blru->blk, source->blksize,

+                             & nread, "source read failed")))

+    {

+      return ret;

+    }

+

+  if (nread != onblk)

+    {

+      XPR(NT "source file size change: %s\n", sfile->filename);

+      return XD3_INTERNAL;

+    }

+

+  main_blklru_list_push_back (& lru_list, blru);

+

+  if (option_verbose > 2)

+    {

+      if (blru->blkno != -1LL)

+        {

+          XPR(NT "source block %"Q"u ejects %"Q"u (lru_hits=%u, lru_misses=%u, lru_filled=%u)\n",

+              blkno, blru->blkno, lru_hits, lru_misses, lru_filled);

+        }

+      else

+        {

+          XPR(NT "source block %"Q"u read (lru_hits=%u, lru_misses=%u, lru_filled=%u)\n",

+                                          blkno, lru_hits, lru_misses, lru_filled);

+        }

+    }

+

+  blru->blkno      = blkno;

+  source->curblk   = blru->blk;

+  source->curblkno = blkno;

+  source->onblk    = onblk;

+

+  return 0;

+}

+

+/******************************************************************************************

+ Main routines

+ ******************************************************************************************/

+

+/* This is a generic input function.  It calls the xd3_encode_input or xd3_decode_input

+ * functions and makes calls to the various input handling routines above, which

+ * coordinate external decompression.

+ */

+static int

+main_input (xd3_cmd     cmd,

+            main_file   *ifile,

+            main_file   *ofile,

+            main_file   *sfile)

+{

+  int        ret;

+  xd3_stream stream;

+  usize_t    nread;

+  int        stream_flags = 0;

+  xd3_config config;

+  xd3_source source;

+  xoff_t     last_total_in = 0;

+  xoff_t     last_total_out = 0;

+  long       start_time;

+  xoff_t     input_size = 0;

+

+  int (*input_func) (xd3_stream*);

+  int (*output_func) (xd3_stream*, main_file *);

+

+  memset (& source, 0, sizeof (source));

+  memset (& config, 0, sizeof (config));

+

+  config.alloc = main_alloc;

+  config.freef = main_free1;

+  config.sec_data.ngroups = 1;

+  config.sec_addr.ngroups = 1;

+  config.sec_inst.ngroups = 1;

+

+  /* main_input setup. */

+  switch ((int) cmd)

+    {

+#if VCDIFF_TOOLS

+           if (1) { case CMD_PRINTHDR:   stream_flags = XD3_JUST_HDR; }

+      else if (1) { case CMD_PRINTHDRS:  stream_flags = XD3_SKIP_WINDOW; }

+      else        { case CMD_PRINTDELTA: stream_flags = XD3_SKIP_EMIT; }

+      ifile->flags |= RD_NONEXTERNAL;

+      input_func    = xd3_decode_input;

+      output_func   = main_print_func;

+      stream_flags |= XD3_ADLER32_NOVER;

+      break;

+#endif

+#if XD3_ENCODER

+    case CMD_ENCODE:

+      input_func  = xd3_encode_input;

+      output_func = main_write_output;

+

+      if (option_use_checksum) { stream_flags |= XD3_ADLER32; }

+      if (option_use_secondary)

+        {

+          /* The default secondary compressor is DJW, if it's compiled, being used, etc. */

+          if (option_secondary == NULL)

+            {

+              if (SECONDARY_DJW) { stream_flags |= XD3_SEC_DJW; }

+            }

+          else

+            {

+              if (strcmp (option_secondary, "fgk") == 0 && SECONDARY_FGK)

+                {

+                  stream_flags |= XD3_SEC_FGK;

+                }

+              else if (strcmp (option_secondary, "djw") == 0 && SECONDARY_DJW)

+                {

+                  stream_flags |= XD3_SEC_DJW;

+                }

+              else

+                {

+                  XPR(NT "unrecognized secondary compressor type: %s\n", option_secondary);

+                  return EXIT_FAILURE;

+                }

+            }

+        }

+      if (option_no_compress)      { stream_flags |= XD3_NOCOMPRESS; }

+      if (option_use_altcodetable) { stream_flags |= XD3_ALT_CODE_TABLE; }

+      if (option_smatch_config)

+        {

+          char *s = option_smatch_config, *e;

+          int values[XD3_SOFTCFG_VARCNT];

+          int got;

+

+          config.smatch_cfg = XD3_SMATCH_SOFT;

+

+          for (got = 0; got < XD3_SOFTCFG_VARCNT; got += 1, s = e + 1)

+            {

+              values[got] = strtol (s, &e, 10);

+

+              if ((values[got] < 0) ||

+                  (e == s) ||

+                  (got < XD3_SOFTCFG_VARCNT-1 && *e == 0) ||

+                  (got == XD3_SOFTCFG_VARCNT-1 && *e != 0))

+                {

+                  XPR(NT "invalid string match specifier (-C)\n");

+                  return EXIT_FAILURE;

+                }

+            }

+

+          config.smatcher_soft.large_look    = values[0];

+          config.smatcher_soft.large_step    = values[1];

+          config.smatcher_soft.small_look    = values[2];

+          config.smatcher_soft.small_chain   = values[3];

+          config.smatcher_soft.small_lchain  = values[4];

+          config.smatcher_soft.ssmatch       = values[5];

+          config.smatcher_soft.try_lazy      = values[6];

+          config.smatcher_soft.max_lazy      = values[7];

+          config.smatcher_soft.long_enough   = values[8];

+          config.smatcher_soft.promote       = values[9];

+        }

+      else if (option_level < 5) { config.smatch_cfg = XD3_SMATCH_FAST; }

+      else                       { config.smatch_cfg = XD3_SMATCH_SLOW; }

+      break;

+#endif

+    case CMD_DECODE:

+      if (option_use_checksum == 0) { stream_flags |= XD3_ADLER32_NOVER; }

+      stream_flags  = 0;

+      ifile->flags |= RD_NONEXTERNAL;

+      input_func    = xd3_decode_input;

+      output_func   = main_write_output;

+      break;

+    default:

+      XPR(NT "internal error\n");

+      return EXIT_FAILURE;

+    }

+

+  start_time = get_millisecs_now ();

+

+  if (main_file_stat (ifile, & input_size, 0) == 0)

+    {

+      option_winsize = min (input_size, (xoff_t) option_winsize);

+    }

+

+  option_srcwinsz = max(option_srcwinsz, XD3_ALLOCSIZE);

+  option_winsize = max(option_winsize, XD3_ALLOCSIZE);

+

+  source.blksize = (option_srcwinsz / 32) & ~(XD3_ALLOCSIZE-1);

+  source.blksize = max(XD3_DEFAULT_WINSIZE, source.blksize);

+

+  config.srcwin_maxsz = option_srcwinsz;

+  config.winsize = option_winsize;

+  config.memsize = option_memsize;

+  config.getblk = main_getblk_func;

+  config.flags = stream_flags;

+

+  if (option_verbose > 1)

+    {

+      XPR(NT "input buffer size: %u\n", option_winsize);

+    }

+

+  if ((main_bdata = main_malloc (option_winsize)) == NULL)

+    {

+      return EXIT_FAILURE;

+    }

+

+  if ((ret = xd3_config_stream (& stream, & config)))

+    {

+      XPR(NT XD3_LIB_ERRMSG (& stream, ret));

+      return EXIT_FAILURE;

+    }

+

+  if (IS_ENCODE (cmd))

+    {

+      /* When encoding, open the source file, possibly decompress it.  The decoder delays

+       * this step until XD3_GOTHEADER. */

+      if (sfile->filename != NULL && (ret = main_set_source (& stream, cmd, sfile, & source)))

+        {

+          return EXIT_FAILURE;

+        }

+    }

+

+  /* This times each window. */

+  get_millisecs_since ();

+

+  /* Main input loop. */

+  do

+    {

+      xoff_t input_offset;

+      xoff_t input_remain;

+      usize_t try_read;

+

+      input_offset = ifile->nread;

+      /*XD3_ASSERT (input_offset <= option_last_offset);*/

+

+      input_remain = /*option_last_offset*/ XOFF_T_MAX - input_offset;

+

+      try_read = (usize_t) min ((xoff_t) config.winsize, input_remain);

+

+      if ((ret = main_read_primary_input (ifile, main_bdata, try_read, & nread)))

+        {

+          return EXIT_FAILURE;

+        }

+

+      /* If we've reached EOF tell the stream to flush. */

+      if (nread < try_read)

+        {

+          stream_flags |= XD3_FLUSH;

+          xd3_set_flags (& stream, stream_flags);

+        }

+

+#if XD3_ENCODER

+      /* After the first main_read_primary_input completes, we know all the information

+       * needed to encode the application header. */

+      if (cmd == CMD_ENCODE && (ret = main_set_appheader (& stream, ifile, sfile)))

+        {

+          return EXIT_FAILURE;

+        }

+#endif

+      xd3_avail_input (& stream, main_bdata, nread);

+

+      /* If we read zero bytes after encoding at least one window... */

+      if (nread == 0 && stream.current_window > 0) {

+        break;

+      }

+

+    again:

+      ret = input_func (& stream);

+      /*if (option_verbose > 1) { XPR(NT XD3_LIB_ERRMSG (& stream, ret)); }*/

+

+      switch (ret)

+        {

+        case XD3_INPUT:

+          continue;

+

+        case XD3_GOTHEADER:

+          {

+            XD3_ASSERT (stream.current_window == 0);

+

+            /* Need to process the appheader as soon as possible.  It may contain a

+             * suggested default filename/decompression routine for the ofile, and it may

+             * contain default/decompression routine for the sources. */

+            if (cmd == CMD_DECODE)

+              {

+                int have_src = sfile->filename != NULL;

+                int need_src = xd3_decoder_needs_source (& stream);

+                int recv_src;

+

+                /* May need to set the sfile->filename if none was given. */

+                main_get_appheader (& stream, ifile, ofile, sfile);

+

+                recv_src = sfile->filename != NULL;

+

+                /* Check if the user expected a source to be required although it was not. */

+                if (have_src && ! need_src && ! option_quiet)

+                  {

+                    XPR(NT "warning: output window %"Q"u does not copy source\n", stream.current_window);

+                  }

+

+                /* Check if we have no source name and need one. */

+                /* TODO: this doesn't fire due to cpyblocks_ calculation check */

+                if (need_src && ! recv_src)

+                  {

+                    XPR(NT "input requires a source file, use -s\n");

+                    return EXIT_FAILURE;

+                  }

+

+                /* Now open the source file. */

+                if (need_src && (ret = main_set_source (& stream, cmd, sfile, & source)))

+                  {

+                    return EXIT_FAILURE;

+                  }

+              }

+            else if (cmd == CMD_PRINTHDR ||

+                     cmd == CMD_PRINTHDRS ||

+                     cmd == CMD_PRINTDELTA)

+              {

+                if (xd3_decoder_needs_source (& stream) && sfile->filename == NULL)

+                  {

+                    allow_fake_source = 1; 

+                    sfile->filename = "<placeholder>";

+                    main_set_source (& stream, cmd, sfile, & source);

+                  }

+              }

+          }

+        /* FALLTHROUGH */

+        case XD3_WINSTART:

+          {

+            /* Set or unset XD3_SKIP_WINDOW. */

+            /*if (stream.current_window < option_first_window || stream.current_window > option_last_window)

+              { stream_flags |= XD3_SKIP_WINDOW; }

+            else

+              { stream_flags &= ~XD3_SKIP_WINDOW; }*/

+

+            xd3_set_flags (& stream, stream_flags);

+            goto again;

+          }

+

+        case XD3_OUTPUT:

+          {

+            if (option_no_output == 0/* &&

+                stream.current_window >= option_first_window &&

+                stream.current_window <= option_last_window*/)

+              {

+                /* Defer opening the output file until the stream produces its first

+                 * output for both encoder and decoder, this way we delay long enough for

+                 * the decoder to receive the application header.  (Or longer if there are

+                 * skipped windows, but I can't think of any reason not to delay open.) */

+                

+                if (! main_file_isopen (ofile) && (ret = main_open_output (& stream, ofile)) != 0)

+                  {

+                    return EXIT_FAILURE;

+                  }

+                if ((ret = output_func (& stream, ofile)) && (ret != PRINTHDR_SPECIAL))

+                  {

+                    return EXIT_FAILURE;

+                  }

+                if (ret == PRINTHDR_SPECIAL)

+                  {

+                    xd3_abort_stream (& stream);

+                    ret = EXIT_SUCCESS;

+                    goto done;

+                  }

+                ret = 0;

+              }

+

+            xd3_consume_output (& stream);

+            goto again;

+          }

+

+        case XD3_WINFINISH:

+          {

+            if (IS_ENCODE (cmd) || cmd == CMD_DECODE)

+              {

+                int used_source = xd3_encoder_used_source (& stream);

+

+                if (! option_quiet && IS_ENCODE (cmd) && main_file_isopen (sfile) && ! used_source)

+                  {

+                    XPR(NT "warning: input position %"Q"u no source copies\n",

+                        stream.current_window * option_winsize);

+                  }

+

+                if (option_verbose)

+                  {

+                    char rrateavg[32], wrateavg[32], tm[32];

+                    char rdb[32], wdb[32];

+                    char trdb[32], twdb[32];

+                    long millis = get_millisecs_since ();

+                    usize_t this_read = stream.total_in - last_total_in;

+                    usize_t this_write = stream.total_out - last_total_out;

+                    last_total_in = stream.total_in;

+                    last_total_out = stream.total_out;

+

+                    if (option_verbose > 1)

+                      {

+                        XPR(NT "%"Q"u: in %s (%s): out %s (%s): total in %s: out %s: %s\n",

+                            stream.current_window,

+                            main_format_bcnt (this_read, rdb),

+                            main_format_rate (this_read, millis, rrateavg),

+                            main_format_bcnt (this_write, wdb),

+                            main_format_rate (this_write, millis, wrateavg),

+                            main_format_bcnt (stream.total_in, trdb),

+                            main_format_bcnt (stream.total_out, twdb),

+                            main_format_millis (millis, tm));

+                      }

+                    else

+                      {

+                        XPR(NT "%"Q"u: in %s: out %s: total in %s: out %s: %s\n",

+                            stream.current_window,

+                            main_format_bcnt (this_read, rdb),

+                            main_format_bcnt (this_write, wdb),

+                            main_format_bcnt (stream.total_in, trdb),

+                            main_format_bcnt (stream.total_out, twdb),

+                            main_format_millis (millis, tm));

+                      }

+                  }

+              }

+            goto again;

+          }

+

+        default:

+          /* input_func() error */

+          XPR(NT XD3_LIB_ERRMSG (& stream, ret));

+          return EXIT_FAILURE;

+        }

+    }

+  while (nread == config.winsize);

+done:

+  /* Close the inputs. (ifile must be open, sfile may be open) */

+  main_file_close (ifile);

+  main_file_close (sfile);

+

+#if EXTERNAL_COMPRESSION

+  if ((ret = main_external_compression_finish ()))

+    {

+      XPR(NT "external compression commands failed\n");

+      return EXIT_FAILURE;

+    }

+#endif

+

+  /* If output file is not open yet because of delayed-open, it means we never encountered

+   * a window in the delta, but it could have had a VCDIFF header?  TODO: solve this

+   * elsewhere.  For now, it prints "nothing to output" below, but the check doesn't

+   * happen in case of option_no_output. */

+  if (! option_no_output)

+    {

+      if (! main_file_isopen (ofile))

+        {

+          XPR(NT "nothing to output: %s\n", ifile->filename);

+          return EXIT_FAILURE;

+        }

+

+      /* Have to close the output before calling main_external_compression_finish, or else it hangs. */

+      if (main_file_close (ofile) != 0)

+        {

+          return EXIT_FAILURE;

+        }

+    }

+

+  if ((ret = xd3_close_stream (& stream)))

+    {

+      XPR(NT XD3_LIB_ERRMSG (& stream, ret));

+      return EXIT_FAILURE;

+    }

+

+  xd3_free_stream (& stream);

+

+  if (option_verbose)

+    {

+      char tm[32];

+      long end_time = get_millisecs_now ();

+      XPR(NT "command finished in %s\n", main_format_millis (end_time - start_time, tm));

+      XPR(NT "input bytes:  %"Q"u\n",  ifile->nread);

+      XPR(NT "output bytes: %"Q"u\n", ofile->nwrite);

+    }

+

+  return EXIT_SUCCESS;

+}

+

+/* free memory before exit, reset single-use variables. */

+static void

+main_cleanup (void)

+{

+  int i;

+  

+  if (option_appheader) { appheader_used = NULL; }

+

+  main_free (appheader_used);

+  main_free (main_bdata);

+

+#if EXTERNAL_COMPRESSION

+  main_free (ext_tmpfile);

+#endif

+

+  for (i = 0; lru && i < lru_size; i += 1)

+    {

+      main_free (lru[i].blk);

+    }

+

+  main_free (lru);

+

+  lru_hits = 0;

+  lru_misses = 0;

+  lru_filled = 0;

+

+  XD3_ASSERT (main_mallocs == 0);

+}

+

+int

+#if PYTHON_MODULE

+xd3_main_cmdline (int argc, char **argv)

+#else

+main (int argc, char **argv)

+#endif

+{

+  xd3_cmd cmd;

+  main_file ifile;

+  main_file ofile;

+  main_file sfile;

+  static char *flags = "0123456789cdefhnqvDJNORTVs:B:C:E:F:L:O:P:M:W:A::S::";

+  int my_optind;

+  char *my_optarg;

+  char *my_optstr;

+  char *sfilename;

+  int orig_argc = argc;

+  char **orig_argv = argv;

+  int ret;

+

+  main_file_init (& ifile);

+  main_file_init (& ofile);

+  main_file_init (& sfile);

+

+ go:  /* Go. */

+  cmd = CMD_NONE;

+  sfilename = NULL;

+  my_optind = 1;

+  argv = orig_argv;

+  argc = orig_argc;

+  program_name = argv[0];

+  extcomp_types[0].recomp_cmdname = program_name;

+  extcomp_types[0].decomp_cmdname = program_name;

+ takearg:

+  my_optarg = NULL;

+  my_optstr = argv[my_optind];

+  /* This doesn't use getopt() because it makes trouble for -P & python which reenter

+   * main() and thus care about freeing all memory.  I never had much trust for getopt

+   * anyway, it's too opaque.  This implements a fairly standard non-long-option getopt

+   * with support for named operations (e.g., "xdelta3 [encode|decode|printhdr...] < in >

+   * out").  I'll probably add long options at some point. */

+  if (my_optstr)

+    {

+      if (*my_optstr == '-')    { my_optstr += 1; }

+      else if (cmd == CMD_NONE) { goto nonflag; }

+      else                      { my_optstr = NULL; }

+    }

+  while (my_optstr)

+    {

+      char *s;

+      my_optarg = NULL;

+      if ((ret = *my_optstr++) == 0) { my_optind += 1; goto takearg; }

+

+      /* Option handling: first check for one ':' following the option in flags, then

+       * check for two.  The syntax allows:

+       *

+       * 1. -Afoo                   defines optarg="foo"

+       * 2. -A foo                  defines optarg="foo"

+       * 3. -A ""                   defines optarg="" (allows optional empty-string)

+       * 4. -A [EOA or -moreargs]   error (mandatory case)

+       * 5. -A [EOA -moreargs]      defines optarg=NULL (optional case)

+       * 6. -A=foo                  defines optarg="foo"

+       * 7. -A=                     defines optarg="" (mandatory case)

+       * 8. -A=                     defines optarg=NULL (optional case)

+       *

+       * See tests in test_command_line_arguments().

+       */

+      s = strchr (flags, ret);

+      if (s && s[1] && s[1] == ':')

+        {

+          int eqcase = 0;

+          int option = s[2] && s[2] == ':';

+

+          /* Case 1, set optarg to the remaining characters. */

+          my_optarg = my_optstr;

+          my_optstr = "";

+

+          /* Case 2-5 */

+          if (*my_optarg == 0)

+            {

+              /* Condition 4-5 */

+              int have_arg = my_optind < (argc - 1) && *argv[my_optind+1] != '-';

+

+              if (! have_arg)

+                {

+                  if (! option)

+                  {

+                    /* Case 4 */

+                    XPR(NT "-%c: requires an argument\n", ret);

+                    ret = EXIT_FAILURE;

+                    goto cleanup;

+                  }

+                  /* Case 5. */

+                  my_optarg = NULL;

+                }

+              else

+                {

+                  /* Case 2-3. */

+                  my_optarg = argv[++my_optind];

+                }

+            }

+          /* Case 6-8. */

+          else if (*my_optarg == '=')

+            {

+              /* Remove the = in all cases. */

+              my_optarg += 1;

+              eqcase = 1;

+

+              if (option && *my_optarg == 0)

+                {

+                  /* Case 8. */

+                  my_optarg = NULL;

+                }

+            }

+        }

+

+      switch (ret)

+        {

+        /* case: if no '-' was found, maybe check for a command name. */

+        nonflag:

+               if (strcmp (my_optstr, "decode") == 0) { cmd = CMD_DECODE; }

+          else if (strcmp (my_optstr, "encode") == 0)

+            {

+#if XD3_ENCODER

+              cmd = CMD_ENCODE;

+#else

+              XPR(NT "encoder support not compiled\n");

+              return EXIT_FAILURE;

+#endif

+            }

+          else if (strcmp (my_optstr, "config") == 0) { cmd = CMD_CONFIG; }

+#if REGRESSION_TEST

+          else if (strcmp (my_optstr, "test") == 0) { cmd = CMD_TEST; }

+#endif

+#if VCDIFF_TOOLS

+          else if (strcmp (my_optstr, "printhdr") == 0) { cmd = CMD_PRINTHDR; }

+          else if (strcmp (my_optstr, "printhdrs") == 0) { cmd = CMD_PRINTHDRS; }

+          else if (strcmp (my_optstr, "printdelta") == 0) { cmd = CMD_PRINTDELTA; }

+#endif

+

+          /* If no option was found and still no command, let the default command be

+           * encode.  The remaining args are treated as filenames. */

+          if (cmd == CMD_NONE)

+            {

+              cmd = CMD_DEFAULT;

+              my_optstr = NULL;

+              break;

+            }

+          else

+            {

+              /* But if we find a command name, continue the getopt loop. */

+              my_optind += 1;

+              goto takearg;

+            }

+

+          /* gzip-like options */

+        case '0': case '1': case '2': case '3': case '4':

+        case '5': case '6': case '7': case '8': case '9':

+          option_level = ret - '0';

+          break;

+        case 'f': option_force = 1; break;

+        case 'v': option_verbose += 1; option_quiet = 0; break;

+        case 'q': option_quiet = 1; option_verbose = 0; break;

+        case 'c': option_stdout = 1; break;

+        case 'd':

+          if (cmd == CMD_NONE) { cmd = CMD_DECODE; }

+          else { ret = main_help (); goto exit; }

+          break;

+        case 'e':

+#if XD3_ENCODER

+          if (cmd == CMD_NONE) { cmd = CMD_ENCODE; }

+          else { ret = main_help (); goto exit; }

+          break;

+#else

+          XPR(NT "encoder support not compiled\n");

+          return EXIT_FAILURE;

+#endif    

+        case 'P':

+          /* only set profile count once, since... */

+          if (option_profile_cnt == 0)

+            {

+              if ((ret = main_atou(my_optarg, (usize_t*) & option_profile_cnt, 0, 'P'))) { goto exit; }

+

+              if (option_profile_cnt <= 0)

+                {

+                  ret = EXIT_SUCCESS;

+                  goto exit;

+                }

+            }

+          break;

+

+        case 'n': option_use_checksum = 0; break;

+        case 'N': option_no_compress = 1; break;

+        case 'T': option_use_altcodetable = 1; break;

+        case 'C': option_smatch_config = my_optarg; break;

+        case 'J': option_no_output = 1; break;

+        case 'O': option_xdelta1 = my_optarg; break;

+        case 'S': if (my_optarg == NULL) { option_use_secondary = 0; }

+                  else { option_use_secondary = 1; option_secondary = my_optarg; } break;

+        case 'A': if (my_optarg == NULL) { option_use_appheader = 0; }

+                  else { option_appheader = (uint8_t*) my_optarg; } break;

+        case 'B': if ((ret = main_atou (my_optarg, & option_srcwinsz, XD3_ALLOCSIZE, 'B')))

+          {

+            goto exit;

+          }

+          break;

+        case 'W': if ((ret = main_atou (my_optarg, & option_winsize, XD3_ALLOCSIZE, 'W')))

+          {

+            goto exit;

+          }

+          break;

+        case 'M': if ((ret = main_atou (my_optarg, & option_memsize, XD3_ALLOCSIZE, 'M')))

+          {

+            goto exit;

+          }

+          break;

+        case 'D':

+#if EXTERNAL_COMPRESSION == 0

+          if (! option_quiet)

+            {

+              XPR(NT "warning: -D option ignored, "

+                       "external compression support was not compiled\n");

+            }

+#else

+          option_decompress_inputs  = 0;

+#endif

+          break;

+        case 'R':

+#if EXTERNAL_COMPRESSION == 0

+          if (! option_quiet)

+            {

+              XPR(NT "warning: -R option ignored, "

+                       "external compression support was not compiled\n");

+            }

+#else

+          option_recompress_outputs = 0;

+#endif

+          break;

+        case 's':

+          if (sfilename != NULL)

+            {

+              XPR(NT "specify only one source file\n");

+              goto cleanup;

+            }

+

+          sfilename = my_optarg;

+          break;

+

+        case 'V':

+          ret = main_version (); goto exit;

+        default:

+          ret = main_help (); goto exit;

+        }

+    }

+

+  option_source_filename = sfilename;

+

+  /* In case there were no arguments, set the default command. */

+  if (cmd == CMD_NONE) { cmd = CMD_DEFAULT; }

+

+  argc -= my_optind;

+  argv += my_optind;

+

+  /* There may be up to two more arguments. */

+  if (argc > 2)

+    {

+      XPR(NT "too many filenames: %s ...\n", argv[2]);

+      ret = EXIT_FAILURE;

+      goto cleanup;

+    }

+

+  extcomp_types[1].recomp_cmdname = option_xdelta1;

+  extcomp_types[1].decomp_cmdname = option_xdelta1;

+  

+  if (option_verbose > 1)

+    {

+      int l = 1;

+      int i;

+      char buf[1024];

+      for (i = 0; i < orig_argc; i += 1)

+        {

+          l += strlen (orig_argv[i]) + 1;

+        }

+      buf[0] = 0;

+      for (i = 0; i < orig_argc; i += 1)

+        {

+          strcat (buf, orig_argv[i]);

+          strcat (buf, " ");

+        }

+      XPR(NT "command line: %s\n", buf);

+    }      

+

+  ifile.flags    = RD_FIRST;

+  sfile.flags    = RD_FIRST;

+  sfile.filename = option_source_filename;

+

+  /* The infile takes the next argument, if there is one.  But if not, infile is set to

+   * stdin. */

+  if (argc > 0)

+    {

+      ifile.filename = argv[0];

+

+      if ((ret = main_file_open (& ifile, ifile.filename, XO_READ)))

+        {

+          goto cleanup;   

+        }

+    }

+  else

+    {

+      XSTDIN_XF (& ifile);

+    }

+

+  /* The ofile takes the following argument, if there is one.  But if not, it is left NULL

+   * until the application header is processed.  It will be set in main_open_output. */

+  if (argc > 1)

+    {

+      /* Check for conflicting arguments. */

+      if (option_stdout && ! option_quiet)

+        {

+          XPR(NT "warning: -c option overrides output filename: %s\n", argv[1]);

+        }

+

+      if (! option_stdout) { ofile.filename = argv[1]; }

+    }

+

+  switch (cmd)

+    {

+    case CMD_PRINTHDR:

+    case CMD_PRINTHDRS:

+    case CMD_PRINTDELTA:

+#if XD3_ENCODER

+    case CMD_ENCODE:

+      if (cmd == CMD_ENCODE)

+        {

+          do_not_lru = 1;

+        }

+#endif

+    case CMD_DECODE:

+      ret = main_input (cmd, & ifile, & ofile, & sfile);

+      break;

+

+#if REGRESSION_TEST

+    case CMD_TEST:

+      ret = xd3_selftest ();

+      break;

+#endif

+

+    case CMD_CONFIG:

+      ret = main_config ();

+      break;

+

+    default:

+      ret = main_help ();

+      break;

+    }

+

+#if EXTERNAL_COMPRESSION

+  if (ext_tmpfile != NULL) { unlink (ext_tmpfile); }

+#endif

+

+  if (0)

+    {

+    cleanup:

+      ret = EXIT_FAILURE;

+    exit:

+      (void)0;

+    }

+

+  main_file_cleanup (& ifile);

+  main_file_cleanup (& ofile);

+  main_file_cleanup (& sfile);

+

+  main_cleanup ();

+

+  if (--option_profile_cnt > 0 && ret == EXIT_SUCCESS) { goto go; }

+

+  return ret;

+}

+

+static int

+main_help (void)

+{

+  /* TODO: update www/xdelta3-cmdline.html */ 

+  main_version ();

+  P(RINT "usage: xdelta3 [command/options] [input [output]]\n");

+  P(RINT "special command names:\n");

+  P(RINT "    config      prints xdelta3 configuration\n");

+  P(RINT "    decode      decompress the input\n");

+  P(RINT "    encode      compress the input%s\n", XD3_ENCODER ? "" : " [Not compiled]");

+#if REGRESSION_TEST

+  P(RINT "    test        run the builtin tests\n");

+#endif

+#if VCDIFF_TOOLS

+  P(RINT "special commands for VCDIFF inputs:\n");

+  P(RINT "    printdelta  print information about the entire delta\n");

+  P(RINT "    printhdr    print information about the first window\n");

+  P(RINT "    printhdrs   print information about all windows\n");

+#endif

+  P(RINT "standard options:\n");

+  P(RINT "   -0 .. -9     compression level\n");

+  P(RINT "   -c           use stdout\n");

+  P(RINT "   -d           decompress\n");

+  P(RINT "   -e           compress%s\n", XD3_ENCODER ? "" : " [Not compiled]");

+  P(RINT "   -f           force overwrite\n");

+  P(RINT "   -h           show help\n");

+  P(RINT "   -q           be quiet\n");

+  P(RINT "   -v           be verbose (max 2)\n");

+  P(RINT "   -V           show version\n");

+

+  P(RINT "memory options:\n");

+  P(RINT "   -B blksize   source file block size\n");

+  P(RINT "   -M memsize   memory budget for hash tables\n");

+  P(RINT "   -W winsize   input window buffer size\n");

+

+  P(RINT "compression options:\n");

+  P(RINT "   -s source    source file to copy from (if any)\n");

+  P(RINT "   -S [djw|fgk] enable/disable secondary compression\n");

+  P(RINT "   -N           disable small string-matching compression\n");

+  P(RINT "   -D           disable external decompression (encode/decode)\n");

+  P(RINT "   -R           disable external recompression (decode)\n");

+

+#if XD3_DEBUG > 0

+  P(RINT "developer options:\n");

+  P(RINT "   -A [apphead] disable/provide application header\n");

+  P(RINT "   -C           soft config (see xdelta3-cfgs.h)\n");

+  P(RINT "   -J           disable output (check/compute only)\n");

+  P(RINT "   -P           repeat count (for profiling)\n");

+  P(RINT "   -T           use alternate code table\n");

+  P(RINT "   -n           disable checksum (encode/decode)\n");

+#endif

+  return EXIT_FAILURE;

+}

diff --git a/xdelta3/xdelta3-test.h b/xdelta3/xdelta3-test.h
index 3c50bb6..30d8629 100755
--- a/xdelta3/xdelta3-test.h
+++ b/xdelta3/xdelta3-test.h
@@ -17,7 +17,11 @@
  */
 
 #include <math.h>
+
+#ifndef WIN32
 #include <sys/wait.h>
+#else
+#endif
 
 #define MSG_IS(x) (stream->msg != NULL && strcmp ((x), stream->msg) == 0)
 
@@ -30,7 +34,7 @@ static const double TEST_ADD_MAX      = 256;
 static const double TEST_ADD_RATIO    = 0.1;
 static const double TEST_EPSILON      = 0.5;
 
-static int TESTBUFSIZE = 1024 * 16;
+#define TESTBUFSIZE (1024 * 16)
 
 #define TESTFILESIZE (1024)
 
@@ -88,7 +92,7 @@ static int
 test_exponential_dist (usize_t mean, usize_t max)
 {
   double mean_d = mean;
-  double erand  = log (1.0 / drand48 ());
+  double erand  = log (1.0 / (rand () / (double)RAND_MAX));
   usize_t x = (usize_t) (mean_d * erand + 0.5);
 
   return min (x, max);
@@ -141,13 +145,13 @@ test_setup (void)
 static void
 test_unlink (char* file)
 {
+  char buf[TESTBUFSIZE];
   while (unlink (file) != 0)
     {
       if (errno == ENOENT)
-        {
-          break;
-        }
-      char buf[TESTBUFSIZE];
+            {
+              break;
+            }
       sprintf (buf, "rm -f %s", file);
       system (buf);
     }
@@ -170,8 +174,8 @@ test_cleanup (void)
 static int
 test_make_inputs (xd3_stream *stream, xoff_t *ss_out, xoff_t *ts_out)
 {
-  usize_t ts = (lrand48 () % TEST_FILE_MEAN) + TEST_FILE_MEAN;
-  usize_t ss = (lrand48 () % TEST_FILE_MEAN) + TEST_FILE_MEAN;
+  usize_t ts = (rand () % TEST_FILE_MEAN) + TEST_FILE_MEAN;
+  usize_t ss = (rand () % TEST_FILE_MEAN) + TEST_FILE_MEAN;
   uint8_t *buf = malloc (ts + ss), *sbuf = buf /*, *tbuf = buf + ss*/;
   usize_t sadd = 0, sadd_max = ss * TEST_ADD_RATIO;
   FILE  *tf /*, *sf*/;
@@ -199,10 +203,10 @@ test_make_inputs (xd3_stream *stream, xoff_t *ss_out, xoff_t *ts_out)
 
       next = min (left, next);
 
-      if (i > 0 && (next > add_left || drand48 () >= add_prob))
+      if (i > 0 && (next > add_left || (rand() / (double)RAND_MAX) >= add_prob))
         {
           /* Copy */
-          usize_t offset = lrand48 () % i;
+          usize_t offset = rand () % i;
 
           for (j = 0; j < next; j += 1)
             {
@@ -214,7 +218,7 @@ test_make_inputs (xd3_stream *stream, xoff_t *ss_out, xoff_t *ts_out)
           /* Add */
           for (j = 0; j < next; j += 1)
             {
-              sbuf[i++] = lrand48 ();
+              sbuf[i++] = rand ();
             }
         }
     }
@@ -559,7 +563,7 @@ test_address_cache (xd3_stream *stream, int unused)
 
   addrs[0] = 0;
 
-  srand48 (0x9f73f7fc);
+  srand (0x9f73f7fc);
 
   /* First pass: encode addresses */
   xd3_init_cache (& stream->acache);
@@ -571,9 +575,9 @@ test_address_cache (xd3_stream *stream, int unused)
       usize_t prev_i;
       usize_t nearby;
 
-      p         = drand48 ();
-      prev_i    = lrand48 () % offset;
-      nearby    = (lrand48 () % 256) % offset, 1;
+      p         = (rand () / (double)RAND_MAX);
+      prev_i    = rand () % offset;
+      nearby    = (rand () % 256) % offset, 1;
       nearby    = max (1U, nearby);
 
       if (p < 0.1)      { addr = addrs[offset-nearby]; }
@@ -984,7 +988,7 @@ sec_dist_func6 (xd3_stream *stream, xd3_output *data)
   int i, ret, x;
   for (i = 0; i < ALPHABET_SIZE*20; i += 1)
     {
-      x = lrand48 () % (ALPHABET_SIZE/2);
+      x = rand () % (ALPHABET_SIZE/2);
       if ((ret = xd3_emit_byte (stream, & data, x))) { return ret; }
     }
   return 0;
@@ -997,7 +1001,7 @@ sec_dist_func7 (xd3_stream *stream, xd3_output *data)
   int i, ret, x;
   for (i = 0; i < ALPHABET_SIZE*20; i += 1)
     {
-      x = lrand48 () % ALPHABET_SIZE;
+      x = rand () % ALPHABET_SIZE;
       if ((ret = xd3_emit_byte (stream, & data, x))) { return ret; }
     }
   return 0;
@@ -1141,7 +1145,7 @@ test_secondary (xd3_stream *stream, const xd3_sec_type *sec, int groups)
       P(RINT "\n...");
       for (test_i = 0; test_i < SIZEOF_ARRAY (sec_dists); test_i += 1)
         {
-          srand48 (0x84687674);
+          srand (0x84687674);
 
           in_head  = xd3_alloc_output (stream, NULL);
           out_head = xd3_alloc_output (stream, NULL);
@@ -1477,7 +1481,7 @@ test_command_line_arguments (xd3_stream *stream, int ignore)
   xoff_t dsize;
   double ratio;
 
-  srand48 (0x89162337);
+  srand (0x89162337);
 
   for (i = 0; i < pairs; i += 1)
     {
@@ -1617,7 +1621,7 @@ test_externally_compressed_io (xd3_stream *stream, int ignore)
   int i, ret;
   char buf[TESTBUFSIZE];
 
-  srand48 (0x91723913);
+  srand (0x91723913);
 
   if ((ret = test_make_inputs (stream, NULL, NULL))) { return ret; }
 
@@ -1664,7 +1668,7 @@ test_source_decompression (xd3_stream *stream, int ignore)
   char buf[TESTBUFSIZE];
   const main_extcomp *ext;
 
-  srand48 (0x9ff56acb);
+  srand (0x9ff56acb);
 
   test_setup ();
   if ((ret = test_make_inputs (stream, NULL, NULL))) { return ret; }
@@ -1838,7 +1842,7 @@ test_identical_behavior (xd3_stream *stream, int ignore)
   usize_t delpos = 0, recsize;
   xd3_config config;
 
-  for (i = 0; i < IDB_TGTSZ; i += 1) { buf[i] = lrand48 (); } 
+  for (i = 0; i < IDB_TGTSZ; i += 1) { buf[i] = rand (); } 
 
   stream->winsize = IDB_WINSZ;
 
@@ -2140,7 +2144,6 @@ static int
 test_source_cksum_offset (xd3_stream *stream, int ignore)
 {
   xd3_source source;
-  stream->src = &source;
 
   // Inputs are:
   struct {
@@ -2170,12 +2173,15 @@ test_source_cksum_offset (xd3_stream *stream, int ignore)
     { 0, 0, 0, 0, 0 },
   }, *test_ptr;
 
+  stream->src = &source;
+
   for (test_ptr = cksum_test; test_ptr->cpos; test_ptr++) {
+        xoff_t r;
     stream->srcwin_cksum_pos = test_ptr->cpos;
     stream->total_in = test_ptr->ipos;
     stream->src->size = test_ptr->size;
 
-    xoff_t r = xd3_source_cksum_offset(stream, test_ptr->input);
+    r = xd3_source_cksum_offset(stream, test_ptr->input);
     CHECK(r == test_ptr->output);
   }
   return 0;
diff --git a/xdelta3/xdelta3.c b/xdelta3/xdelta3.c
index 828643a..3177168 100755
--- a/xdelta3/xdelta3.c
+++ b/xdelta3/xdelta3.c
@@ -315,8 +315,12 @@
 #endif
 
 #if XD3_USE_LARGEFILE64          /* How does everyone else do this? */
+#ifndef WIN32
 #define Q "q"
 #else
+#define Q "I64"
+#endif
+#else
 #define Q
 #endif
 
diff --git a/xdelta3/xdelta3.h b/xdelta3/xdelta3.h
index 1c1b7aa..0375122 100755
--- a/xdelta3/xdelta3.h
+++ b/xdelta3/xdelta3.h
@@ -1,996 +1,1000 @@
-/* xdelta 3 - delta compression tools and library
- * Copyright (C) 2001, 2003, 2004, 2005, 2006.  Joshua P. MacDonald
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-/* Welcome to Xdelta.  If you want to know more about Xdelta, start by reading xdelta3.c.
- * If you are ready to use the API, continue reading here.  There are two interfaces --
- * xd3_encode_input and xd3_decode_input -- plus a dozen or so related calls.  This
- * interface is styled after Zlib. */
-
-#ifndef _XDELTA3_H_
-#define _XDELTA3_H_
-
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-
-/**********************************************************************/
-
-/* Default configured value of stream->winsize.  If the program supplies
- * xd3_encode_input() with data smaller than winsize the stream will
- * automatically buffer the input, otherwise the input buffer is used directly.
- */
-#ifndef XD3_DEFAULT_WINSIZE
-#define XD3_DEFAULT_WINSIZE (1U << 18)
-#endif
-
-/* This is a unit of how far to advance the checksum position in one go. */
-#ifndef XD3_DEFAULT_CKSUM_ADVANCE
-#define XD3_DEFAULT_CKSUM_ADVANCE (1U << 14)
-#endif
-
-/* Default total size of the source window used in xdelta3-main.h */
-#ifndef XD3_DEFAULT_SRCWINSZ
-#define XD3_DEFAULT_SRCWINSZ (1U << 23)
-#endif
-
-/* Default configured value of stream->memsize.  This dictates how much memory Xdelta will
- * use for string-matching data structures. */
-#ifndef XD3_DEFAULT_MEMSIZE
-#define XD3_DEFAULT_MEMSIZE (1U << 18)
-#endif
-
-/* When Xdelta requests a memory allocation for certain buffers, it rounds up to units of
- * at least this size.  The code assumes (and asserts) that this is a power-of-two. */
-#ifndef XD3_ALLOCSIZE
-#define XD3_ALLOCSIZE (1U<<14)
-#endif
-
-/* The XD3_HARDMAXWINSIZE parameter is a safety mechanism to protect decoders against
- * malicious files.  The decoder will never decode a window larger than this.  If the file
- * specifies VCD_TARGET the decoder may require two buffers of this size.  Rationale for
- * choosing 22-bits as a maximum: this means that in the worst case, any VCDIFF address
- * without a copy window will require 3 bytes to encode (7 bits per byte, HERE and SAME
- * modes making every address within half the window away. */
-#ifndef XD3_HARDMAXWINSIZE
-#define XD3_HARDMAXWINSIZE (1U<<23)
-#endif
-
-/* The XD3_NODECOMPRESSSIZE parameter tells the xdelta main routine not to try to
- * externally-decompress source inputs that are too large.  Since these files must be
- * seekable, they are decompressed to a temporary file location and the user may not wish
- * for this. */
-#ifndef XD3_NODECOMPRESSSIZE
-#define XD3_NODECOMPRESSSIZE (1U<<24)
-#endif
-
-/* The IOPT_SIZE value sets the size of a buffer used to batch overlapping copy
- * instructions before they are optimized by picking the best non-overlapping ranges.  The
- * larger this buffer, the longer a forced xd3_srcwin_setup() decision is held off. */
-#ifndef XD3_DEFAULT_IOPT_SIZE
-#define XD3_DEFAULT_IOPT_SIZE    128
-#endif
-
-/* The maximum distance backward to search for small matches */
-#ifndef XD3_DEFAULT_SPREVSZ
-#define XD3_DEFAULT_SPREVSZ (1U << 16)
-#endif
-
-/* Sizes and addresses within VCDIFF windows are represented as usize_t
- *
- * For source-file offsets and total file sizes, total input and output counts, the xoff_t
- * type is used.  The decoder and encoder generally check for overflow of the xoff_t size,
- * and this is tested at the 32bit boundary [xdelta3-test.h].
- */
-#ifndef _WIN32
-typedef unsigned int    usize_t;
-typedef u_int8_t        uint8_t;
-typedef u_int16_t       uint16_t;
-typedef u_int32_t       uint32_t;
-typedef u_int64_t       uint64_t;
-#else
-#include <windows.h>
-#define inline
-typedef unsigned int   uint;
-typedef unsigned int   usize_t;
-typedef unsigned char  uint8_t;
-typedef unsigned short uint16_t;
-typedef unsigned long  uint32_t;
-typedef ULONGLONG      uint64_t;
-#endif
-
-#define SIZEOF_USIZE_T 4
-
-#ifndef XD3_USE_LARGEFILE64
-#define XD3_USE_LARGEFILE64 1
-#endif
-
-#if XD3_USE_LARGEFILE64
-#define __USE_FILE_OFFSET64 1 /* GLIBC: for 64bit fileops, ... ? */
-typedef uint64_t xoff_t;
-#define SIZEOF_XOFF_T 8
-#else
-typedef uint32_t xoff_t;
-#define SIZEOF_XOFF_T 4
-#endif
-
-#define USE_UINT32 (SIZEOF_USIZE_T == 4 || SIZEOF_XOFF_T == 4 || REGRESSION_TEST)
-#define USE_UINT64 (SIZEOF_USIZE_T == 8 || SIZEOF_XOFF_T == 8 || REGRESSION_TEST)
-
-/**********************************************************************/
-
-#ifndef INLINE
-#define INLINE inline
-#endif
-
-/* Whether to build the encoder, otherwise only build the decoder. */
-#ifndef XD3_ENCODER
-#define XD3_ENCODER 1
-#endif
-
-/* The code returned when main() fails, also defined in system includes. */
-#ifndef EXIT_FAILURE
-#define EXIT_FAILURE 1
-#endif
-
-/* REGRESSION TEST enables the "xdelta3 test" command, which runs a series of self-tests. */
-#ifndef REGRESSION_TEST
-#define REGRESSION_TEST 0
-#endif
-
-/* XD3_DEBUG=1 enables assertions and various statistics.  Levels > 1 enable some
- * additional output only useful during development and debugging. */
-#ifndef XD3_DEBUG
-#define XD3_DEBUG 0
-#endif
-
-#ifndef PYTHON_MODULE
-#define PYTHON_MODULE 0
-#endif
-
-/* There are three string matching functions supplied: one fast, one slow (default), and
- * one soft-configurable.  To disable any of these, use the following definitions. */
-#ifndef XD3_BUILD_SLOW
-#define XD3_BUILD_SLOW 1
-#endif
-#ifndef XD3_BUILD_FAST
-#define XD3_BUILD_FAST 1
-#endif
-#ifndef XD3_BUILD_SOFT
-#define XD3_BUILD_SOFT 1
-#endif
-
-#if XD3_DEBUG
-#include <stdio.h>
-#endif
-
-/* XPRINT.  Debug output and VCDIFF_TOOLS functions report to stderr.  I have used an
- * irregular style to abbreviate [fprintf(stderr, "] as [P(RINT "]. */
-#define P    fprintf
-#define RINT stderr,
-
-typedef struct _xd3_stream             xd3_stream;
-typedef struct _xd3_source             xd3_source;
-typedef struct _xd3_hash_cfg           xd3_hash_cfg;
-typedef struct _xd3_smatcher           xd3_smatcher;
-typedef struct _xd3_rinst              xd3_rinst;
-typedef struct _xd3_dinst              xd3_dinst;
-typedef struct _xd3_hinst              xd3_hinst;
-typedef struct _xd3_rpage              xd3_rpage;
-typedef struct _xd3_addr_cache         xd3_addr_cache;
-typedef struct _xd3_output             xd3_output;
-typedef struct _xd3_desect             xd3_desect;
-typedef struct _xd3_iopt_buf           xd3_iopt_buf;
-typedef struct _xd3_rlist              xd3_rlist;
-typedef struct _xd3_sec_type           xd3_sec_type;
-typedef struct _xd3_sec_cfg            xd3_sec_cfg;
-typedef struct _xd3_sec_stream         xd3_sec_stream;
-typedef struct _xd3_config             xd3_config;
-typedef struct _xd3_code_table_desc    xd3_code_table_desc;
-typedef struct _xd3_code_table_sizes   xd3_code_table_sizes;
-typedef struct _xd3_slist              xd3_slist;
-
-/* The stream configuration has three callbacks functions, all of which may be supplied
- * with NULL values.  If config->getblk is provided as NULL, the stream returns
- * XD3_GETSRCBLK. */
-
-typedef void*  (xd3_alloc_func)    (void       *opaque,
-                                    usize_t      items,
-                                    usize_t      size);
-typedef void   (xd3_free_func)     (void       *opaque,
-                                    void       *address);
-
-typedef int    (xd3_getblk_func)   (xd3_stream *stream,
-                                    xd3_source *source,
-                                    xoff_t      blkno);
-
-/* These are internal functions to delay construction of encoding tables and support
- * alternate code tables.  See the comments & code enabled by GENERIC_ENCODE_TABLES. */
-
-typedef const xd3_dinst* (xd3_code_table_func) (void);
-typedef int              (xd3_comp_table_func) (xd3_stream *stream,
-                                                const uint8_t **data,
-                                                usize_t *size);
-
-
-/* Some junk. */
-
-#ifndef XD3_ASSERT
-#if XD3_DEBUG
-#define XD3_ASSERT(x) \
-    do { if (! (x)) { P(RINT "%s:%d: XD3 assertion failed: %s\n", __FILE__, __LINE__, #x); \
-    abort (); } } while (0)
-#else
-#define XD3_ASSERT(x) (void)0
-#endif
-#endif
-
-#ifdef __GNUC__
-/* As seen on linux-kernel. */
-#ifndef max
-#define max(x,y) ({ \
-        const typeof(x) _x = (x);       \
-        const typeof(y) _y = (y);       \
-        (void) (&_x == &_y);            \
-        _x > _y ? _x : _y; })
-#endif
-
-#ifndef min
-#define min(x,y) ({ \
-        const typeof(x) _x = (x);       \
-        const typeof(y) _y = (y);       \
-        (void) (&_x == &_y);            \
-        _x < _y ? _x : _y; })
-#endif
-#else
-#ifndef max
-#define max(x,y) ((x) < (y) ? (y) : (x))
-#endif
-#ifndef min
-#define min(x,y) ((x) < (y) ? (x) : (y))
-#endif
-#endif
-
-/******************************************************************************************
- PUBLIC ENUMS
- ******************************************************************************************/
-
-/* These are the five ordinary status codes returned by the xd3_encode_input() and
- * xd3_decode_input() state machines. */
-typedef enum {
-
-  /* An application must be prepared to handle these five return values from either
-   * xd3_encode_input or xd3_decode_input, except in the case of no-source compression, in
-   * which case XD3_GETSRCBLK is never returned.  More detailed comments for these are
-   * given in xd3_encode_input and xd3_decode_input comments, below. */
-  XD3_INPUT     = -17703, /* need input */
-  XD3_OUTPUT    = -17704, /* have output */
-  XD3_GETSRCBLK = -17705, /* need a block of source input (with no xd3_getblk function),
-                           * a chance to do non-blocking read. */
-  XD3_GOTHEADER = -17706, /* (decode-only) after the initial VCDIFF & first window header */
-  XD3_WINSTART  = -17707, /* notification: returned before a window is processed, giving a
-                           * chance to XD3_SKIP_WINDOW or not XD3_SKIP_EMIT that window. */
-  XD3_WINFINISH  = -17708, /* notification: returned after encode/decode & output for a window */
-  XD3_TOOFARBACK = -17709, /* (encoder only) may be returned by getblk() if the block is too old */
-  XD3_INTERNAL   = -17710, /* internal error */
-
-} xd3_rvalues;
-
-/* special values in config->flags */
-typedef enum
-{
-  XD3_JUST_HDR       = (1 << 1),   /* used by VCDIFF tools, see xdelta3-main.h. */
-  XD3_SKIP_WINDOW    = (1 << 2),   /* used by VCDIFF tools, see xdelta3-main.h. */
-  XD3_SKIP_EMIT      = (1 << 3),   /* used by VCDIFF tools, see xdelta3-main.h. */
-  XD3_FLUSH          = (1 << 4),   /* flush the stream buffer to prepare for xd3_stream_close(). */
-
-  XD3_SEC_DJW        = (1 << 5),   /* use DJW static huffman */
-  XD3_SEC_FGK        = (1 << 6),   /* use FGK adaptive huffman */
-  XD3_SEC_TYPE       = (XD3_SEC_DJW | XD3_SEC_FGK),
-
-  XD3_SEC_NODATA     = (1 << 7),   /* disable secondary compression of the data section. */
-  XD3_SEC_NOINST     = (1 << 8),   /* disable secondary compression of the inst section. */
-  XD3_SEC_NOADDR     = (1 << 9),   /* disable secondary compression of the addr section. */
-
-  XD3_SEC_OTHER      = (XD3_SEC_NODATA | XD3_SEC_NOINST | XD3_SEC_NOADDR),
-
-  XD3_ADLER32        = (1 << 10),  /* enable checksum computation in the encoder. */
-  XD3_ADLER32_NOVER  = (1 << 11),  /* disable checksum verification in the decoder. */
-
-  XD3_ALT_CODE_TABLE = (1 << 12),  /* for testing the alternate code table encoding. */
-
-  XD3_NOCOMPRESS     = (1 << 13),  /* disable ordinary data compression feature,
-                                    * only search the source, not the target. */
-  XD3_BEGREEDY       = (1 << 14),  /* disable the "1.5-pass algorithm", instead use
-                                    * greedy matching.  Greedy is off by default. */
-} xd3_flags;
-
-/* The values of this enumeration are set in xd3_config using the smatch_cfg variable.  It
- * can be set to slow, fast, soft, or default.  The fast and slow setting uses preset,
- * hardcoded parameters and the soft setting is accompanied by user-supplied parameters.
- * If the user supplies 'default' the code selects one of the available string matchers.
- * Due to compile-time settings (see XD3_SLOW_SMATCHER, XD3_FAST_SMATCHER,
- * XD3_SOFT_SMATCHER variables), not all options may be available. */
-typedef enum
-{
-  XD3_SMATCH_DEFAULT = 0,
-  XD3_SMATCH_SLOW    = 1,
-  XD3_SMATCH_FAST    = 2,
-  XD3_SMATCH_SOFT    = 3,
-} xd3_smatch_cfg;
-
-/******************************************************************************************
- PRIVATE ENUMS
- ******************************************************************************************/
-
-/* stream->match_state is part of the xd3_encode_input state machine for source matching:
- *
- *  1. the XD3_GETSRCBLK block-read mechanism means reentrant matching
- *  2. this state spans encoder windows: a match and end-of-window will continue in the next
- *  3. the initial target byte and source byte are a presumed match, to avoid some computation
- *  in case the inputs are identical.
- */
-typedef enum {
-
-  MATCH_TARGET    = 0, /* in this state, attempt to match the start of the target with the
-                        * previously set source address (initially 0). */
-  MATCH_BACKWARD  = 1, /* currently expanding a match backward in the source/target. */
-  MATCH_FORWARD   = 2, /* currently expanding a match forward in the source/target. */
-  MATCH_SEARCHING = 3, /* currently searching for a match. */
-
-} xd3_match_state;
-
-/* The xd3_encode_input state machine steps through these states in the following order.
- * The matcher is reentrant and returns XD3_INPUT whenever it requires more data.  After
- * receiving XD3_INPUT, if the application reads EOF it should call xd3_stream_close().
- */
-typedef enum {
-
-  ENC_INIT      = 0, /* xd3_encode_input has never been called. */
-  ENC_INPUT     = 1, /* waiting for xd3_avail_input () to be called. */
-  ENC_SEARCH    = 2, /* currently searching for matches. */
-  ENC_FLUSH     = 3, /* currently emitting output. */
-  ENC_POSTOUT   = 4, /* after an output section. */
-  ENC_POSTWIN   = 5, /* after all output sections. */
-  ENC_ABORTED   = 6, /* abort. */
-} xd3_encode_state;
-
-/* The xd3_decode_input state machine steps through these states in the following order.
- * The matcher is reentrant and returns XD3_INPUT whenever it requires more data.  After
- * receiving XD3_INPUT, if the application reads EOF it should call xd3_stream_close().
- *
- * 0-8:   the VCDIFF header
- * 9-18:  the VCDIFF window header
- * 19-21: the three primary sections: data (which I think should have gone last), inst, addr
- * 22:    producing output: returns XD3_OUTPUT, possibly XD3_GETSRCBLK,
- * 23:    return XD3_WINFINISH, set state=9 to decode more input
- */
-typedef enum {
-
-  DEC_VCHEAD   = 0, /* VCDIFF header */
-  DEC_HDRIND   = 1, /* header indicator */
-
-  DEC_SECONDID = 2, /* secondary compressor ID */
-
-  DEC_TABLEN   = 3, /* code table length */
-  DEC_NEAR     = 4, /* code table near */
-  DEC_SAME     = 5, /* code table same */
-  DEC_TABDAT   = 6, /* code table data */
-
-  DEC_APPLEN   = 7, /* application data length */
-  DEC_APPDAT   = 8, /* application data */
-
-  DEC_WININD   = 9, /* window indicator */
-
-  DEC_CPYLEN   = 10, /* copy window length */
-  DEC_CPYOFF   = 11, /* copy window offset */
-
-  DEC_ENCLEN   = 12, /* length of delta encoding */
-  DEC_TGTLEN   = 13, /* length of target window */
-  DEC_DELIND   = 14, /* delta indicator */
-
-  DEC_DATALEN  = 15, /* length of ADD+RUN data */
-  DEC_INSTLEN  = 16, /* length of instruction data */
-  DEC_ADDRLEN  = 17, /* length of address data */
-
-  DEC_CKSUM    = 18, /* window checksum */
-
-  DEC_DATA     = 19, /* data section */
-  DEC_INST     = 20, /* instruction section */
-  DEC_ADDR     = 21, /* address section */
-
-  DEC_EMIT     = 22, /* producing data */
-
-  DEC_FINISH   = 23, /* window finished */
-
-  DEC_ABORTED  = 24, /* xd3_abort_stream */
-} xd3_decode_state;
-
-/* An application never sees these internal codes: */  
-typedef enum {
-  XD3_NOSECOND  = -17708, /* when secondary compression finds no improvement. */
-} xd3_pvalues;
-
-/******************************************************************************************
- internal types
- ******************************************************************************************/
-
-/* instruction lists used in the IOPT buffer */
-struct _xd3_rlist
-{
-  xd3_rlist  *next;
-  xd3_rlist  *prev;
-};
-
-/* the raw encoding of an instruction used in the IOPT buffer */
-struct _xd3_rinst
-{
-  uint8_t     type;
-  uint8_t     xtra;
-  uint8_t     code1;
-  uint8_t     code2;
-  usize_t      pos;
-  usize_t      size;
-  xoff_t      addr;
-  xd3_rlist   link;
-};
-
-/* the code-table form of an single- or double-instruction */
-struct _xd3_dinst
-{
-  uint8_t     type1;
-  uint8_t     size1;
-  uint8_t     type2;
-  uint8_t     size2;
-};
-
-/* the decoded form of a single (half) instruction. */
-struct _xd3_hinst
-{
-  uint8_t     type;
-  usize_t      size;
-  usize_t      addr;
-};
-
-/* used by the encoder to buffer output in sections.  list of blocks. */
-struct _xd3_output
-{
-  uint8_t    *base;
-  usize_t      next;
-  usize_t      avail;
-  xd3_output *next_page;
-};
-
-/* the VCDIFF address cache, see the RFC */
-struct _xd3_addr_cache
-{
-  uint     s_near;
-  uint     s_same;
-  usize_t  next_slot;  /* the circular index for near */
-  usize_t *near_array; /* array of size s_near        */
-  usize_t *same_array; /* array of size s_same*256    */
-};
-
-/* the IOPT buffer has a used list of (ordered) instructions, possibly overlapping in
- * target addresses, awaiting a flush */
-struct _xd3_iopt_buf
-{
-  xd3_rlist  used;
-  xd3_rlist  free;
-  xd3_rinst *buffer;
-};
-
-/* This is the record of a pre-compiled configuration, a subset of xd3_config. */
-struct _xd3_smatcher
-{
-  const char        *name;
-  int             (*string_match) (xd3_stream  *stream);
-  uint               large_look;
-  uint               large_step;
-  uint               small_look;
-  uint               small_chain;
-  uint               small_lchain;
-  uint               ssmatch;
-  uint               try_lazy;
-  uint               max_lazy;
-  uint               long_enough;
-  uint               promote;
-};
-
-/* hash table size & power-of-two hash function. */
-struct _xd3_hash_cfg
-{
-  usize_t           size;
-  usize_t           shift;
-  usize_t           mask;
-};
-
-/* a hash-chain link in the small match table, embedded with position and checksum */
-struct _xd3_slist
-{
-  xd3_slist *next;
-  xd3_slist *prev;
-  usize_t     pos;
-  usize_t     scksum;
-};
-
-/* a decoder section (data, inst, or addr).  there is an optimization to avoid copying
- * these sections if all the input is available, related to the copied field below.
- * secondation compression uses the copied2 field. */
-struct _xd3_desect
-{
-  const uint8_t *buf;
-  const uint8_t *buf_max;
-  usize_t         size;
-  usize_t         pos;
-  uint8_t       *copied1;
-  usize_t         alloc1;
-  uint8_t       *copied2;
-  usize_t         alloc2;
-};
-
-/******************************************************************************************
- public types
- ******************************************************************************************/
-
-/* Settings for the secondary compressor. */
-struct _xd3_sec_cfg
-{
-  int                data_type;     /* Which section. (set automatically) */
-  int                ngroups;       /* Number of DJW Huffman groups. */
-  int                sector_size;   /* Sector size. */
-  int                inefficient;   /* If true, ignore efficiency check [avoid XD3_NOSECOND]. */
-};
-
-/* This is the user-visible stream configuration. */
-struct _xd3_config
-{
-  usize_t             memsize;       /* How much memory Xdelta may allocate */
-  usize_t             winsize;       /* The encoder window size. */
-  usize_t             sprevsz;       /* How far back small string matching goes */
-  usize_t             iopt_size;     /* entries in the instruction-optimizing buffer */
-  usize_t             srcwin_size;   /* Initial size of the source-window lookahead */
-  usize_t             srcwin_maxsz;  /* srcwin_size grows by a factor of 2 when no matches are found */
-
-  xd3_getblk_func   *getblk;        /* The three callbacks. */
-  xd3_alloc_func    *alloc;
-  xd3_free_func     *freef;
-  void              *opaque;        /* Not used. */
-  int                flags;         /* stream->flags are initialized from xd3_config &
-                                     * never modified by the library.  Use xd3_set_flags
-                                     * to modify flags settings mid-stream. */
-
-  xd3_sec_cfg       sec_data;       /* Secondary compressor config: data */
-  xd3_sec_cfg       sec_inst;       /* Secondary compressor config: inst */
-  xd3_sec_cfg       sec_addr;       /* Secondary compressor config: addr */
-
-  xd3_smatch_cfg     smatch_cfg;    /* See enum: use fields below for soft config */
-  xd3_smatcher       smatcher_soft;
-};
-
-/* The primary source file object. You create one of these objects and initialize the
- * first four fields.  This library maintains the next 5 fields.  The configured getblk
- * implementation is responsible for setting the final 3 fields when called (and/or when
- * XD3_GETSRCBLK is returned).
- */
-struct _xd3_source
-{
-  /* you set */
-  xoff_t              size;          /* size of this source */
-  usize_t             blksize;       /* block size */
-  const char         *name;          /* its name, for debug/print purposes */
-  void               *ioh;           /* opaque handle */
-
-  /* xd3 sets */
-  usize_t             srclen;        /* length of this source window */
-  xoff_t              srcbase;       /* offset of this source window in the source itself */
-  xoff_t              blocks;        /* the total number of blocks in this source */
-  usize_t             cpyoff_blocks; /* offset of copy window in blocks */
-  usize_t             cpyoff_blkoff; /* offset of copy window in blocks, remainder */
-  xoff_t              getblkno;      /* request block number: xd3 sets current getblk request */
-
-  /* getblk sets */
-  xoff_t              curblkno;      /* current block number: client sets after getblk request */
-  usize_t             onblk;         /* number of bytes on current block: client sets, xd3 verifies */
-  const uint8_t      *curblk;        /* current block array: client sets after getblk request */
-};
-
-/* The primary xd3_stream object, used for encoding and decoding.  You may access only two
- * fields: avail_out, next_out.  Use the methods above to operate on xd3_stream. */
-struct _xd3_stream
-{
-  /* input state */
-  const uint8_t    *next_in;          /* next input byte */
-  usize_t           avail_in;         /* number of bytes available at next_in */
-  xoff_t            total_in;         /* how many bytes in */
-
-  /* output state */
-  uint8_t          *next_out;         /* next output byte */
-  usize_t           avail_out;        /* number of bytes available at next_out */
-  usize_t           space_out;        /* total out space */
-  xoff_t            current_window;   /* number of windows encoded/decoded */
-  xoff_t            total_out;        /* how many bytes out */
-
-  /* to indicate an error, xd3 sets */
-  const char       *msg;              /* last error message, NULL if no error */
-
-  /* source configuration */
-  xd3_source       *src;              /* source array */
-
-  /* encoder memory configuration */
-  usize_t           winsize;          /* suggested window size */
-  usize_t           memsize;          /* memory size parameter */
-  usize_t           sprevsz;          /* small string, previous window size (power of 2) */
-  usize_t           sprevmask;        /* small string, previous window size mask */
-  uint              iopt_size;
-  uint              srcwin_size;
-  uint              srcwin_maxsz;
-
-  /* general configuration */
-  xd3_getblk_func  *getblk;           /* set nxtblk, nxtblkno to scanblkno */
-  xd3_alloc_func   *alloc;            /* malloc function */
-  xd3_free_func    *free;             /* free function */
-  void*             opaque;           /* private data object passed to alloc, free, and getblk */
-  int               flags;            /* various options */
-  
-  /* secondary compressor configuration */
-  xd3_sec_cfg       sec_data;         /* Secondary compressor config: data */
-  xd3_sec_cfg       sec_inst;         /* Secondary compressor config: inst */
-  xd3_sec_cfg       sec_addr;         /* Secondary compressor config: addr */
-
-  xd3_smatcher      smatcher;
-
-  usize_t           *large_table;      /* table of large checksums */
-  xd3_hash_cfg       large_hash;       /* large hash config */
-
-  usize_t           *small_table;      /* table of small checksums */
-  xd3_slist         *small_prev;       /* table of previous offsets, circular linked list */
-  int                small_reset;      /* true if small table should be reset */
-
-  xd3_hash_cfg       small_hash;       /* small hash config */
-  xd3_addr_cache     acache;           /* the vcdiff address cache */
-  xd3_encode_state   enc_state;        /* state of the encoder */
-
-  usize_t            taroff;           /* base offset of the target input */
-  usize_t            input_position;   /* current input position */
-  usize_t            min_match;        /* current minimum match length, avoids redundent matches */
-  usize_t            unencoded_offset; /* current input, first unencoded offset. this value is <= the first
-                                       * instruction's position in the iopt buffer, if there is at least one
-                                       * match in the buffer. */
-
-  // SRCWIN
-  // these variables plus srcwin_size, srcwin_maxsz above (set by config)
-  int                srcwin_decided;    /* boolean: true if the srclen,srcbase have been decided. */
-  xoff_t             srcwin_cksum_pos;  /* Source checksum position */
-
-  // MATCH
-  xd3_match_state    match_state;      /* encoder match state */
-  xoff_t             match_srcpos;     /* current match source position relative to srcbase */
-  xoff_t             match_minaddr;    /* smallest matching address to set window params
-                                       * (reset each window xd3_encode_reset) */
-  xoff_t             match_maxaddr;    /* largest matching address to set window params
-                                       * (reset each window xd3_encode_reset) */
-  usize_t            match_back;       /* match extends back so far */
-  usize_t            match_maxback;    /* match extends back maximum */
-  usize_t            match_fwd;        /* match extends forward so far */
-  usize_t            match_maxfwd;     /* match extends forward maximum */
-
-  xoff_t             maxsrcaddr;      /* address of the last source match (across windows) */
-
-  uint8_t          *buf_in;           /* for saving buffered input */
-  usize_t            buf_avail;        /* amount of saved input */
-  const uint8_t    *buf_leftover;     /* leftover content of next_in (i.e., user's buffer) */
-  usize_t            buf_leftavail;    /* amount of leftover content */
-
-  xd3_output       *enc_current;      /* current output buffer */
-  xd3_output       *enc_free;         /* free output buffers */
-  xd3_output       *enc_heads[4];     /* array of encoded outputs: head of chain */
-  xd3_output       *enc_tails[4];     /* array of encoded outputs: tail of chain */
-
-  xd3_iopt_buf      iopt;             /* instruction optimizing buffer */
-  xd3_rinst        *iout;             /* next single instruction */
-
-  const uint8_t    *enc_appheader;    /* application header to encode */
-  usize_t            enc_appheadsz;    /* application header size */
-
-  /* decoder stuff */
-  xd3_decode_state  dec_state;        /* current DEC_XXX value */
-  uint              dec_hdr_ind;      /* VCDIFF header indicator */
-  uint              dec_win_ind;      /* VCDIFF window indicator */
-  uint              dec_del_ind;      /* VCDIFF delta indicator */
-
-  uint8_t           dec_magic[4];     /* First four bytes */
-  usize_t            dec_magicbytes;   /* Magic position. */
-
-  uint               dec_secondid;     /* Optional secondary compressor ID. */
-
-  usize_t            dec_codetblsz;    /* Optional code table: length. */
-  uint8_t          *dec_codetbl;      /* Optional code table: storage. */
-  usize_t            dec_codetblbytes; /* Optional code table: position. */
-
-  usize_t            dec_appheadsz;    /* Optional application header: size. */
-  uint8_t          *dec_appheader;    /* Optional application header: storage */
-  usize_t            dec_appheadbytes; /* Optional application header: position. */
-
-  usize_t            dec_cksumbytes;   /* Optional checksum: position. */
-  uint8_t           dec_cksum[4];     /* Optional checksum: storage. */
-  uint32_t          dec_adler32;      /* Optional checksum: value. */
-
-  usize_t            dec_cpylen;       /* length of copy window (VCD_SOURCE or VCD_TARGET) */
-  xoff_t            dec_cpyoff;       /* offset of copy window (VCD_SOURCE or VCD_TARGET)  */
-  usize_t            dec_enclen;       /* length of delta encoding */
-  usize_t            dec_tgtlen;       /* length of target window */
-
-#if USE_UINT64
-  uint64_t          dec_64part;       /* part of a decoded uint64_t */
-#endif
-#if USE_UINT32
-  uint32_t          dec_32part;       /* part of a decoded uint32_t */
-#endif
-
-  xoff_t            dec_winstart;     /* offset of the start of current target window */
-  xoff_t            dec_window_count; /* == current_window + 1 in DEC_FINISH */
-  usize_t            dec_winbytes;     /* bytes of the three sections so far consumed */
-  usize_t            dec_hdrsize;      /* VCDIFF + app header size */
-
-  const uint8_t    *dec_tgtaddrbase;  /* Base of decoded target addresses (addr >= dec_cpylen). */
-  const uint8_t    *dec_cpyaddrbase;  /* Base of decoded copy addresses (addr < dec_cpylen). */
-
-  usize_t            dec_position;     /* current decoder position counting the cpylen offset */
-  usize_t            dec_maxpos;       /* maximum decoder position counting the cpylen offset */
-  xd3_hinst         dec_current1;     /* current instruction */
-  xd3_hinst         dec_current2;     /* current instruction */
-
-  uint8_t          *dec_buffer;       /* Decode buffer */
-  uint8_t          *dec_lastwin;      /* In case of VCD_TARGET, the last target window. */
-  usize_t            dec_lastlen;      /* length of the last target window */
-  xoff_t            dec_laststart;    /* offset of the start of last target window */
-  usize_t            dec_lastspace;    /* allocated space of last target window, for reuse */
-
-  xd3_desect        inst_sect;        /* staging area for decoding window sections */
-  xd3_desect        addr_sect;
-  xd3_desect        data_sect;
-
-  xd3_code_table_func       *code_table_func;
-  xd3_comp_table_func       *comp_table_func;
-  const xd3_dinst           *code_table;
-  const xd3_code_table_desc *code_table_desc;
-  xd3_dinst                 *code_table_alloc;
-
-  /* secondary compression */
-  const xd3_sec_type *sec_type;
-  xd3_sec_stream     *sec_stream_d;
-  xd3_sec_stream     *sec_stream_i;
-  xd3_sec_stream     *sec_stream_a;
-
-#if XD3_DEBUG
-  /* statistics */
-  usize_t            n_cpy;
-  usize_t            n_add;
-  usize_t            n_run;
-
-  usize_t            n_ibytes;
-  usize_t            n_sbytes;
-  usize_t            n_dbytes;
-
-  usize_t            l_cpy;
-  usize_t            l_add;
-  usize_t            l_run;
-
-  usize_t            sh_searches;
-  usize_t            sh_compares;
-
-  usize_t           *i_freqs;
-  usize_t           *i_modes;
-  usize_t           *i_sizes;
-
-  usize_t            large_ckcnt;
-
-  /* memory usage */
-  usize_t            alloc_cnt;
-  usize_t            free_cnt;
-
-  xoff_t            n_emit;
-#endif
-};
-
-/******************************************************************************************
- PUBLIC FUNCTIONS
- ******************************************************************************************/
-
-/* The two I/O disciplines, encode and decode, have similar stream semantics.  It is
- * recommended that applications use the same code for compression and decompression -
- * because there are only a few differences in handling encoding/decoding.
- *
- * See also the xd3_avail_input() and xd3_consume_output() routines, inlined below.
- *
- *   XD3_INPUT:  the process requires more input: call xd3_avail_input() then repeat
- *   XD3_OUTPUT: the process has more output: read stream->next_out, stream->avail_out,
- *               then call xd3_consume_output(), then repeat
- *   XD3_GOTHEADER: (decoder-only) notification returned following the VCDIFF header and
- *               first window header.  the decoder may use the header to configure itself.
- *   XD3_WINSTART: a general notification returned once for each window except the 0-th
- *               window, which is implied by XD3_GOTHEADER.  It is recommended to
- *               use a switch-stmt such as:
- *                 ...
- *               again:
- *                 switch ((ret = xd3_decode_input (stream))) {
- *                    case XD3_GOTHEADER: {
- *                      assert(stream->current_window == 0);
- *                      stuff;
- *                    }
- *                    // fallthrough 
- *                    case XD3_WINSTART: {
- *                      something(stream->current_window);
- *                      goto again;
- *                    }
- *                    ...
- *   XD3_WINFINISH: a general notification, following the complete input & output of a
- *               window.  at this point, stream->total_in and stream->total_out are
- *               consistent for either encoding or decoding.
- *   XD3_GETSRCBLK: If the xd3_getblk() callback is NULL, this value is returned to
- *               initiate a non-blocking source read.
- *
- * For simple usage, see the xd3_process_completely() function, which underlies
- * xd3_encode_completely() and xd3_decode_completely() [xdelta3.c].  For real application
- * usage, including the application header, the see command-line utility [xdelta3-main.h].
- *
- * main_input() implements the command-line encode and decode as well as the optional
- * VCDIFF_TOOLS printhdr, printhdrs, and printdelta with a single loop [xdelta3-main.h].
- */
-int     xd3_decode_input  (xd3_stream    *stream);
-int     xd3_encode_input  (xd3_stream    *stream);
-
-/* The xd3_config structure is used to initialize a stream - all data is copied into
- * stream so config may be a temporary variable.  See the [documentation] or comments on
- * the xd3_config structure. */
-int     xd3_config_stream (xd3_stream    *stream,
-                           xd3_config    *config);
-
-/* Since Xdelta3 doesn't open any files, xd3_close_stream is just an error check that the
- * stream is in a proper state to be closed: this means the encoder is flushed and the
- * decoder is at a window boundary.  The application is responsible for freeing any of the
- * resources it supplied. */
-int     xd3_close_stream (xd3_stream    *stream);
-
-/* This unconditionally closes/frees the stream, future close() will succeed.*/
-void    xd3_abort_stream (xd3_stream    *stream);
-
-/* xd3_free_stream frees all memory allocated for the stream.  The application is
- * responsible for freeing any of the resources it supplied. */
-void    xd3_free_stream   (xd3_stream    *stream);
-
-/* This function informs the encoder or decoder that source matching (i.e.,
- * delta-compression) is possible.  For encoding, this should be called before the first
- * xd3_encode_input.  A NULL source is ignored.  For decoding, this should be called
- * before the first window is decoded, but the appheader may be read first
- * (XD3_GOTHEADER).  At this point, consult xd3_decoder_needs_source(), inlined below, to
- * determine if a source is expected by the decoder. */
-int     xd3_set_source    (xd3_stream    *stream,
-                           xd3_source    *source);
-
-/* This function invokes xd3_encode_input using whole-file, in-memory inputs.  The output
- * array must be large enough to hold the output or else ENOSPC is returned. */
-int     xd3_encode_completely (xd3_stream    *stream,
-                               const uint8_t *input,
-                               usize_t         input_size,
-                               uint8_t       *output,
-                               usize_t        *output_size,
-                               usize_t         avail_output);
-
-/* This function invokes xd3_decode_input using whole-file, in-memory inputs.  The output
- * array must be large enough to hold the output or else ENOSPC is returned. */
-int     xd3_decode_completely (xd3_stream    *stream,
-                               const uint8_t *input,
-                               usize_t         input_size,
-                               uint8_t       *output,
-                               usize_t        *output_size,
-                               usize_t         avail_size);
-
-/* This should be called before the first call to xd3_encode_input() to include
- * application-specific data in the VCDIFF header. */
-void    xd3_set_appheader (xd3_stream    *stream,
-                           const uint8_t *data,
-                           usize_t         size);
-
-/* xd3_get_appheader may be called in the decoder after XD3_GOTHEADER.  For convenience,
- * the decoder always adds a single byte padding to the end of the application header,
- * which is set to zero in case the application header is a string. */
-int     xd3_get_appheader (xd3_stream     *stream,
-                           uint8_t       **data,
-                           usize_t         *size);
-
-/* After receiving XD3_GOTHEADER, the decoder should check this function which returns 1
- * if the decoder will require source data. */
-int     xd3_decoder_needs_source (xd3_stream *stream);
-
-/* Includes the above rvalues */
-const char* xd3_strerror (int ret);
-
-/* For convenience, zero & initialize the xd3_config structure with specified flags. */
-static inline
-void    xd3_init_config (xd3_config *config,
-                         int         flags)
-{
-  memset (config, 0, sizeof (*config));
-  config->flags = flags;
-}
-
-/* This supplies some input to the stream. */
-static inline
-void    xd3_avail_input  (xd3_stream    *stream,
-                          const uint8_t *idata,
-                          usize_t         isize)
-{
-  /* Even if isize is zero, the code expects a non-NULL idata.  Why?  It uses this value
-   * to determine whether xd3_avail_input has ever been called.  If xd3_encode_input is
-   * called before xd3_avail_input it will return XD3_INPUT right away without allocating
-   * a stream->winsize buffer.  This is to avoid an unwanted allocation. */
-  XD3_ASSERT (idata != NULL);
-
-  stream->next_in  = idata;
-  stream->avail_in = isize;
-}
-
-/* This acknowledges receipt of output data, must be called after any XD3_OUTPUT
- * return. */
-static inline
-void xd3_consume_output (xd3_stream  *stream)
-{
-  stream->avail_out  = 0;
-}
-
-/* These are set for each XD3_WINFINISH return. */
-static inline
-int     xd3_encoder_used_source (xd3_stream *stream) { return stream->src != NULL && stream->src->srclen > 0; }
-static inline
-xoff_t  xd3_encoder_srcbase (xd3_stream *stream) { return stream->src->srcbase; }
-static inline
-usize_t  xd3_encoder_srclen (xd3_stream *stream) { return stream->src->srclen; }
-
-/* Checks for legal flag changes. */
-static inline
-void xd3_set_flags (xd3_stream *stream, int flags)
-{
-  /* The bitwise difference should contain only XD3_FLUSH or XD3_SKIP_WINDOW */
-  XD3_ASSERT(((flags ^ stream->flags) & ~(XD3_FLUSH | XD3_SKIP_WINDOW)) == 0);
-  stream->flags = flags;
-}
-
-/* Gives some extra information about the latest library error, if any is known. */
-static inline
-const char* xd3_errstring (xd3_stream  *stream)
-{
-  return stream->msg ? stream->msg : "";
-}
-
-/* This function tells the number of bytes expected to be set in source->onblk after a
- * getblk request.  This is for convenience of handling a partial last block. */
-static inline
-usize_t xd3_bytes_on_srcblk (xd3_source *source, xoff_t blkno)
-{
-  XD3_ASSERT (blkno < source->blocks);
-
-  if (blkno != source->blocks - 1)
-    {
-      return source->blksize;
-    }
-
-  return (usize_t)((source->size - 1) % source->blksize) + 1;
-}
-
-#endif /* _XDELTA3_H_ */
+/* xdelta 3 - delta compression tools and library

+ * Copyright (C) 2001, 2003, 2004, 2005, 2006.  Joshua P. MacDonald

+ *

+ *  This program is free software; you can redistribute it and/or modify

+ *  it under the terms of the GNU General Public License as published by

+ *  the Free Software Foundation; either version 2 of the License, or

+ *  (at your option) any later version.

+ *

+ *  This program is distributed in the hope that it will be useful,

+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of

+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ *  GNU General Public License for more details.

+ *

+ *  You should have received a copy of the GNU General Public License

+ *  along with this program; if not, write to the Free Software

+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

+ */

+

+/* Welcome to Xdelta.  If you want to know more about Xdelta, start by reading xdelta3.c.

+ * If you are ready to use the API, continue reading here.  There are two interfaces --

+ * xd3_encode_input and xd3_decode_input -- plus a dozen or so related calls.  This

+ * interface is styled after Zlib. */

+

+#ifndef _XDELTA3_H_

+#define _XDELTA3_H_

+

+#include <stdlib.h>

+#include <string.h>

+#include <sys/types.h>

+

+/**********************************************************************/

+

+/* Default configured value of stream->winsize.  If the program supplies

+ * xd3_encode_input() with data smaller than winsize the stream will

+ * automatically buffer the input, otherwise the input buffer is used directly.

+ */

+#ifndef XD3_DEFAULT_WINSIZE

+#define XD3_DEFAULT_WINSIZE (1U << 18)

+#endif

+

+/* This is a unit of how far to advance the checksum position in one go. */

+#ifndef XD3_DEFAULT_CKSUM_ADVANCE

+#define XD3_DEFAULT_CKSUM_ADVANCE (1U << 14)

+#endif

+

+/* Default total size of the source window used in xdelta3-main.h */

+#ifndef XD3_DEFAULT_SRCWINSZ

+#define XD3_DEFAULT_SRCWINSZ (1U << 23)

+#endif

+

+/* Default configured value of stream->memsize.  This dictates how much memory Xdelta will

+ * use for string-matching data structures. */

+#ifndef XD3_DEFAULT_MEMSIZE

+#define XD3_DEFAULT_MEMSIZE (1U << 18)

+#endif

+

+/* When Xdelta requests a memory allocation for certain buffers, it rounds up to units of

+ * at least this size.  The code assumes (and asserts) that this is a power-of-two. */

+#ifndef XD3_ALLOCSIZE

+#define XD3_ALLOCSIZE (1U<<14)

+#endif

+

+/* The XD3_HARDMAXWINSIZE parameter is a safety mechanism to protect decoders against

+ * malicious files.  The decoder will never decode a window larger than this.  If the file

+ * specifies VCD_TARGET the decoder may require two buffers of this size.  Rationale for

+ * choosing 22-bits as a maximum: this means that in the worst case, any VCDIFF address

+ * without a copy window will require 3 bytes to encode (7 bits per byte, HERE and SAME

+ * modes making every address within half the window away. */

+#ifndef XD3_HARDMAXWINSIZE

+#define XD3_HARDMAXWINSIZE (1U<<23)

+#endif

+

+/* The XD3_NODECOMPRESSSIZE parameter tells the xdelta main routine not to try to

+ * externally-decompress source inputs that are too large.  Since these files must be

+ * seekable, they are decompressed to a temporary file location and the user may not wish

+ * for this. */

+#ifndef XD3_NODECOMPRESSSIZE

+#define XD3_NODECOMPRESSSIZE (1U<<24)

+#endif

+

+/* The IOPT_SIZE value sets the size of a buffer used to batch overlapping copy

+ * instructions before they are optimized by picking the best non-overlapping ranges.  The

+ * larger this buffer, the longer a forced xd3_srcwin_setup() decision is held off. */

+#ifndef XD3_DEFAULT_IOPT_SIZE

+#define XD3_DEFAULT_IOPT_SIZE    128

+#endif

+

+/* The maximum distance backward to search for small matches */

+#ifndef XD3_DEFAULT_SPREVSZ

+#define XD3_DEFAULT_SPREVSZ (1U << 16)

+#endif

+

+/* Sizes and addresses within VCDIFF windows are represented as usize_t

+ *

+ * For source-file offsets and total file sizes, total input and output counts, the xoff_t

+ * type is used.  The decoder and encoder generally check for overflow of the xoff_t size,

+ * and this is tested at the 32bit boundary [xdelta3-test.h].

+ */

+#ifndef _WIN32

+typedef unsigned int    usize_t;

+typedef u_int8_t        uint8_t;

+typedef u_int16_t       uint16_t;

+typedef u_int32_t       uint32_t;

+typedef u_int64_t       uint64_t;

+#else

+// CRT_SECURE_NO_DEPRECATE silences warnings about sprintf() and strerror()

+#define _CRT_SECURE_NO_DEPRECATE 1

+#define _CRT_SECURE_COPP_OVERLOAD_STANDARD_NAMES 1

+#define WIN32_LEAN_AND_MEAN

+#include <windows.h>

+#define inline

+typedef unsigned int   uint;

+typedef unsigned int   usize_t;

+typedef unsigned char  uint8_t;

+typedef unsigned short uint16_t;

+typedef unsigned long  uint32_t;

+typedef ULONGLONG      uint64_t;

+#endif

+

+#define SIZEOF_USIZE_T 4

+

+#ifndef XD3_USE_LARGEFILE64

+#define XD3_USE_LARGEFILE64 1

+#endif

+

+#if XD3_USE_LARGEFILE64

+#define __USE_FILE_OFFSET64 1 /* GLIBC: for 64bit fileops, ... ? */

+typedef uint64_t xoff_t;

+#define SIZEOF_XOFF_T 8

+#else

+typedef uint32_t xoff_t;

+#define SIZEOF_XOFF_T 4

+#endif

+

+#define USE_UINT32 (SIZEOF_USIZE_T == 4 || SIZEOF_XOFF_T == 4 || REGRESSION_TEST)

+#define USE_UINT64 (SIZEOF_USIZE_T == 8 || SIZEOF_XOFF_T == 8 || REGRESSION_TEST)

+

+/**********************************************************************/

+

+#ifndef INLINE

+#define INLINE inline

+#endif

+

+/* Whether to build the encoder, otherwise only build the decoder. */

+#ifndef XD3_ENCODER

+#define XD3_ENCODER 1

+#endif

+

+/* The code returned when main() fails, also defined in system includes. */

+#ifndef EXIT_FAILURE

+#define EXIT_FAILURE 1

+#endif

+

+/* REGRESSION TEST enables the "xdelta3 test" command, which runs a series of self-tests. */

+#ifndef REGRESSION_TEST

+#define REGRESSION_TEST 0

+#endif

+

+/* XD3_DEBUG=1 enables assertions and various statistics.  Levels > 1 enable some

+ * additional output only useful during development and debugging. */

+#ifndef XD3_DEBUG

+#define XD3_DEBUG 0

+#endif

+

+#ifndef PYTHON_MODULE

+#define PYTHON_MODULE 0

+#endif

+

+/* There are three string matching functions supplied: one fast, one slow (default), and

+ * one soft-configurable.  To disable any of these, use the following definitions. */

+#ifndef XD3_BUILD_SLOW

+#define XD3_BUILD_SLOW 1

+#endif

+#ifndef XD3_BUILD_FAST

+#define XD3_BUILD_FAST 1

+#endif

+#ifndef XD3_BUILD_SOFT

+#define XD3_BUILD_SOFT 1

+#endif

+

+#if XD3_DEBUG

+#include <stdio.h>

+#endif

+

+/* XPRINT.  Debug output and VCDIFF_TOOLS functions report to stderr.  I have used an

+ * irregular style to abbreviate [fprintf(stderr, "] as [P(RINT "]. */

+#define P    fprintf

+#define RINT stderr,

+

+typedef struct _xd3_stream             xd3_stream;

+typedef struct _xd3_source             xd3_source;

+typedef struct _xd3_hash_cfg           xd3_hash_cfg;

+typedef struct _xd3_smatcher           xd3_smatcher;

+typedef struct _xd3_rinst              xd3_rinst;

+typedef struct _xd3_dinst              xd3_dinst;

+typedef struct _xd3_hinst              xd3_hinst;

+typedef struct _xd3_rpage              xd3_rpage;

+typedef struct _xd3_addr_cache         xd3_addr_cache;

+typedef struct _xd3_output             xd3_output;

+typedef struct _xd3_desect             xd3_desect;

+typedef struct _xd3_iopt_buf           xd3_iopt_buf;

+typedef struct _xd3_rlist              xd3_rlist;

+typedef struct _xd3_sec_type           xd3_sec_type;

+typedef struct _xd3_sec_cfg            xd3_sec_cfg;

+typedef struct _xd3_sec_stream         xd3_sec_stream;

+typedef struct _xd3_config             xd3_config;

+typedef struct _xd3_code_table_desc    xd3_code_table_desc;

+typedef struct _xd3_code_table_sizes   xd3_code_table_sizes;

+typedef struct _xd3_slist              xd3_slist;

+

+/* The stream configuration has three callbacks functions, all of which may be supplied

+ * with NULL values.  If config->getblk is provided as NULL, the stream returns

+ * XD3_GETSRCBLK. */

+

+typedef void*  (xd3_alloc_func)    (void       *opaque,

+                                    usize_t      items,

+                                    usize_t      size);

+typedef void   (xd3_free_func)     (void       *opaque,

+                                    void       *address);

+

+typedef int    (xd3_getblk_func)   (xd3_stream *stream,

+                                    xd3_source *source,

+                                    xoff_t      blkno);

+

+/* These are internal functions to delay construction of encoding tables and support

+ * alternate code tables.  See the comments & code enabled by GENERIC_ENCODE_TABLES. */

+

+typedef const xd3_dinst* (xd3_code_table_func) (void);

+typedef int              (xd3_comp_table_func) (xd3_stream *stream,

+                                                const uint8_t **data,

+                                                usize_t *size);

+

+

+/* Some junk. */

+

+#ifndef XD3_ASSERT

+#if XD3_DEBUG

+#define XD3_ASSERT(x) \

+    do { if (! (x)) { P(RINT "%s:%d: XD3 assertion failed: %s\n", __FILE__, __LINE__, #x); \

+    abort (); } } while (0)

+#else

+#define XD3_ASSERT(x) (void)0

+#endif

+#endif

+

+#ifdef __GNUC__

+/* As seen on linux-kernel. */

+#ifndef max

+#define max(x,y) ({ \

+        const typeof(x) _x = (x);       \

+        const typeof(y) _y = (y);       \

+        (void) (&_x == &_y);            \

+        _x > _y ? _x : _y; })

+#endif

+

+#ifndef min

+#define min(x,y) ({ \

+        const typeof(x) _x = (x);       \

+        const typeof(y) _y = (y);       \

+        (void) (&_x == &_y);            \

+        _x < _y ? _x : _y; })

+#endif

+#else

+#ifndef max

+#define max(x,y) ((x) < (y) ? (y) : (x))

+#endif

+#ifndef min

+#define min(x,y) ((x) < (y) ? (x) : (y))

+#endif

+#endif

+

+/******************************************************************************************

+ PUBLIC ENUMS

+ ******************************************************************************************/

+

+/* These are the five ordinary status codes returned by the xd3_encode_input() and

+ * xd3_decode_input() state machines. */

+typedef enum {

+

+  /* An application must be prepared to handle these five return values from either

+   * xd3_encode_input or xd3_decode_input, except in the case of no-source compression, in

+   * which case XD3_GETSRCBLK is never returned.  More detailed comments for these are

+   * given in xd3_encode_input and xd3_decode_input comments, below. */

+  XD3_INPUT     = -17703, /* need input */

+  XD3_OUTPUT    = -17704, /* have output */

+  XD3_GETSRCBLK = -17705, /* need a block of source input (with no xd3_getblk function),

+                           * a chance to do non-blocking read. */

+  XD3_GOTHEADER = -17706, /* (decode-only) after the initial VCDIFF & first window header */

+  XD3_WINSTART  = -17707, /* notification: returned before a window is processed, giving a

+                           * chance to XD3_SKIP_WINDOW or not XD3_SKIP_EMIT that window. */

+  XD3_WINFINISH  = -17708, /* notification: returned after encode/decode & output for a window */

+  XD3_TOOFARBACK = -17709, /* (encoder only) may be returned by getblk() if the block is too old */

+  XD3_INTERNAL   = -17710, /* internal error */

+

+} xd3_rvalues;

+

+/* special values in config->flags */

+typedef enum

+{

+  XD3_JUST_HDR       = (1 << 1),   /* used by VCDIFF tools, see xdelta3-main.h. */

+  XD3_SKIP_WINDOW    = (1 << 2),   /* used by VCDIFF tools, see xdelta3-main.h. */

+  XD3_SKIP_EMIT      = (1 << 3),   /* used by VCDIFF tools, see xdelta3-main.h. */

+  XD3_FLUSH          = (1 << 4),   /* flush the stream buffer to prepare for xd3_stream_close(). */

+

+  XD3_SEC_DJW        = (1 << 5),   /* use DJW static huffman */

+  XD3_SEC_FGK        = (1 << 6),   /* use FGK adaptive huffman */

+  XD3_SEC_TYPE       = (XD3_SEC_DJW | XD3_SEC_FGK),

+

+  XD3_SEC_NODATA     = (1 << 7),   /* disable secondary compression of the data section. */

+  XD3_SEC_NOINST     = (1 << 8),   /* disable secondary compression of the inst section. */

+  XD3_SEC_NOADDR     = (1 << 9),   /* disable secondary compression of the addr section. */

+

+  XD3_SEC_OTHER      = (XD3_SEC_NODATA | XD3_SEC_NOINST | XD3_SEC_NOADDR),

+

+  XD3_ADLER32        = (1 << 10),  /* enable checksum computation in the encoder. */

+  XD3_ADLER32_NOVER  = (1 << 11),  /* disable checksum verification in the decoder. */

+

+  XD3_ALT_CODE_TABLE = (1 << 12),  /* for testing the alternate code table encoding. */

+

+  XD3_NOCOMPRESS     = (1 << 13),  /* disable ordinary data compression feature,

+                                    * only search the source, not the target. */

+  XD3_BEGREEDY       = (1 << 14),  /* disable the "1.5-pass algorithm", instead use

+                                    * greedy matching.  Greedy is off by default. */

+} xd3_flags;

+

+/* The values of this enumeration are set in xd3_config using the smatch_cfg variable.  It

+ * can be set to slow, fast, soft, or default.  The fast and slow setting uses preset,

+ * hardcoded parameters and the soft setting is accompanied by user-supplied parameters.

+ * If the user supplies 'default' the code selects one of the available string matchers.

+ * Due to compile-time settings (see XD3_SLOW_SMATCHER, XD3_FAST_SMATCHER,

+ * XD3_SOFT_SMATCHER variables), not all options may be available. */

+typedef enum

+{

+  XD3_SMATCH_DEFAULT = 0,

+  XD3_SMATCH_SLOW    = 1,

+  XD3_SMATCH_FAST    = 2,

+  XD3_SMATCH_SOFT    = 3,

+} xd3_smatch_cfg;

+

+/******************************************************************************************

+ PRIVATE ENUMS

+ ******************************************************************************************/

+

+/* stream->match_state is part of the xd3_encode_input state machine for source matching:

+ *

+ *  1. the XD3_GETSRCBLK block-read mechanism means reentrant matching

+ *  2. this state spans encoder windows: a match and end-of-window will continue in the next

+ *  3. the initial target byte and source byte are a presumed match, to avoid some computation

+ *  in case the inputs are identical.

+ */

+typedef enum {

+

+  MATCH_TARGET    = 0, /* in this state, attempt to match the start of the target with the

+                        * previously set source address (initially 0). */

+  MATCH_BACKWARD  = 1, /* currently expanding a match backward in the source/target. */

+  MATCH_FORWARD   = 2, /* currently expanding a match forward in the source/target. */

+  MATCH_SEARCHING = 3, /* currently searching for a match. */

+

+} xd3_match_state;

+

+/* The xd3_encode_input state machine steps through these states in the following order.

+ * The matcher is reentrant and returns XD3_INPUT whenever it requires more data.  After

+ * receiving XD3_INPUT, if the application reads EOF it should call xd3_stream_close().

+ */

+typedef enum {

+

+  ENC_INIT      = 0, /* xd3_encode_input has never been called. */

+  ENC_INPUT     = 1, /* waiting for xd3_avail_input () to be called. */

+  ENC_SEARCH    = 2, /* currently searching for matches. */

+  ENC_FLUSH     = 3, /* currently emitting output. */

+  ENC_POSTOUT   = 4, /* after an output section. */

+  ENC_POSTWIN   = 5, /* after all output sections. */

+  ENC_ABORTED   = 6, /* abort. */

+} xd3_encode_state;

+

+/* The xd3_decode_input state machine steps through these states in the following order.

+ * The matcher is reentrant and returns XD3_INPUT whenever it requires more data.  After

+ * receiving XD3_INPUT, if the application reads EOF it should call xd3_stream_close().

+ *

+ * 0-8:   the VCDIFF header

+ * 9-18:  the VCDIFF window header

+ * 19-21: the three primary sections: data (which I think should have gone last), inst, addr

+ * 22:    producing output: returns XD3_OUTPUT, possibly XD3_GETSRCBLK,

+ * 23:    return XD3_WINFINISH, set state=9 to decode more input

+ */

+typedef enum {

+

+  DEC_VCHEAD   = 0, /* VCDIFF header */

+  DEC_HDRIND   = 1, /* header indicator */

+

+  DEC_SECONDID = 2, /* secondary compressor ID */

+

+  DEC_TABLEN   = 3, /* code table length */

+  DEC_NEAR     = 4, /* code table near */

+  DEC_SAME     = 5, /* code table same */

+  DEC_TABDAT   = 6, /* code table data */

+

+  DEC_APPLEN   = 7, /* application data length */

+  DEC_APPDAT   = 8, /* application data */

+

+  DEC_WININD   = 9, /* window indicator */

+

+  DEC_CPYLEN   = 10, /* copy window length */

+  DEC_CPYOFF   = 11, /* copy window offset */

+

+  DEC_ENCLEN   = 12, /* length of delta encoding */

+  DEC_TGTLEN   = 13, /* length of target window */

+  DEC_DELIND   = 14, /* delta indicator */

+

+  DEC_DATALEN  = 15, /* length of ADD+RUN data */

+  DEC_INSTLEN  = 16, /* length of instruction data */

+  DEC_ADDRLEN  = 17, /* length of address data */

+

+  DEC_CKSUM    = 18, /* window checksum */

+

+  DEC_DATA     = 19, /* data section */

+  DEC_INST     = 20, /* instruction section */

+  DEC_ADDR     = 21, /* address section */

+

+  DEC_EMIT     = 22, /* producing data */

+

+  DEC_FINISH   = 23, /* window finished */

+

+  DEC_ABORTED  = 24, /* xd3_abort_stream */

+} xd3_decode_state;

+

+/* An application never sees these internal codes: */  

+typedef enum {

+  XD3_NOSECOND  = -17708, /* when secondary compression finds no improvement. */

+} xd3_pvalues;

+

+/******************************************************************************************

+ internal types

+ ******************************************************************************************/

+

+/* instruction lists used in the IOPT buffer */

+struct _xd3_rlist

+{

+  xd3_rlist  *next;

+  xd3_rlist  *prev;

+};

+

+/* the raw encoding of an instruction used in the IOPT buffer */

+struct _xd3_rinst

+{

+  uint8_t     type;

+  uint8_t     xtra;

+  uint8_t     code1;

+  uint8_t     code2;

+  usize_t      pos;

+  usize_t      size;

+  xoff_t      addr;

+  xd3_rlist   link;

+};

+

+/* the code-table form of an single- or double-instruction */

+struct _xd3_dinst

+{

+  uint8_t     type1;

+  uint8_t     size1;

+  uint8_t     type2;

+  uint8_t     size2;

+};

+

+/* the decoded form of a single (half) instruction. */

+struct _xd3_hinst

+{

+  uint8_t     type;

+  usize_t      size;

+  usize_t      addr;

+};

+

+/* used by the encoder to buffer output in sections.  list of blocks. */

+struct _xd3_output

+{

+  uint8_t    *base;

+  usize_t      next;

+  usize_t      avail;

+  xd3_output *next_page;

+};

+

+/* the VCDIFF address cache, see the RFC */

+struct _xd3_addr_cache

+{

+  uint     s_near;

+  uint     s_same;

+  usize_t  next_slot;  /* the circular index for near */

+  usize_t *near_array; /* array of size s_near        */

+  usize_t *same_array; /* array of size s_same*256    */

+};

+

+/* the IOPT buffer has a used list of (ordered) instructions, possibly overlapping in

+ * target addresses, awaiting a flush */

+struct _xd3_iopt_buf

+{

+  xd3_rlist  used;

+  xd3_rlist  free;

+  xd3_rinst *buffer;

+};

+

+/* This is the record of a pre-compiled configuration, a subset of xd3_config. */

+struct _xd3_smatcher

+{

+  const char        *name;

+  int             (*string_match) (xd3_stream  *stream);

+  uint               large_look;

+  uint               large_step;

+  uint               small_look;

+  uint               small_chain;

+  uint               small_lchain;

+  uint               ssmatch;

+  uint               try_lazy;

+  uint               max_lazy;

+  uint               long_enough;

+  uint               promote;

+};

+

+/* hash table size & power-of-two hash function. */

+struct _xd3_hash_cfg

+{

+  usize_t           size;

+  usize_t           shift;

+  usize_t           mask;

+};

+

+/* a hash-chain link in the small match table, embedded with position and checksum */

+struct _xd3_slist

+{

+  xd3_slist *next;

+  xd3_slist *prev;

+  usize_t     pos;

+  usize_t     scksum;

+};

+

+/* a decoder section (data, inst, or addr).  there is an optimization to avoid copying

+ * these sections if all the input is available, related to the copied field below.

+ * secondation compression uses the copied2 field. */

+struct _xd3_desect

+{

+  const uint8_t *buf;

+  const uint8_t *buf_max;

+  usize_t         size;

+  usize_t         pos;

+  uint8_t       *copied1;

+  usize_t         alloc1;

+  uint8_t       *copied2;

+  usize_t         alloc2;

+};

+

+/******************************************************************************************

+ public types

+ ******************************************************************************************/

+

+/* Settings for the secondary compressor. */

+struct _xd3_sec_cfg

+{

+  int                data_type;     /* Which section. (set automatically) */

+  int                ngroups;       /* Number of DJW Huffman groups. */

+  int                sector_size;   /* Sector size. */

+  int                inefficient;   /* If true, ignore efficiency check [avoid XD3_NOSECOND]. */

+};

+

+/* This is the user-visible stream configuration. */

+struct _xd3_config

+{

+  usize_t             memsize;       /* How much memory Xdelta may allocate */

+  usize_t             winsize;       /* The encoder window size. */

+  usize_t             sprevsz;       /* How far back small string matching goes */

+  usize_t             iopt_size;     /* entries in the instruction-optimizing buffer */

+  usize_t             srcwin_size;   /* Initial size of the source-window lookahead */

+  usize_t             srcwin_maxsz;  /* srcwin_size grows by a factor of 2 when no matches are found */

+

+  xd3_getblk_func   *getblk;        /* The three callbacks. */

+  xd3_alloc_func    *alloc;

+  xd3_free_func     *freef;

+  void              *opaque;        /* Not used. */

+  int                flags;         /* stream->flags are initialized from xd3_config &

+                                     * never modified by the library.  Use xd3_set_flags

+                                     * to modify flags settings mid-stream. */

+

+  xd3_sec_cfg       sec_data;       /* Secondary compressor config: data */

+  xd3_sec_cfg       sec_inst;       /* Secondary compressor config: inst */

+  xd3_sec_cfg       sec_addr;       /* Secondary compressor config: addr */

+

+  xd3_smatch_cfg     smatch_cfg;    /* See enum: use fields below for soft config */

+  xd3_smatcher       smatcher_soft;

+};

+

+/* The primary source file object. You create one of these objects and initialize the

+ * first four fields.  This library maintains the next 5 fields.  The configured getblk

+ * implementation is responsible for setting the final 3 fields when called (and/or when

+ * XD3_GETSRCBLK is returned).

+ */

+struct _xd3_source

+{

+  /* you set */

+  xoff_t              size;          /* size of this source */

+  usize_t             blksize;       /* block size */

+  const char         *name;          /* its name, for debug/print purposes */

+  void               *ioh;           /* opaque handle */

+

+  /* xd3 sets */

+  usize_t             srclen;        /* length of this source window */

+  xoff_t              srcbase;       /* offset of this source window in the source itself */

+  xoff_t              blocks;        /* the total number of blocks in this source */

+  usize_t             cpyoff_blocks; /* offset of copy window in blocks */

+  usize_t             cpyoff_blkoff; /* offset of copy window in blocks, remainder */

+  xoff_t              getblkno;      /* request block number: xd3 sets current getblk request */

+

+  /* getblk sets */

+  xoff_t              curblkno;      /* current block number: client sets after getblk request */

+  usize_t             onblk;         /* number of bytes on current block: client sets, xd3 verifies */

+  const uint8_t      *curblk;        /* current block array: client sets after getblk request */

+};

+

+/* The primary xd3_stream object, used for encoding and decoding.  You may access only two

+ * fields: avail_out, next_out.  Use the methods above to operate on xd3_stream. */

+struct _xd3_stream

+{

+  /* input state */

+  const uint8_t    *next_in;          /* next input byte */

+  usize_t           avail_in;         /* number of bytes available at next_in */

+  xoff_t            total_in;         /* how many bytes in */

+

+  /* output state */

+  uint8_t          *next_out;         /* next output byte */

+  usize_t           avail_out;        /* number of bytes available at next_out */

+  usize_t           space_out;        /* total out space */

+  xoff_t            current_window;   /* number of windows encoded/decoded */

+  xoff_t            total_out;        /* how many bytes out */

+

+  /* to indicate an error, xd3 sets */

+  const char       *msg;              /* last error message, NULL if no error */

+

+  /* source configuration */

+  xd3_source       *src;              /* source array */

+

+  /* encoder memory configuration */

+  usize_t           winsize;          /* suggested window size */

+  usize_t           memsize;          /* memory size parameter */

+  usize_t           sprevsz;          /* small string, previous window size (power of 2) */

+  usize_t           sprevmask;        /* small string, previous window size mask */

+  uint              iopt_size;

+  uint              srcwin_size;

+  uint              srcwin_maxsz;

+

+  /* general configuration */

+  xd3_getblk_func  *getblk;           /* set nxtblk, nxtblkno to scanblkno */

+  xd3_alloc_func   *alloc;            /* malloc function */

+  xd3_free_func    *free;             /* free function */

+  void*             opaque;           /* private data object passed to alloc, free, and getblk */

+  int               flags;            /* various options */

+  

+  /* secondary compressor configuration */

+  xd3_sec_cfg       sec_data;         /* Secondary compressor config: data */

+  xd3_sec_cfg       sec_inst;         /* Secondary compressor config: inst */

+  xd3_sec_cfg       sec_addr;         /* Secondary compressor config: addr */

+

+  xd3_smatcher      smatcher;

+

+  usize_t           *large_table;      /* table of large checksums */

+  xd3_hash_cfg       large_hash;       /* large hash config */

+

+  usize_t           *small_table;      /* table of small checksums */

+  xd3_slist         *small_prev;       /* table of previous offsets, circular linked list */

+  int                small_reset;      /* true if small table should be reset */

+

+  xd3_hash_cfg       small_hash;       /* small hash config */

+  xd3_addr_cache     acache;           /* the vcdiff address cache */

+  xd3_encode_state   enc_state;        /* state of the encoder */

+

+  usize_t            taroff;           /* base offset of the target input */

+  usize_t            input_position;   /* current input position */

+  usize_t            min_match;        /* current minimum match length, avoids redundent matches */

+  usize_t            unencoded_offset; /* current input, first unencoded offset. this value is <= the first

+                                       * instruction's position in the iopt buffer, if there is at least one

+                                       * match in the buffer. */

+

+  // SRCWIN

+  // these variables plus srcwin_size, srcwin_maxsz above (set by config)

+  int                srcwin_decided;    /* boolean: true if the srclen,srcbase have been decided. */

+  xoff_t             srcwin_cksum_pos;  /* Source checksum position */

+

+  // MATCH

+  xd3_match_state    match_state;      /* encoder match state */

+  xoff_t             match_srcpos;     /* current match source position relative to srcbase */

+  xoff_t             match_minaddr;    /* smallest matching address to set window params

+                                       * (reset each window xd3_encode_reset) */

+  xoff_t             match_maxaddr;    /* largest matching address to set window params

+                                       * (reset each window xd3_encode_reset) */

+  usize_t            match_back;       /* match extends back so far */

+  usize_t            match_maxback;    /* match extends back maximum */

+  usize_t            match_fwd;        /* match extends forward so far */

+  usize_t            match_maxfwd;     /* match extends forward maximum */

+

+  xoff_t             maxsrcaddr;      /* address of the last source match (across windows) */

+

+  uint8_t          *buf_in;           /* for saving buffered input */

+  usize_t            buf_avail;        /* amount of saved input */

+  const uint8_t    *buf_leftover;     /* leftover content of next_in (i.e., user's buffer) */

+  usize_t            buf_leftavail;    /* amount of leftover content */

+

+  xd3_output       *enc_current;      /* current output buffer */

+  xd3_output       *enc_free;         /* free output buffers */

+  xd3_output       *enc_heads[4];     /* array of encoded outputs: head of chain */

+  xd3_output       *enc_tails[4];     /* array of encoded outputs: tail of chain */

+

+  xd3_iopt_buf      iopt;             /* instruction optimizing buffer */

+  xd3_rinst        *iout;             /* next single instruction */

+

+  const uint8_t    *enc_appheader;    /* application header to encode */

+  usize_t            enc_appheadsz;    /* application header size */

+

+  /* decoder stuff */

+  xd3_decode_state  dec_state;        /* current DEC_XXX value */

+  uint              dec_hdr_ind;      /* VCDIFF header indicator */

+  uint              dec_win_ind;      /* VCDIFF window indicator */

+  uint              dec_del_ind;      /* VCDIFF delta indicator */

+

+  uint8_t           dec_magic[4];     /* First four bytes */

+  usize_t            dec_magicbytes;   /* Magic position. */

+

+  uint               dec_secondid;     /* Optional secondary compressor ID. */

+

+  usize_t            dec_codetblsz;    /* Optional code table: length. */

+  uint8_t          *dec_codetbl;      /* Optional code table: storage. */

+  usize_t            dec_codetblbytes; /* Optional code table: position. */

+

+  usize_t            dec_appheadsz;    /* Optional application header: size. */

+  uint8_t          *dec_appheader;    /* Optional application header: storage */

+  usize_t            dec_appheadbytes; /* Optional application header: position. */

+

+  usize_t            dec_cksumbytes;   /* Optional checksum: position. */

+  uint8_t           dec_cksum[4];     /* Optional checksum: storage. */

+  uint32_t          dec_adler32;      /* Optional checksum: value. */

+

+  usize_t            dec_cpylen;       /* length of copy window (VCD_SOURCE or VCD_TARGET) */

+  xoff_t            dec_cpyoff;       /* offset of copy window (VCD_SOURCE or VCD_TARGET)  */

+  usize_t            dec_enclen;       /* length of delta encoding */

+  usize_t            dec_tgtlen;       /* length of target window */

+

+#if USE_UINT64

+  uint64_t          dec_64part;       /* part of a decoded uint64_t */

+#endif

+#if USE_UINT32

+  uint32_t          dec_32part;       /* part of a decoded uint32_t */

+#endif

+

+  xoff_t            dec_winstart;     /* offset of the start of current target window */

+  xoff_t            dec_window_count; /* == current_window + 1 in DEC_FINISH */

+  usize_t            dec_winbytes;     /* bytes of the three sections so far consumed */

+  usize_t            dec_hdrsize;      /* VCDIFF + app header size */

+

+  const uint8_t    *dec_tgtaddrbase;  /* Base of decoded target addresses (addr >= dec_cpylen). */

+  const uint8_t    *dec_cpyaddrbase;  /* Base of decoded copy addresses (addr < dec_cpylen). */

+

+  usize_t            dec_position;     /* current decoder position counting the cpylen offset */

+  usize_t            dec_maxpos;       /* maximum decoder position counting the cpylen offset */

+  xd3_hinst         dec_current1;     /* current instruction */

+  xd3_hinst         dec_current2;     /* current instruction */

+

+  uint8_t          *dec_buffer;       /* Decode buffer */

+  uint8_t          *dec_lastwin;      /* In case of VCD_TARGET, the last target window. */

+  usize_t            dec_lastlen;      /* length of the last target window */

+  xoff_t            dec_laststart;    /* offset of the start of last target window */

+  usize_t            dec_lastspace;    /* allocated space of last target window, for reuse */

+

+  xd3_desect        inst_sect;        /* staging area for decoding window sections */

+  xd3_desect        addr_sect;

+  xd3_desect        data_sect;

+

+  xd3_code_table_func       *code_table_func;

+  xd3_comp_table_func       *comp_table_func;

+  const xd3_dinst           *code_table;

+  const xd3_code_table_desc *code_table_desc;

+  xd3_dinst                 *code_table_alloc;

+

+  /* secondary compression */

+  const xd3_sec_type *sec_type;

+  xd3_sec_stream     *sec_stream_d;

+  xd3_sec_stream     *sec_stream_i;

+  xd3_sec_stream     *sec_stream_a;

+

+#if XD3_DEBUG

+  /* statistics */

+  usize_t            n_cpy;

+  usize_t            n_add;

+  usize_t            n_run;

+

+  usize_t            n_ibytes;

+  usize_t            n_sbytes;

+  usize_t            n_dbytes;

+

+  usize_t            l_cpy;

+  usize_t            l_add;

+  usize_t            l_run;

+

+  usize_t            sh_searches;

+  usize_t            sh_compares;

+

+  usize_t           *i_freqs;

+  usize_t           *i_modes;

+  usize_t           *i_sizes;

+

+  usize_t            large_ckcnt;

+

+  /* memory usage */

+  usize_t            alloc_cnt;

+  usize_t            free_cnt;

+

+  xoff_t            n_emit;

+#endif

+};

+

+/******************************************************************************************

+ PUBLIC FUNCTIONS

+ ******************************************************************************************/

+

+/* The two I/O disciplines, encode and decode, have similar stream semantics.  It is

+ * recommended that applications use the same code for compression and decompression -

+ * because there are only a few differences in handling encoding/decoding.

+ *

+ * See also the xd3_avail_input() and xd3_consume_output() routines, inlined below.

+ *

+ *   XD3_INPUT:  the process requires more input: call xd3_avail_input() then repeat

+ *   XD3_OUTPUT: the process has more output: read stream->next_out, stream->avail_out,

+ *               then call xd3_consume_output(), then repeat

+ *   XD3_GOTHEADER: (decoder-only) notification returned following the VCDIFF header and

+ *               first window header.  the decoder may use the header to configure itself.

+ *   XD3_WINSTART: a general notification returned once for each window except the 0-th

+ *               window, which is implied by XD3_GOTHEADER.  It is recommended to

+ *               use a switch-stmt such as:

+ *                 ...

+ *               again:

+ *                 switch ((ret = xd3_decode_input (stream))) {

+ *                    case XD3_GOTHEADER: {

+ *                      assert(stream->current_window == 0);

+ *                      stuff;

+ *                    }

+ *                    // fallthrough 

+ *                    case XD3_WINSTART: {

+ *                      something(stream->current_window);

+ *                      goto again;

+ *                    }

+ *                    ...

+ *   XD3_WINFINISH: a general notification, following the complete input & output of a

+ *               window.  at this point, stream->total_in and stream->total_out are

+ *               consistent for either encoding or decoding.

+ *   XD3_GETSRCBLK: If the xd3_getblk() callback is NULL, this value is returned to

+ *               initiate a non-blocking source read.

+ *

+ * For simple usage, see the xd3_process_completely() function, which underlies

+ * xd3_encode_completely() and xd3_decode_completely() [xdelta3.c].  For real application

+ * usage, including the application header, the see command-line utility [xdelta3-main.h].

+ *

+ * main_input() implements the command-line encode and decode as well as the optional

+ * VCDIFF_TOOLS printhdr, printhdrs, and printdelta with a single loop [xdelta3-main.h].

+ */

+int     xd3_decode_input  (xd3_stream    *stream);

+int     xd3_encode_input  (xd3_stream    *stream);

+

+/* The xd3_config structure is used to initialize a stream - all data is copied into

+ * stream so config may be a temporary variable.  See the [documentation] or comments on

+ * the xd3_config structure. */

+int     xd3_config_stream (xd3_stream    *stream,

+                           xd3_config    *config);

+

+/* Since Xdelta3 doesn't open any files, xd3_close_stream is just an error check that the

+ * stream is in a proper state to be closed: this means the encoder is flushed and the

+ * decoder is at a window boundary.  The application is responsible for freeing any of the

+ * resources it supplied. */

+int     xd3_close_stream (xd3_stream    *stream);

+

+/* This unconditionally closes/frees the stream, future close() will succeed.*/

+void    xd3_abort_stream (xd3_stream    *stream);

+

+/* xd3_free_stream frees all memory allocated for the stream.  The application is

+ * responsible for freeing any of the resources it supplied. */

+void    xd3_free_stream   (xd3_stream    *stream);

+

+/* This function informs the encoder or decoder that source matching (i.e.,

+ * delta-compression) is possible.  For encoding, this should be called before the first

+ * xd3_encode_input.  A NULL source is ignored.  For decoding, this should be called

+ * before the first window is decoded, but the appheader may be read first

+ * (XD3_GOTHEADER).  At this point, consult xd3_decoder_needs_source(), inlined below, to

+ * determine if a source is expected by the decoder. */

+int     xd3_set_source    (xd3_stream    *stream,

+                           xd3_source    *source);

+

+/* This function invokes xd3_encode_input using whole-file, in-memory inputs.  The output

+ * array must be large enough to hold the output or else ENOSPC is returned. */

+int     xd3_encode_completely (xd3_stream    *stream,

+                               const uint8_t *input,

+                               usize_t         input_size,

+                               uint8_t       *output,

+                               usize_t        *output_size,

+                               usize_t         avail_output);

+

+/* This function invokes xd3_decode_input using whole-file, in-memory inputs.  The output

+ * array must be large enough to hold the output or else ENOSPC is returned. */

+int     xd3_decode_completely (xd3_stream    *stream,

+                               const uint8_t *input,

+                               usize_t         input_size,

+                               uint8_t       *output,

+                               usize_t        *output_size,

+                               usize_t         avail_size);

+

+/* This should be called before the first call to xd3_encode_input() to include

+ * application-specific data in the VCDIFF header. */

+void    xd3_set_appheader (xd3_stream    *stream,

+                           const uint8_t *data,

+                           usize_t         size);

+

+/* xd3_get_appheader may be called in the decoder after XD3_GOTHEADER.  For convenience,

+ * the decoder always adds a single byte padding to the end of the application header,

+ * which is set to zero in case the application header is a string. */

+int     xd3_get_appheader (xd3_stream     *stream,

+                           uint8_t       **data,

+                           usize_t         *size);

+

+/* After receiving XD3_GOTHEADER, the decoder should check this function which returns 1

+ * if the decoder will require source data. */

+int     xd3_decoder_needs_source (xd3_stream *stream);

+

+/* Includes the above rvalues */

+const char* xd3_strerror (int ret);

+

+/* For convenience, zero & initialize the xd3_config structure with specified flags. */

+static inline

+void    xd3_init_config (xd3_config *config,

+                         int         flags)

+{

+  memset (config, 0, sizeof (*config));

+  config->flags = flags;

+}

+

+/* This supplies some input to the stream. */

+static inline

+void    xd3_avail_input  (xd3_stream    *stream,

+                          const uint8_t *idata,

+                          usize_t         isize)

+{

+  /* Even if isize is zero, the code expects a non-NULL idata.  Why?  It uses this value

+   * to determine whether xd3_avail_input has ever been called.  If xd3_encode_input is

+   * called before xd3_avail_input it will return XD3_INPUT right away without allocating

+   * a stream->winsize buffer.  This is to avoid an unwanted allocation. */

+  XD3_ASSERT (idata != NULL);

+

+  stream->next_in  = idata;

+  stream->avail_in = isize;

+}

+

+/* This acknowledges receipt of output data, must be called after any XD3_OUTPUT

+ * return. */

+static inline

+void xd3_consume_output (xd3_stream  *stream)

+{

+  stream->avail_out  = 0;

+}

+

+/* These are set for each XD3_WINFINISH return. */

+static inline

+int     xd3_encoder_used_source (xd3_stream *stream) { return stream->src != NULL && stream->src->srclen > 0; }

+static inline

+xoff_t  xd3_encoder_srcbase (xd3_stream *stream) { return stream->src->srcbase; }

+static inline

+usize_t  xd3_encoder_srclen (xd3_stream *stream) { return stream->src->srclen; }

+

+/* Checks for legal flag changes. */

+static inline

+void xd3_set_flags (xd3_stream *stream, int flags)

+{

+  /* The bitwise difference should contain only XD3_FLUSH or XD3_SKIP_WINDOW */

+  XD3_ASSERT(((flags ^ stream->flags) & ~(XD3_FLUSH | XD3_SKIP_WINDOW)) == 0);

+  stream->flags = flags;

+}

+

+/* Gives some extra information about the latest library error, if any is known. */

+static inline

+const char* xd3_errstring (xd3_stream  *stream)

+{

+  return stream->msg ? stream->msg : "";

+}

+

+/* This function tells the number of bytes expected to be set in source->onblk after a

+ * getblk request.  This is for convenience of handling a partial last block. */

+static inline

+usize_t xd3_bytes_on_srcblk (xd3_source *source, xoff_t blkno)

+{

+  XD3_ASSERT (blkno < source->blocks);

+

+  if (blkno != source->blocks - 1)

+    {

+      return source->blksize;

+    }

+

+  return (usize_t)((source->size - 1) % source->blksize) + 1;

+}

+

+#endif /* _XDELTA3_H_ */