From 2874dd83f1efa51d152fc0fc15ac024e1249dd4e Mon Sep 17 00:00:00 2001 From: dotdotisdead Date: Sun, 10 Dec 2006 21:52:57 +0000 Subject: Adds some comments about LZMA external compression. --- xdelta3/Makefile | 2 +- xdelta3/xdelta3-main.h | 5927 ++++++++++++++++++++++++------------------------ xdelta3/xdelta3-test.h | 6 +- 3 files changed, 2972 insertions(+), 2963 deletions(-) diff --git a/xdelta3/Makefile b/xdelta3/Makefile index a23ce28..dcbbace 100755 --- a/xdelta3/Makefile +++ b/xdelta3/Makefile @@ -62,7 +62,7 @@ xdelta3: $(SOURCES) xdelta3-debug: $(SOURCES) $(CC) -g -Wall -Wshadow xdelta3.c -o xdelta3-debug -DXD3_MAIN=1 -DGENERIC_ENCODE_TABLES=1 \ - -DXD3_USE_LARGEFILE64=1 -DXD3_STDIO=1 -DREGRESSION_TEST=1 -DXD3_DEBUG=2 -DSECONDARY_DJW=1 -lm + -DXD3_USE_LARGEFILE64=1 -DXD3_STDIO=1 -DREGRESSION_TEST=1 -DXD3_DEBUG=1 -DSECONDARY_DJW=1 -DSECONDARY_FGK=1 -lm $(PYTGT): $(SOURCES) $(PYTHON) setup.py install --verbose --compile --force diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h index eb6ca9c..0b21c2f 100755 --- a/xdelta3/xdelta3-main.h +++ b/xdelta3/xdelta3-main.h @@ -1,2959 +1,2968 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2001 and onward. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -/* This is all the extra stuff you need for convenience to users in a command line - * application. It contains these major components: - * - * 1. VCDIFF tools - * 2. external compression support (this is POSIX-specific). - * 3. a general read/write loop that handles all of the Xdelta decode/encode/VCDIFF-print - * functions - * 4. command-line interpreter - * 5. an Xdelta application header which stores default filename, external compression settings - * 6. output/error printing - * 7. basic file support and OS interface - */ - -/* TODO list: - * 1. do exact gzip-like filename, stdout handling. make a .xz extension, refuse - * to encode to stdout without -cf, etc. - * 2. Allow the user to add a comment string to the app header without disturbing the default - * behavior. - */ - -/* On error handling and printing: - * - * The xdelta library sets stream->msg to indicate what condition caused an internal - * failure, but many failures originate here and are printed here. The return convention - * is 0 for success, as throughout Xdelta code, but special attention is required here for - * the operating system calls with different error handling. See the main_file_* routines. - * All errors in this file have a message printed at the time of occurance. Since some of - * these calls occur within calls to the library, the error may end up being printed again - * with a more general error message. - */ - -/******************************************************************************************/ - -#ifndef XD3_POSIX -#define XD3_POSIX 0 -#endif -#ifndef XD3_STDIO -#define XD3_STDIO 0 -#endif - -/* XPRINTX (used by main) prefixes an "xdelta3: " to the output. */ -#define XPR fprintf -#define NT stderr, "xdelta3: " - -#define VC fprintf -#define UT vcout, - -/* If none are set, default to posix. */ -#if (XD3_POSIX + XD3_STDIO) == 0 -#undef XD3_POSIX -#define XD3_POSIX 1 -#endif - -/* Handle externally-compressed inputs. */ -#ifndef EXTERNAL_COMPRESSION -#define EXTERNAL_COMPRESSION 1 -#endif - -#define PRINTHDR_SPECIAL -4378291 - -/* The number of soft-config variables. */ -#define XD3_SOFTCFG_VARCNT 1 - -/* this is used as in XPR(NT XD3_LIB_ERRMSG (stream, ret)) to print an error message - * from the library. */ -#define XD3_LIB_ERRMSG(stream, ret) "%s: %s\n", xd3_errstring (stream), xd3_strerror (ret) - -#include /* fprintf */ - -#if XD3_POSIX -#include /* close, read, write... */ -#include -#include -#endif - -#ifndef _WIN32 -#include /* lots */ -#include /* gettimeofday() */ -#include /* stat() and fstat() */ -#else -#define strtoll _strtoi64 -#include -#include -#ifndef WIFEXITED -# define WIFEXITED(stat) (((*((int *) &(stat))) & 0xff) == 0) -#endif -#ifndef WEXITSTATUS -# define WEXITSTATUS(stat) (((*((int *) &(stat))) >> 8) & 0xff) -#endif -#ifndef S_ISREG -# ifdef S_IFREG -# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) -# else -# define S_ISREG(m) 1 -# endif -#endif /* !S_ISREG */ -#endif - -/****************************************************************************************** - ENUMS and TYPES - ******************************************************************************************/ - -/* These flags (mainly pertaining to main_read() operations) are set in the - * main_file->flags variable. All are related to with external decompression support. - * - * RD_FIRST causes the external decompression check when the input is first read. - * - * RD_NONEXTERNAL disables external decompression for reading a compressed input, in the - * case of Xdelta inputs. Note: Xdelta is supported as an external compression type, - * which makes is the reason for this flag. An example to justify this is: to create a - * delta between two files that are VCDIFF-compressed. Two external Xdelta decoders are - * run to supply decompressed source and target inputs to the Xdelta encoder. */ -typedef enum -{ - RD_FIRST = (1 << 0), - RD_NONEXTERNAL = (1 << 1), - RD_EXTERNAL_V1 = (1 << 2), -} xd3_read_flags; - -/* main_file->mode values */ -typedef enum -{ - XO_READ = 0, - XO_WRITE = 1, -} main_file_modes; - -/* Main commands. For example, CMD_PRINTHDR is the "xdelta printhdr" command. */ -typedef enum -{ - CMD_NONE = 0, - CMD_PRINTHDR, - CMD_PRINTHDRS, - CMD_PRINTDELTA, -#if XD3_ENCODER - CMD_ENCODE, -#endif - CMD_DECODE, - CMD_TEST, - CMD_CONFIG, -} xd3_cmd; - -#if XD3_ENCODER -#define CMD_DEFAULT CMD_ENCODE -#define IS_ENCODE(cmd) (cmd == CMD_ENCODE) -#else -#define CMD_DEFAULT CMD_DECODE -#define IS_ENCODE(cmd) (0) -#endif - -typedef struct _main_file main_file; -typedef struct _main_extcomp main_extcomp; -typedef struct _main_blklru main_blklru; -typedef struct _main_blklru_list main_blklru_list; - -/* The main_file object supports abstract system calls like open, close, read, write, seek, - * stat. The program uses these to represent both seekable files and non-seekable files. - * Source files must be seekable, but the target input and any output file do not require - * seekability. - */ -struct _main_file -{ -#if XD3_STDIO - FILE *file; -#elif XD3_POSIX - int file; -#endif - - int mode; /* XO_READ and XO_WRITE */ - const char *filename; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */ - char *filename_copy; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */ - const char *realname; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */ - const main_extcomp *compressor; /* External compression struct. */ - int flags; /* RD_FIRST, RD_NONEXTERNAL, ... */ - xoff_t nread; /* for input position */ - xoff_t nwrite; /* for output position */ -}; - -/* Various strings and magic values used to detect and call external compression. See - * below for examples. */ -struct _main_extcomp -{ - const char *recomp_cmdname; - const char *recomp_options; - - const char *decomp_cmdname; - const char *decomp_options; - - const char *ident; - const char *magic; - int magic_size; - int flags; -}; - -/* This file implements a small LRU of source blocks. For encoding purposes, - * we prevent paging in blocks we've already scanned in the source (return - * XD3_NOTAVAIL). */ -struct _main_blklru_list -{ - main_blklru_list *next; - main_blklru_list *prev; -}; - -struct _main_blklru -{ - uint8_t *blk; - xoff_t blkno; - main_blklru_list link; -}; - -/* ... represented as a list (no cache index). */ -XD3_MAKELIST(main_blklru_list,main_blklru,link); - -/* Program options: various command line flags and options. */ -static int option_stdout = 0; -static int option_force = 0; -static int option_verbose = 0; -static int option_quiet = 0; -static int option_level = 6; -static int option_use_appheader = 1; -static uint8_t* option_appheader = NULL; -static int option_use_secondary = /* until-standardized, leave this off */ 0; -static char* option_secondary = NULL; -static int option_use_checksum = 1; -static int option_use_altcodetable = 0; -static char* option_smatch_config = NULL; -static int option_no_compress = 0; -static int option_no_output = 0; /* go through the motions, but do not open or write output */ -static const char *option_source_filename = NULL; -static const char *option_xdelta1 = "xdelta1"; - -static usize_t option_winsize = XD3_DEFAULT_WINSIZE; -static usize_t option_srcwinsz = XD3_DEFAULT_SRCWINSZ; -static usize_t option_memsize = XD3_DEFAULT_MEMSIZE; - -/* This controls the number of times main repeats itself, only for profiling. */ -static int option_profile_cnt = 0; - -/* These variables are supressed to avoid their use w/o support. main() warns - * appropriately. */ -#if EXTERNAL_COMPRESSION -static int option_decompress_inputs = 1; -static int option_recompress_outputs = 1; -#endif - -/* This is for comparing "printdelta" output without attention to - * copy-instruction modes, useful for reverse engineering. */ -#if VCDIFF_TOOLS -static int option_print_cpymode = 1; -#endif - -/* Static variables */ -IF_DEBUG(static int main_mallocs = 0;) - -static char* program_name = NULL; -static uint8_t* appheader_used = NULL; -static uint8_t* main_bdata = NULL; - -/* The LRU: obviously this is shared by all callers. */ -static int lru_size = 0; -static main_blklru *lru = NULL; /* array of lru_size elts */ -static main_blklru_list lru_list; -static main_blklru_list lru_free; -static int do_not_lru = 0; /* set to avoid lru, instead discard oldest */ - -static int lru_hits = 0; -static int lru_misses = 0; -static int lru_filled = 0; - -/* Hacks for VCDIFF tools */ -static int allow_fake_source = 0; - -/* This array of compressor types is compiled even if EXTERNAL_COMPRESSION is false just so - * the program knows the mapping of IDENT->NAME. */ -static main_extcomp extcomp_types[] = -{ - /* The entry for xdelta3/1 must be 0/1 because the program_names are set there. */ - { "xdelta3", "-cfq", "xdelta3", "-dcfq", "X", "\xd6\xc3\xc4", 3, RD_NONEXTERNAL }, - { "xdelta1", "delta", "xdelta1", "patch", "1", "%XD", 3, RD_EXTERNAL_V1 }, - { "bzip2", "-cf", "bzip2", "-dcf", "B", "BZh", 3, 0 }, - { "gzip", "-cf", "gzip", "-dcf", "G", "\037\213", 2, 0 }, - { "compress", "-cf", "uncompress", "-cf", "Z", "\037\235", 2, 0 }, -}; - -static void main_get_appheader (xd3_stream *stream, main_file *ifile, - main_file *output, main_file *sfile); - -static int main_help (void); - -static int -main_version (void) -{ - /* $Format: " P(RINT \"VERSION=3.$Xdelta3Version$\\n\");" $ */ - P(RINT "VERSION=3.0h_pre0\n"); - return EXIT_SUCCESS; -} - -static int -main_config (void) -{ - main_version (); - - P(RINT "EXTERNAL_COMPRESSION=%d\n", EXTERNAL_COMPRESSION); - P(RINT "GENERIC_ENCODE_TABLES=%d\n", GENERIC_ENCODE_TABLES); - P(RINT "GENERIC_ENCODE_TABLES_COMPUTE=%d\n", GENERIC_ENCODE_TABLES_COMPUTE); - P(RINT "REGRESSION_TEST=%d\n", REGRESSION_TEST); - P(RINT "SECONDARY_DJW=%d\n", SECONDARY_DJW); - P(RINT "SECONDARY_FGK=%d\n", SECONDARY_FGK); - P(RINT "VCDIFF_TOOLS=%d\n", VCDIFF_TOOLS); - P(RINT "XD3_ALLOCSIZE=%d\n", XD3_ALLOCSIZE); - P(RINT "XD3_DEBUG=%d\n", XD3_DEBUG); - P(RINT "XD3_DEFAULT_CKSUM_SIZE=%d\n", XD3_DEFAULT_CKSUM_ADVANCE); - P(RINT "XD3_DEFAULT_IOPT_SIZE=%d\n", XD3_DEFAULT_IOPT_SIZE); - P(RINT "XD3_DEFAULT_MEMSIZE=%d\n", XD3_DEFAULT_MEMSIZE); - P(RINT "XD3_DEFAULT_SPREVSZ=%d\n", XD3_DEFAULT_SPREVSZ); - P(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ); - P(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE); - P(RINT "XD3_ENCODER=%d\n", XD3_ENCODER); - P(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE); - P(RINT "XD3_NODECOMPRESSSIZE=%d\n", XD3_NODECOMPRESSSIZE); - P(RINT "XD3_POSIX=%d\n", XD3_POSIX); - P(RINT "XD3_USE_LARGEFILE64=%d\n", XD3_USE_LARGEFILE64); - - return EXIT_SUCCESS; -} - -static void* -main_malloc1 (usize_t size) -{ - void* r = malloc (size); - if (r == NULL) { XPR(NT "malloc: %s\n", xd3_strerror (ENOMEM)); } - else if (option_verbose > 2) { XPR(NT "malloc: %u: %p\n", size, r); } - return r; -} - -static void* -main_malloc (usize_t size) -{ - void *r = main_malloc1 (size); - if (r) { IF_DEBUG (main_mallocs += 1); } - return r; -} - -static void* -main_alloc (void *opaque, - usize_t items, - usize_t size) -{ - return main_malloc1 (items * size); -} - -static void -main_free1 (void *opaque, void *ptr) -{ - if (option_verbose > 2) { XPR(NT "free: %p\n", ptr); } - free (ptr); -} - -static void -main_free (void *ptr) -{ - if (ptr) - { - IF_DEBUG (main_mallocs -= 1); - main_free1 (NULL, ptr); - } -} - -/* This ensures that (ret = errno) always indicates failure, in case errno was - * accidentally not set. If this prints there's a bug somewhere. */ -static int -get_errno (void) -{ - if (errno == 0) - { - XPR(NT "you found a bug: expected errno != 0\n"); - errno = XD3_INTERNAL; - } - return errno; -} - - - -static long -get_millisecs_now (void) -{ -#ifndef WIN32 - struct timeval tv; - - gettimeofday (& tv, NULL); - - return (tv.tv_sec) * 1000L + (tv.tv_usec) / 1000; -#else - // Found this in an example on www.codeproject.com - // It doesn't matter that the offset is Jan 1, 1601 - // Result is the numbre of 100 nanosecond units - // 100ns * 10,000 = 1ms - SYSTEMTIME st; - FILETIME ft; - __int64 *pi = (__int64*)&ft; - GetLocalTime(&st); - SystemTimeToFileTime(&st, &ft); - return (long)((*pi) / 10000); -#endif -} - -/* Always >= 1 millisec, right? */ -static long -get_millisecs_since (void) -{ - static long last = 0; - long now = get_millisecs_now(); - long diff = now - last; - last = now; - return diff; -} - -static char* -main_format_bcnt (xoff_t r, char *buf) -{ - static const char* fmts[] = { "B", "KB", "MB", "GB" }; - int i; - - for (i = 0; i < SIZEOF_ARRAY(fmts); i += 1) - { - if (r < 10 * 1e3 || i == (-1 + (int)SIZEOF_ARRAY(fmts))) - { - sprintf (buf, "%"Q"u %s", r, fmts[i]); - break; - } - r /= 1000; - } - return buf; -} - -static char* -main_format_rate (xoff_t bytes, long millis, char *buf) -{ - xoff_t r = (xoff_t)(1.0 * bytes / (1.0 * millis / 1000.0)); - static char lbuf[32]; - - main_format_bcnt (r, lbuf); - sprintf (buf, "%s/sec", lbuf); - return buf; -} - -static char* -main_format_millis (long millis, char *buf) -{ - if (millis < 1000) { sprintf (buf, "%lu ms", millis); } - else if (millis < 10000) { sprintf (buf, "%.1f sec", millis / 1000.0); } - else { sprintf (buf, "%lu sec", millis / 1000L); } - return buf; -} - -/* A safe version of strtol for xoff_t. */ -static int -main_strtoxoff (const char* s, xoff_t *xo, char which) -{ - char *e; - xoff_t x; - - XD3_ASSERT(s && *s != 0); - - { - /* Should check LONG_MIN, LONG_MAX, LLONG_MIN, LLONG_MAX? */ -#if SIZEOF_XOFF_T == 4 - long xx = strtol (s, &e, 0); -#else - long long xx = strtoll (s, &e, 0); -#endif - - if (xx < 0) - { - XPR(NT "-%c: negative integer: %s\n", which, s); - return EXIT_FAILURE; - } - - x = xx; - } - - if (*e != 0) - { - XPR(NT "-%c: invalid integer: %s\n", which, s); - return EXIT_FAILURE; - } - - (*xo) = x; - return 0; -} - -static int -main_atou (const char* arg, usize_t *xo, usize_t low, char which) -{ - xoff_t x; - int ret; - - if ((ret = main_strtoxoff (arg, & x, which))) { return ret; } - - if (x > USIZE_T_MAX || x < low) - { - XPR(NT "-%c: minimum value: %u", which, low); - return EXIT_FAILURE; - } - (*xo) = (usize_t)x; - return 0; -} - -/****************************************************************************************** - FILE BASICS - ******************************************************************************************/ - -/* With all the variation in file system-call semantics, arguments, return values and - * error-handling for the POSIX and STDIO file APIs, the insides of these functions make - * me sick, which is why these wrappers exist. */ - -#define XOPEN_OPNAME (xfile->mode == XO_READ ? "read" : "write") -#define XOPEN_STDIO (xfile->mode == XO_READ ? "rb" : "wb") -#define XOPEN_POSIX (xfile->mode == XO_READ ? O_RDONLY : O_WRONLY | O_CREAT | O_TRUNC) -#define XOPEN_MODE (xfile->mode == XO_READ ? 0 : 0666) - -#define XF_ERROR(op, name, ret) XPR(NT "file %s failed: %s: %s: %s\n", (op), XOPEN_OPNAME, (name), xd3_strerror (ret)) - -#if XD3_STDIO -#define XFNO(f) fileno(f->file) -#define XSTDOUT_XF(f) { (f)->file = stdout; (f)->filename = "/dev/stdout"; } -#define XSTDERR_XF(f) { (f)->file = stderr; (f)->filename = "/dev/stderr"; } -#define XSTDIN_XF(f) { (f)->file = stdin; (f)->filename = "/dev/stdin"; } - -#elif XD3_POSIX -#define XFNO(f) f->file -#define XSTDOUT_XF(f) { (f)->file = STDOUT_FILENO; (f)->filename = "/dev/stdout"; } -#define XSTDERR_XF(f) { (f)->file = STDERR_FILENO; (f)->filename = "/dev/stderr"; } -#define XSTDIN_XF(f) { (f)->file = STDIN_FILENO; (f)->filename = "/dev/stdin"; } -#endif - -static void -main_file_init (main_file *xfile) -{ - memset (xfile, 0, sizeof (*xfile)); - -#if XD3_POSIX - xfile->file = -1; -#endif -} - -static void -main_file_cleanup (main_file *xfile) -{ - if (xfile->filename_copy) { - main_free(xfile->filename_copy); - } -} - -static int -main_file_isopen (main_file *xfile) -{ -#if XD3_STDIO - return xfile->file != NULL; - -#elif XD3_POSIX - return xfile->file != -1; -#endif -} - -static int -main_file_close (main_file *xfile) -{ - int ret = 0; - - if (! main_file_isopen (xfile)) - { - return 0; - } - -#if XD3_STDIO - ret = fclose (xfile->file); - xfile->file = NULL; - -#elif XD3_POSIX - ret = close (xfile->file); - xfile->file = -1; -#endif - - if (ret != 0) { XF_ERROR ("close", xfile->filename, ret = get_errno ()); } - return ret; -} - -static int -main_file_open (main_file *xfile, const char* name, int mode) -{ - int ret = 0; - - xfile->mode = mode; - - XD3_ASSERT (! main_file_isopen (xfile)); - -#if XD3_STDIO - xfile->file = fopen (name, XOPEN_STDIO); - - ret = (xfile->file == NULL) ? get_errno () : 0; - -#elif XD3_POSIX - if ((ret = open (name, XOPEN_POSIX, XOPEN_MODE)) < 0) - { - ret = get_errno (); - } - else - { - xfile->file = ret; - ret = 0; - } -#endif - if (ret) { XF_ERROR ("open", name, ret); } - else { xfile->realname = name; xfile->nread = 0; } - return ret; -} - -static int -main_file_stat (main_file *xfile, xoff_t *size, int err_ifnoseek) -{ - int ret; - struct stat sbuf; - - XD3_ASSERT (main_file_isopen (xfile)); - - if (fstat (XFNO (xfile), & sbuf) < 0) - { - ret = get_errno (); - if (err_ifnoseek) { XF_ERROR ("stat", xfile->filename, ret); } - return ret; - } - - if (! S_ISREG (sbuf.st_mode)) - { - if (err_ifnoseek) { XPR(NT "source file must be seekable: %s\n", xfile->filename); } - return ESPIPE; - } - - (*size) = sbuf.st_size; - return 0; -} - -static int -main_file_exists (main_file *xfile) -{ - struct stat sbuf; - return stat (xfile->filename, & sbuf) == 0 && S_ISREG (sbuf.st_mode); -} - -#if (XD3_POSIX || EXTERNAL_COMPRESSION) -/* POSIX-generic code takes a function pointer to read() or write(). This calls the - * function repeatedly until the buffer is full or EOF. The NREAD parameter is not - * set for write, NULL is passed. Return is signed, < 0 indicate errors, otherwise - * byte count. */ -typedef int (xd3_posix_func) (int fd, uint8_t *buf, usize_t size); - -static int -xd3_posix_io (int fd, uint8_t *buf, usize_t size, xd3_posix_func *func, usize_t *nread) -{ - int ret; - usize_t nproc = 0; - - while (nproc < size) - { - int result = (*func) (fd, buf + nproc, size - nproc); - - if (result < 0) - { - ret = get_errno (); - if (ret != EAGAIN && ret != EINTR) - { - return ret; - } - result = 0; - } - - if (nread != NULL && result == 0) { break; } - - nproc += result; - } - if (nread != NULL) { (*nread) = nproc; } - return 0; -} -#endif - -/* POSIX is unbuffered, while STDIO is buffered. main_file_read() should always be called - * on blocks. */ -static int -main_file_read (main_file *ifile, - uint8_t *buf, - usize_t size, - usize_t *nread, - const char *msg) -{ - int ret = 0; - -#if XD3_STDIO - usize_t result; - - result = fread (buf, 1, size, ifile->file); - - if (result < size && ferror (ifile->file)) - { - ret = get_errno (); - } - else - { - *nread = result; - } - -#elif XD3_POSIX - ret = xd3_posix_io (ifile->file, buf, size, (xd3_posix_func*) &read, nread); -#endif - - if (ret) - { - XPR(NT "%s: %s: %s\n", msg, ifile->filename, xd3_strerror (ret)); - } - else - { - if (option_verbose > 2) { XPR(NT "main read: %s: %u\n", ifile->filename, (*nread)); } - ifile->nread += (*nread); - } - - return ret; -} - -static int -main_file_write (main_file *ofile, uint8_t *buf, usize_t size, const char *msg) -{ - int ret = 0; - -#if XD3_STDIO - usize_t result; - - result = fwrite (buf, 1, size, ofile->file); - - if (result != size) { ret = get_errno (); } - -#elif XD3_POSIX - ret = xd3_posix_io (ofile->file, buf, size, (xd3_posix_func*) &write, NULL); -#endif - - if (ret) - { - XPR(NT "%s: %s: %s\n", msg, ofile->filename, xd3_strerror (ret)); - } - else - { - if (option_verbose > 2) { XPR(NT "main write: %s: %u\n", ofile->filename, size); } - ofile->nwrite += size; - } - - return ret; -} - -static int -main_file_seek (main_file *xfile, xoff_t pos) -{ - int ret = 0; - -#if XD3_STDIO - if (fseek (xfile->file, pos, SEEK_SET) != 0) { ret = get_errno (); } -#else - if (lseek (xfile->file, pos, SEEK_SET) != pos) { ret = get_errno (); } -#endif - - if (ret) - { - XPR(NT "seek failed: %s: %s\n", xfile->filename, xd3_strerror (ret)); - } - - return ret; -} - -/****************************************************************************************** - VCDIFF TOOLS - ******************************************************************************************/ - -#if VCDIFF_TOOLS -/* This function prints a single VCDIFF window, mainly for debugging purposes. */ -static int -main_print_window (xd3_stream* stream, FILE *vcout) -{ - int ret; - usize_t size = 0; - - VC(UT " Offset Code Type1 Size1 @Addr1 + Type2 Size2 @Addr2\n"); - - while (stream->inst_sect.buf < stream->inst_sect.buf_max) - { - uint code = stream->inst_sect.buf[0]; - - if ((ret = xd3_decode_instruction (stream))) { return ret; } - - VC(UT " %06"Q"u %03u %s %3u", stream->dec_winstart + size, code, - xd3_rtype_to_string (stream->dec_current1.type, option_print_cpymode), - stream->dec_current1.size); - - if (stream->dec_current1.type != XD3_NOOP) - { - size += stream->dec_current1.size; - if (stream->dec_current1.type >= XD3_CPY) - { - VC(UT " @%-6u", stream->dec_current1.addr); - } - else - { - VC(UT " "); - } - } - - if (stream->dec_current2.type != XD3_NOOP) - { - size += stream->dec_current2.size; - VC(UT " %s %3u", - xd3_rtype_to_string (stream->dec_current2.type, option_print_cpymode), - stream->dec_current2.size); - - if (stream->dec_current2.type >= XD3_CPY) - { - VC(UT " @%-6u", stream->dec_current2.addr); - } - } - - VC(UT "\n"); - } - - if (stream->dec_tgtlen != size && (stream->flags & XD3_SKIP_WINDOW) == 0) - { - XPR(NT "target window size inconsistency"); - return XD3_INTERNAL; - } - - if (stream->dec_position != stream->dec_maxpos) - { - XPR(NT "target window position inconsistency"); - return XD3_INTERNAL; - } - - if (stream->addr_sect.buf != stream->addr_sect.buf_max) - { - XPR(NT "address section inconsistency"); - return XD3_INTERNAL; - } - - IF_DEBUG (VC(UT "SIZE=%u TGTLEN=%u\n", size, stream->dec_tgtlen)); - - return 0; -} - -static void -main_print_vcdiff_file (main_file *file, const char *type, FILE *vcout) -{ - if (file->filename) { VC(UT "XDELTA filename (%s): %s\n", type, file->filename); } - if (file->compressor) { VC(UT "XDELTA ext comp (%s): %s\n", type, file->compressor->recomp_cmdname); } -} - -/* This function prints a VCDIFF input, mainly for debugging purposes. */ -static int -main_print_func (xd3_stream* stream, main_file *xfile) -{ - int ret; - FILE *vcout; -#if XD3_POSIX - if (! (vcout = fdopen (dup(xfile->file), "w"))) - { - ret = get_errno (); - XPR(NT "fdopen: %s: %s\n", xfile->filename, xd3_strerror (ret)); - return ret; - } -#elif XD3_STDIO - vcout = xfile->file; -#endif - XD3_ASSERT (vcout); - if (stream->dec_winstart == 0) - { - VC(UT "VCDIFF version: 0\n"); - - VC(UT "VCDIFF header size: %d\n", stream->dec_hdrsize); - VC(UT "VCDIFF header indicator: "); - if ((stream->dec_hdr_ind & VCD_SECONDARY) != 0) VC(UT "VCD_SECONDARY "); - if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) VC(UT "VCD_CODETABLE "); - if ((stream->dec_hdr_ind & VCD_APPHEADER) != 0) VC(UT "VCD_APPHEADER "); - if (stream->dec_hdr_ind == 0) VC(UT "none"); - VC(UT "\n"); - - IF_SEC(VC(UT "VCDIFF secondary compressor: %s\n", stream->sec_type ? stream->sec_type->name : "none")); - IF_NSEC(VC(UT "VCDIFF secondary compressor: unsupported\n")); - - if (stream->dec_hdr_ind & VCD_APPHEADER) - { - uint8_t *apphead; - usize_t appheadsz; - ret = xd3_get_appheader (stream, & apphead, & appheadsz); - - if (ret == 0 && appheadsz > 0) - { - int sq = option_quiet; - main_file i, o, s; - XD3_ASSERT (apphead != NULL); - VC(UT "VCDIFF application header: "); - fwrite (apphead, 1, appheadsz, vcout); - VC(UT "\n"); - - main_file_init (& i); - main_file_init (& o); - main_file_init (& s); - option_quiet = 1; - main_get_appheader (stream, &i, & o, & s); - option_quiet = sq; - main_print_vcdiff_file (& o, "output", vcout); - main_print_vcdiff_file (& s, "source", vcout); - main_file_cleanup (& i); - main_file_cleanup (& o); - main_file_cleanup (& s); - } - } - } - else - { - VC(UT "\n"); - } - - VC(UT "VCDIFF window number: %"Q"u\n", stream->current_window); - VC(UT "VCDIFF window indicator: "); - if ((stream->dec_win_ind & VCD_SOURCE) != 0) VC(UT "VCD_SOURCE "); - if ((stream->dec_win_ind & VCD_TARGET) != 0) VC(UT "VCD_TARGET "); - if ((stream->dec_win_ind & VCD_ADLER32) != 0) VC(UT "VCD_ADLER32 "); - if (stream->dec_win_ind == 0) VC(UT "none"); - VC(UT "\n"); - - if ((stream->dec_win_ind & VCD_ADLER32) != 0) - { - VC(UT "VCDIFF adler32 checksum: %08X\n", stream->dec_adler32); - } - - if (stream->dec_del_ind != 0) - { - VC(UT "VCDIFF delta indicator: "); - if ((stream->dec_del_ind & VCD_DATACOMP) != 0) VC(UT "VCD_DATACOMP "); - if ((stream->dec_del_ind & VCD_INSTCOMP) != 0) VC(UT "VCD_INSTCOMP "); - if ((stream->dec_del_ind & VCD_ADDRCOMP) != 0) VC(UT "VCD_ADDRCOMP "); - if (stream->dec_del_ind == 0) VC(UT "none"); - VC(UT "\n"); - } - - if (stream->dec_winstart != 0) - { - VC(UT "VCDIFF window at offset: %"Q"u\n", stream->dec_winstart); - } - - if (SRCORTGT (stream->dec_win_ind)) - { - VC(UT "VCDIFF copy window length: %u\n", stream->dec_cpylen); - VC(UT "VCDIFF copy window offset: %"Q"u\n", stream->dec_cpyoff); - } - - VC(UT "VCDIFF delta encoding length: %u\n", stream->dec_enclen); - VC(UT "VCDIFF target window length: %u\n", stream->dec_tgtlen); - - VC(UT "VCDIFF data section length: %u\n", stream->data_sect.size); - VC(UT "VCDIFF inst section length: %u\n", stream->inst_sect.size); - VC(UT "VCDIFF addr section length: %u\n", stream->addr_sect.size); - - ret = 0; - if ((stream->flags & XD3_JUST_HDR) != 0) - { - /* Print a header -- finished! */ - ret = PRINTHDR_SPECIAL; - } - else if ((stream->flags & XD3_SKIP_WINDOW) == 0) - { - ret = main_print_window (stream, vcout); - } - - fclose (vcout); - return ret; -} -#endif /* VCDIFF_TOOLS */ - -/****************************************************************************************** - Input decompression, output recompression - ******************************************************************************************/ - -#if EXTERNAL_COMPRESSION -/* This is tricky POSIX-specific code with lots of fork(), pipe(), dup(), waitpid(), and - * exec() business. Most of this code originated in PRCS1, which did automatic - * package-file decompression. It works with both XD3_POSIX and XD3_STDIO file - * disciplines. - * - * To automatically detect compressed inputs requires a child process to reconstruct the - * input stream, which was advanced in order to detect compression, because it may not be - * seekable. In other words, the main program reads part of the input stream, and if it - * detects a compressed input it then forks a pipe copier process, which copies the - * first-read block out of the main-program's memory, then streams the remaining - * compressed input into the input-decompression pipe. - */ - -#include -#include -#include - -/* Remember which pipe FD is which. */ -#define PIPE_READ_FD 0 -#define PIPE_WRITE_FD 1 - -static pid_t ext_subprocs[2]; -static char* ext_tmpfile = NULL; - -/* Like write(), but makes repeated calls to empty the buffer. */ -static int -main_pipe_write (int outfd, const uint8_t *exist_buf, usize_t remain) -{ - int ret; - - if ((ret = xd3_posix_io (outfd, (uint8_t*) exist_buf, remain, (xd3_posix_func*) &write, NULL))) - { - XPR(NT "pipe write failed: %s", xd3_strerror (ret)); - return ret; - } - - return 0; -} - -/* A simple error-reporting waitpid interface. */ -static int -main_waitpid_check(pid_t pid) -{ - int status; - int ret = 0; - - if (waitpid (pid, & status, 0) < 0) - { - ret = get_errno (); - XPR(NT "compression subprocess: wait: %s\n", xd3_strerror (ret)); - } - else if (! WIFEXITED (status)) - { - ret = ECHILD; - XPR(NT "compression subprocess: signal %d\n", - WIFSIGNALED (status) ? WTERMSIG (status) : WSTOPSIG (status)); - } - else if (WEXITSTATUS (status) != 0) - { - ret = ECHILD; - XPR(NT "compression subprocess: exit %d\n", WEXITSTATUS (status)); - } - - return ret; -} - -/* Wait for any existing child processes to check for abnormal exit. */ -static int -main_external_compression_finish (void) -{ - int i; - int ret; - - for (i = 0; i < 2; i += 1) - { - if (! ext_subprocs[i]) { continue; } - - if ((ret = main_waitpid_check (ext_subprocs[i]))) - { - return ret; - } - } - - return 0; -} - -/* This runs as a forked process of main_input_decompress_setup() to copy input to the - * decompression process. First, the available input is copied out of the existing - * buffer, then the buffer is reused to continue reading from the compressed input - * file. */ -static int -main_pipe_copier (uint8_t *pipe_buf, - usize_t pipe_bufsize, - usize_t nread, - main_file *ifile, - int outfd) -{ - int ret; - - for (;;) - { - if (nread > 0 && (ret = main_pipe_write (outfd, pipe_buf, nread))) - { - return ret; - } - - if (nread < pipe_bufsize) - { - break; - } - - if ((ret = main_file_read (ifile, pipe_buf, pipe_bufsize, & nread, "pipe read failed")) < 0) - { - return ret; - } - } - - return 0; -} - -/* This function is called after we have read some amount of data from the input file and - * detected a compressed input. Here we start a decompression subprocess by forking - * twice. The first process runs the decompression command, the second process copies - * data to the input of the first. */ -static int -main_input_decompress_setup (const main_extcomp *decomp, - main_file *ifile, - uint8_t *input_buf, - usize_t input_bufsize, - uint8_t *pipe_buf, - usize_t pipe_bufsize, - usize_t pipe_avail, - usize_t *nread) -{ - int outpipefd[2], inpipefd[2]; /* The two pipes: input and output file descriptors. */ - int input_fd = -1; /* The resulting input_fd (output of decompression). */ - pid_t decomp_id, copier_id; /* The two subprocs. */ - int ret; - - outpipefd[0] = outpipefd[1] = -1; - inpipefd[0] = inpipefd[1] = -1; - - if (pipe (outpipefd) || pipe (inpipefd)) - { - XPR(NT "pipe failed: %s\n", xd3_strerror (ret = get_errno ())); - goto pipe_cleanup; - } - - if ((decomp_id = fork ()) < 0) - { - XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ())); - goto pipe_cleanup; - } - - /* The first child runs the decompression process: */ - if (decomp_id == 0) - { - /* Setup pipes: write to the outpipe, read from the inpipe. */ - if (dup2 (outpipefd[PIPE_WRITE_FD], STDOUT_FILENO) < 0 || - dup2 (inpipefd[PIPE_READ_FD], STDIN_FILENO) < 0 || - close (outpipefd[PIPE_READ_FD]) || - close (outpipefd[PIPE_WRITE_FD]) || - close (inpipefd[PIPE_READ_FD]) || - close (inpipefd[PIPE_WRITE_FD]) || - execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, decomp->decomp_options, NULL)) - { - XPR(NT "child process %s failed to execute: %s\n", decomp->decomp_cmdname, xd3_strerror (get_errno ())); - } - - _exit (127); - } - - ext_subprocs[0] = decomp_id; - - if ((copier_id = fork ()) < 0) - { - XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ())); - goto pipe_cleanup; - } - - /* The second child runs the copier process: */ - if (copier_id == 0) - { - int exitval = 0; - - if (close (inpipefd[PIPE_READ_FD]) || - main_pipe_copier (pipe_buf, pipe_bufsize, pipe_avail, ifile, inpipefd[PIPE_WRITE_FD]) || - close (inpipefd[PIPE_WRITE_FD])) - { - XPR(NT "child copier process failed: %s\n", xd3_strerror (get_errno ())); - exitval = 1; - } - - _exit (exitval); - } - - ext_subprocs[1] = copier_id; - - /* The parent closes both pipes after duplicating the output of compression. */ - input_fd = dup (outpipefd[PIPE_READ_FD]); - - if (input_fd < 0 || - main_file_close (ifile) || - close (outpipefd[PIPE_READ_FD]) || - close (outpipefd[PIPE_WRITE_FD]) || - close (inpipefd[PIPE_READ_FD]) || - close (inpipefd[PIPE_WRITE_FD])) - { - XPR(NT "dup/close failed: %s\n", xd3_strerror (ret = get_errno ())); - goto pipe_cleanup; - } - -#if XD3_STDIO - /* Note: fdopen() acquires the fd, closes it when finished. */ - if ((ifile->file = fdopen (input_fd, "r")) == NULL) - { - XPR(NT "fdopen failed: %s\n", xd3_strerror (ret = get_errno ())); - goto pipe_cleanup; - } - -#elif XD3_POSIX - ifile->file = input_fd; -#endif - - ifile->compressor = decomp; - - /* Now the input file is decompressed. */ - return main_file_read (ifile, input_buf, input_bufsize, nread, "input decompression failed"); - - pipe_cleanup: - close (input_fd); - close (outpipefd[PIPE_READ_FD]); - close (outpipefd[PIPE_WRITE_FD]); - close (inpipefd[PIPE_READ_FD]); - close (inpipefd[PIPE_WRITE_FD]); - return ret; -} - - -/* This routine is called when the first buffer of input data is read by the main program - * (unless input decompression is disabled by command-line option). If it recognizes the - * magic number of a known input type it invokes decompression. - * - * Skips decompression if the decompression type or the file type is RD_NONEXTERNAL. - * - * Behaves exactly like main_file_read, otherwise. - * - * This function uses a separate buffer to read the first small block of input. If a - * compressed input is detected, the separate buffer is passed to the pipe copier. This - * avoids using the same size buffer in both cases. */ -static int -main_decompress_input_check (main_file *ifile, - uint8_t *input_buf, - usize_t input_size, - usize_t *nread) -{ - int i; - int ret; - uint8_t check_buf[XD3_ALLOCSIZE]; - usize_t check_nread; - - if ((ret = main_file_read (ifile, check_buf, min (input_size, XD3_ALLOCSIZE), & check_nread, "input read failed"))) - { - return ret; - } - - for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) - { - const main_extcomp *decomp = & extcomp_types[i]; - - if ((check_nread > decomp->magic_size) && - /* The following expr skips decompression if we are trying to read a VCDIFF - * input and that is the magic number. */ - !((decomp->flags & RD_NONEXTERNAL) && (ifile->flags & RD_NONEXTERNAL)) && - memcmp (check_buf, decomp->magic, decomp->magic_size) == 0) - { - if (! option_quiet) - { - XPR(NT "%s | %s %s\n", - ifile->filename, - decomp->decomp_cmdname, - decomp->decomp_options); - } - - return main_input_decompress_setup (decomp, ifile, - input_buf, input_size, - check_buf, XD3_ALLOCSIZE, - check_nread, nread); - } - } - - /* Now read the rest of the input block. */ - (*nread) = 0; - - if (check_nread == XD3_ALLOCSIZE) - { - ret = main_file_read (ifile, input_buf + XD3_ALLOCSIZE, - input_size - XD3_ALLOCSIZE, nread, - "input read failed"); - } - - memcpy (input_buf, check_buf, check_nread); - - (*nread) += check_nread; - - return 0; -} - -/* This is called when the source file needs to be decompressed. We fork/exec a - * decompression command with the proper input and output to a temporary file. */ -static int -main_decompress_source (main_file *sfile, xd3_source *source) -{ - const main_extcomp *decomp = sfile->compressor; - pid_t decomp_id; /* One subproc. */ - int input_fd = -1; - int output_fd = -1; - int ret; - char *tmpname = NULL; - char *tmpdir = getenv ("TMPDIR"); - static const char tmpl[] = "/xd3src.XXXXXX"; - - /* Make a template for mkstmp() */ - if (tmpdir == NULL) { tmpdir = "/tmp"; } - if ((tmpname = main_malloc (strlen (tmpdir) + sizeof (tmpl) + 1)) == NULL) { return ENOMEM; } - sprintf (tmpname, "%s%s", tmpdir, tmpl); - - XD3_ASSERT (ext_tmpfile == NULL); - ext_tmpfile = tmpname; - - /* Open the output FD. */ - if ((output_fd = mkstemp (tmpname)) < 0) - { - XPR(NT "mkstemp failed: %s: %s", tmpname, xd3_strerror (ret = get_errno ())); - goto cleanup; - } - - /* Copy the input FD, reset file position. */ - XD3_ASSERT (main_file_isopen (sfile)); -#if XD3_STDIO - if ((input_fd = dup (fileno (sfile->file))) < 0) - { - XPR(NT "dup failed: %s", xd3_strerror (ret = get_errno ())); - goto cleanup; - } - main_file_close (sfile); - sfile->file = NULL; -#elif XD3_POSIX - input_fd = sfile->file; - sfile->file = -1; -#endif - - if ((ret = lseek (input_fd, SEEK_SET, 0)) != 0) - { - XPR(NT "lseek failed: : %s", xd3_strerror (ret = get_errno ())); - goto cleanup; - } - - if ((decomp_id = fork ()) < 0) - { - XPR(NT "fork failed: %s", xd3_strerror (ret = get_errno ())); - goto cleanup; - } - - /* The child runs the decompression process: */ - if (decomp_id == 0) - { - /* Setup pipes: write to the output file, read from the pipe. */ - if (dup2 (input_fd, STDIN_FILENO) < 0 || - dup2 (output_fd, STDOUT_FILENO) < 0 || - execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, decomp->decomp_options, NULL)) - { - XPR(NT "child process %s failed to execute: %s\n", - decomp->decomp_cmdname, xd3_strerror (get_errno ())); - } - - _exit (127); - } - - close (input_fd); - close (output_fd); - input_fd = -1; - output_fd = -1; - - /* Then wait for completion. */ - if ((ret = main_waitpid_check (decomp_id))) - { - goto cleanup; - } - - /* Open/stat the decompressed source file. */ - if ((ret = main_file_open (sfile, tmpname, XO_READ))) { goto cleanup; } - if ((ret = main_file_stat (sfile, & source->size, 1))) { goto cleanup; } - return 0; - - cleanup: - close (input_fd); - close (output_fd); - if (tmpname) { free (tmpname); } - ext_tmpfile = NULL; - return ret; -} - -/* Initiate re-compression of the output stream. This is easier than input decompression - * because we know beforehand that the stream will be compressed, whereas the input has - * already been read when we decide it should be decompressed. Thus, it only requires one - * subprocess and one pipe. */ -static int -main_recompress_output (main_file *ofile) -{ - pid_t recomp_id; /* One subproc. */ - int pipefd[2]; /* One pipe. */ - int output_fd = -1; - int ret; - const main_extcomp *recomp = ofile->compressor; - - pipefd[0] = pipefd[1] = -1; - - if (pipe (pipefd)) - { - XPR(NT "pipe failed: %s\n", xd3_strerror (ret = get_errno ())); - goto pipe_cleanup; - } - - if ((recomp_id = fork ()) < 0) - { - XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ())); - goto pipe_cleanup; - } - - /* The child runs the recompression process: */ - if (recomp_id == 0) - { - /* Setup pipes: write to the output file, read from the pipe. */ - if (dup2 (XFNO (ofile), STDOUT_FILENO) < 0 || - dup2 (pipefd[PIPE_READ_FD], STDIN_FILENO) < 0 || - close (pipefd[PIPE_READ_FD]) || - close (pipefd[PIPE_WRITE_FD]) || - execlp (recomp->recomp_cmdname, recomp->recomp_cmdname, recomp->recomp_options, NULL)) - { - XPR(NT "child process %s failed to execute: %s\n", recomp->recomp_cmdname, xd3_strerror (get_errno ())); - } - - _exit (127); - } - - ext_subprocs[0] = recomp_id; - - /* The parent closes both pipes after duplicating the output-fd for writing to the - * compression pipe. */ - output_fd = dup (pipefd[PIPE_WRITE_FD]); - - if (output_fd < 0 || - main_file_close (ofile) || - close (pipefd[PIPE_READ_FD]) || - close (pipefd[PIPE_WRITE_FD])) - { - XPR(NT "close failed: %s\n", xd3_strerror (ret = get_errno ())); - goto pipe_cleanup; - } - -#if XD3_STDIO - /* Note: fdopen() acquires the fd, closes it when finished. */ - if ((ofile->file = fdopen (output_fd, "w")) == NULL) - { - XPR(NT "fdopen failed: %s\n", xd3_strerror (ret = get_errno ())); - goto pipe_cleanup; - } - -#elif XD3_POSIX - ofile->file = output_fd; -#endif - - /* Now the output file will be compressed. */ - return 0; - - pipe_cleanup: - close (output_fd); - close (pipefd[PIPE_READ_FD]); - close (pipefd[PIPE_WRITE_FD]); - return ret; -} -#endif /* EXTERNAL_COMPRESSION */ - -/* Identify the compressor that was used based on its ident string, which is passed in the - * application header. */ -static const main_extcomp* -main_ident_compressor (const char *ident) -{ - int i; - - for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) - { - if (strcmp (extcomp_types[i].ident, ident) == 0) - { - return & extcomp_types[i]; - } - } - - return NULL; -} - -/* Return the main_extcomp record to use for this identifier, if possible. */ -static const main_extcomp* -main_get_compressor (const char *ident) -{ - const main_extcomp *ext = main_ident_compressor (ident); - - if (ext == NULL) - { - if (! option_quiet) - { - XPR(NT "warning: cannot recompress output: " - "unrecognized external compression ID: %s\n", ident); - } - return NULL; - } - else if (! EXTERNAL_COMPRESSION) - { - if (! option_quiet) - { - XPR(NT "warning: external support not compiled: " - "original input was compressed: %s\n", ext->recomp_cmdname); - } - return NULL; - } - else - { - return ext; - } -} - -/****************************************************************************************** - APPLICATION HEADER - ******************************************************************************************/ - -#if XD3_ENCODER -static const char* -main_apphead_string (const char* x) -{ - const char *y; - - if (x == NULL) { return ""; } - - if (strcmp (x, "/dev/stdin") == 0 || - strcmp (x, "/dev/stdout") == 0 || - strcmp (x, "/dev/stderr") == 0) { return "-"; } - - // TODO: this is not portable - return (y = strrchr (x, '/')) == NULL ? x : y + 1; -} - -static int -main_set_appheader (xd3_stream *stream, main_file *input, main_file *sfile) -{ - /* The user may disable the application header. Once the appheader is set, this - * disables setting it again. */ - if (appheader_used || ! option_use_appheader) { return 0; } - - /* The user may specify the application header, otherwise format the default header. */ - if (option_appheader) - { - appheader_used = option_appheader; - } - else - { - const char *iname; - const char *icomp; - const char *sname; - const char *scomp; - int len; - - iname = main_apphead_string (input->filename); - icomp = (input->compressor == NULL) ? "" : input->compressor->ident; - len = strlen (iname) + strlen (icomp) + 2; - - if (sfile->filename != NULL) - { - sname = main_apphead_string (sfile->filename); - scomp = (sfile->compressor == NULL) ? "" : sfile->compressor->ident; - len += strlen (sname) + strlen (scomp) + 2; - } - else - { - sname = scomp = ""; - } - - if ((appheader_used = main_malloc (len)) == NULL) - { - return ENOMEM; - } - - if (sfile->filename == NULL) - { - sprintf ((char*)appheader_used, "%s/%s", iname, icomp); - } - else - { - sprintf ((char*)appheader_used, "%s/%s/%s/%s", iname, icomp, sname, scomp); - } - } - - xd3_set_appheader (stream, appheader_used, strlen ((char*)appheader_used)); - - return 0; -} -#endif - -static void -main_get_appheader_params (main_file *file, char **parsed, int output, const char *type, - main_file *other) -{ - /* Set the filename if it was not specified. If output, option_stdout (-c) overrides. */ - if (file->filename == NULL && ! (output && option_stdout) && strcmp (parsed[0], "-") != 0) - { - file->filename = parsed[0]; - - if (other->filename != NULL) { - /* Take directory from the other file, if it has one. */ - char *last_slash = strrchr(other->filename, '/'); - - if (last_slash != NULL) { - int dlen = last_slash - other->filename; - file->filename_copy = main_malloc(dlen + 2 + strlen(file->filename)); - - strncpy(file->filename_copy, other->filename, dlen); - file->filename_copy[dlen] = '/'; - strcpy(file->filename_copy + dlen + 1, parsed[0]); - - file->filename = file->filename_copy; - } - } - - if (! option_quiet) - { - XPR(NT "using default %s filename: %s\n", type, file->filename); - } - } - - /* Set the compressor, initiate de/recompression later. */ - if (file->compressor == NULL && *parsed[1] != 0) - { - file->compressor = main_get_compressor (parsed[1]); - } -} - -static void -main_get_appheader (xd3_stream *stream, main_file *ifile, main_file *output, main_file *sfile) -{ - uint8_t *apphead; - usize_t appheadsz; - int ret; - - /* The user may disable the application header. Once the appheader is set, this - * disables setting it again. */ - if (! option_use_appheader) { return; } - - ret = xd3_get_appheader (stream, & apphead, & appheadsz); - - /* Ignore failure, it only means we haven't received a header yet. */ - if (ret != 0) { return; } - - if (appheadsz > 0) - { - char *start = (char*)apphead; - char *slash; - int place = 0; - char *parsed[4]; - - memset (parsed, 0, sizeof (parsed)); - - while ((slash = strchr (start, '/')) != NULL) - { - *slash = 0; - parsed[place++] = start; - start = slash + 1; - } - - parsed[place++] = start; - - /* First take the output parameters. */ - if (place == 2 || place == 4) - { - main_get_appheader_params (output, parsed, 1, "output", ifile); - } - - /* Then take the source parameters. */ - if (place == 4) - { - main_get_appheader_params (sfile, parsed+2, 0, "source", ifile); - } - } - - option_use_appheader = 0; - return; -} - -/****************************************************************************************** - Main I/O routines - ******************************************************************************************/ - -/* This function acts like the above except it may also try to recognize a compressed - * input when the first buffer of data is read. The EXTERNAL_COMPRESSION code is called - * to search for magic numbers. */ -static int -main_read_primary_input (main_file *ifile, - uint8_t *buf, - usize_t size, - usize_t *nread) -{ -#if EXTERNAL_COMPRESSION - if (option_decompress_inputs && ifile->flags & RD_FIRST) - { - ifile->flags &= ~RD_FIRST; - - return main_decompress_input_check (ifile, buf, size, nread); - } -#endif - - return main_file_read (ifile, buf, size, nread, "input read failed"); -} - -/* This function simply writes the stream output buffer, if there is any. This is used - * for both encode and decode commands. (The VCDIFF tools use main_print_func()). */ -static int -main_write_output (xd3_stream* stream, main_file *ofile) -{ - int ret; - - if (stream->avail_out > 0 && (ret = main_file_write (ofile, stream->next_out, stream->avail_out, "write failed"))) - { - return ret; - } - - return 0; -} - -/* Open the main output file, sets a default file name, initiate recompression. This - * function is expected to fprint any error messages. */ -static int -main_open_output (xd3_stream *stream, main_file *ofile) -{ - int ret; - - if (ofile->filename == NULL) - { - XSTDOUT_XF (ofile); - - if (option_verbose > 1) { XPR(NT "using standard output: %s\n", ofile->filename); } - } - else - { - /* Stat the file to check for overwrite. */ - if (option_force == 0 && main_file_exists (ofile)) - { - XPR(NT "to overwrite output file specify -f: %s\n", ofile->filename); - return EEXIST; - } - - if ((ret = main_file_open (ofile, ofile->filename, XO_WRITE))) - { - return ret; - } - - if (option_verbose > 1) { XPR(NT "open output: %s\n", ofile->filename); } - } - -#if EXTERNAL_COMPRESSION - /* Do output recompression. */ - if (ofile->compressor != NULL && option_recompress_outputs == 1) - { - if (! option_quiet) - { - XPR(NT "%s %s | %s\n", - ofile->compressor->recomp_cmdname, - ofile->compressor->recomp_options, - ofile->filename); - } - - if ((ret = main_recompress_output (ofile))) - { - return ret; - } - } -#endif - - return 0; -} - -/* This is called at different times for encoding and decoding. The encoder calls it - * immediately, the decoder delays until the application header is received. */ -static int -main_set_source (xd3_stream *stream, int cmd, main_file *sfile, xd3_source *source) -{ - int ret, i; - - /* Open it, check for seekability, set required xd3_source fields. */ - if (allow_fake_source) - { - sfile->mode = XO_READ; - sfile->realname = sfile->filename; - sfile->nread = 0; - source->size = UINT64_MAX; - } - else if ((ret = main_file_open (sfile, sfile->filename, XO_READ)) || - (ret = main_file_stat (sfile, & source->size, 1))) - { - return ret; - } - - source->name = sfile->filename; - source->ioh = sfile; - source->curblkno = (xoff_t) -1; - source->curblk = NULL; - - /* Source block LRU init. */ - main_blklru_list_init (& lru_list); - main_blklru_list_init (& lru_free); - - option_srcwinsz = min(source->size, (xoff_t) option_srcwinsz); - - if (option_verbose > 1) { XPR(NT "source window size: %u\n", option_srcwinsz); } - if (option_verbose > 1) { XPR(NT "source block size: %u\n", source->blksize); } - if (option_verbose > 1) { XPR(NT "memory size: %u\n", option_memsize); } - - lru_size = (option_srcwinsz / source->blksize); - lru_size = max(1, lru_size); - - if ((lru = main_malloc (sizeof (main_blklru) * lru_size)) == NULL) - { - return ENOMEM; - } - - for (i = 0; i < lru_size; i += 1) - { - lru[i].blkno = (xoff_t) -1; - - if ((lru[i].blk = main_malloc (source->blksize)) == NULL) - { - return ENOMEM; - } - - main_blklru_list_push_back (& lru_free, & lru[i]); - } - -#if EXTERNAL_COMPRESSION - if (option_decompress_inputs) - { - if (IS_ENCODE (cmd)) - { - usize_t nread; - - source->curblk = lru[0].blk; - - /* If encoding, read the first block now to check for decompression. */ - if ((ret = main_file_read (sfile, (uint8_t*) source->curblk, source->blksize, & nread, "source read failed"))) - { - return ret; - } - - /* Check known magic numbers. */ - for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) - { - const main_extcomp *decomp = & extcomp_types[i]; - - if ((nread > decomp->magic_size) && memcmp (source->curblk, decomp->magic, decomp->magic_size) == 0) - { - sfile->compressor = decomp; - break; - } - } - - /* If no decompression, the current buffer is now a valid source->curblock. */ - if (sfile->compressor == NULL) - { - main_blklru_list_remove (& lru[0]); - main_blklru_list_push_back (& lru_list, & lru[0]); - - lru[0].blkno = 0; - source->curblkno = 0; - source->onblk = nread; - - if (option_verbose > 2) - { - XPR(NT "source block 0 read (not compressed)\n"); - } - } - } - - /* In either the encoder or decoder, start decompression. */ - if (sfile->compressor) - { - xoff_t osize = source->size; - - if (osize > XD3_NODECOMPRESSSIZE) - { - XPR(NT "source file too large for external decompression: %s: %"Q"u\n", - sfile->filename, osize); - return XD3_INTERNAL; - } - - if ((ret = main_decompress_source (sfile, source))) - { - return ret; - } - - if (! option_quiet) - { - char s1[32], s2[32]; - XPR(NT "%s | %s %s => %s %.1f%% [ %s , %s ]\n", - sfile->filename, - sfile->compressor->decomp_cmdname, - sfile->compressor->decomp_options, - sfile->realname, - 100.0 * source->size / osize, - main_format_bcnt (osize, s1), - main_format_bcnt (source->size, s2)); - } - } - } -#endif - - if (option_verbose > 1) { XPR(NT "source file: %s: %"Q"u bytes\n", sfile->realname, source->size); } - - if ((ret = xd3_set_source (stream, source))) - { - XPR(NT XD3_LIB_ERRMSG (stream, ret)); - return EXIT_FAILURE; - } - - return 0; -} - -/****************************************************************************************** - Source routines - ******************************************************************************************/ - -/* This is the callback for reading a block of source. This function is blocking and it - * implements a small LRU. - * - * Note that it is possible for main_input() to handle getblk requests in a non-blocking - * manner. If the callback is NULL then the caller of xd3_*_input() must handle the - * XD3_GETSRCBLK return value and fill the source in the same way. See xd3_getblk for - * details. To see an example of non-blocking getblk, see xdelta-test.h. */ -static int -main_getblk_func (xd3_stream *stream, - xd3_source *source, - xoff_t blkno) -{ - xoff_t pos = blkno * source->blksize; - main_file *sfile = (main_file*) source->ioh; - main_blklru *blru = NULL; - usize_t onblk = xd3_bytes_on_srcblk (source, blkno); - usize_t nread; - int ret; - int i; - - if (allow_fake_source) - { - source->curblkno = blkno; - source->onblk = onblk; - source->curblk = lru[0].blk; - return 0; - } - - if (do_not_lru) - { - /* Direct lookup assumes sequential scan w/o skipping blocks. */ - int idx = blkno % lru_size; - if (lru[idx].blkno == blkno) - { - source->curblkno = blkno; - source->onblk = onblk; - source->curblk = lru[idx].blk; - lru_hits += 1; - return 0; - } - - if (lru[idx].blkno != -1LL && - lru[idx].blkno != blkno - lru_size) - { - return XD3_TOOFARBACK; - } - } - else - { - /* Sequential search through LRU. */ - for (i = 0; i < lru_size; i += 1) - { - if (lru[i].blkno == blkno) - { - main_blklru_list_remove (& lru[i]); - main_blklru_list_push_back (& lru_list, & lru[i]); - - source->curblkno = blkno; - source->onblk = onblk; - source->curblk = lru[i].blk; - lru_hits += 1; - return 0; - } - } - } - - if (! main_blklru_list_empty (& lru_free)) - { - blru = main_blklru_list_pop_front (& lru_free); - } - else if (! main_blklru_list_empty (& lru_list)) - { - if (do_not_lru) { - blru = & lru[blkno % lru_size]; - main_blklru_list_remove(blru); - } else { - blru = main_blklru_list_pop_front (& lru_list); - } - lru_misses += 1; - } - - lru_filled += 1; - - if ((ret = main_file_seek (sfile, pos))) - { - return ret; - } - - if ((ret = main_file_read (sfile, (uint8_t*) blru->blk, source->blksize, - & nread, "source read failed"))) - { - return ret; - } - - if (nread != onblk) - { - XPR(NT "source file size change: %s\n", sfile->filename); - return XD3_INTERNAL; - } - - main_blklru_list_push_back (& lru_list, blru); - - if (option_verbose > 2) - { - if (blru->blkno != -1LL) - { - XPR(NT "source block %"Q"u ejects %"Q"u (lru_hits=%u, lru_misses=%u, lru_filled=%u)\n", - blkno, blru->blkno, lru_hits, lru_misses, lru_filled); - } - else - { - XPR(NT "source block %"Q"u read (lru_hits=%u, lru_misses=%u, lru_filled=%u)\n", - blkno, lru_hits, lru_misses, lru_filled); - } - } - - blru->blkno = blkno; - source->curblk = blru->blk; - source->curblkno = blkno; - source->onblk = onblk; - - return 0; -} - -/****************************************************************************************** - Main routines - ******************************************************************************************/ - -/* This is a generic input function. It calls the xd3_encode_input or xd3_decode_input - * functions and makes calls to the various input handling routines above, which - * coordinate external decompression. - */ -static int -main_input (xd3_cmd cmd, - main_file *ifile, - main_file *ofile, - main_file *sfile) -{ - int ret; - xd3_stream stream; - usize_t nread; - int stream_flags = 0; - xd3_config config; - xd3_source source; - xoff_t last_total_in = 0; - xoff_t last_total_out = 0; - long start_time; - xoff_t input_size = 0; - - int (*input_func) (xd3_stream*); - int (*output_func) (xd3_stream*, main_file *); - - memset (& source, 0, sizeof (source)); - memset (& config, 0, sizeof (config)); - - config.alloc = main_alloc; - config.freef = main_free1; - config.sec_data.ngroups = 1; - config.sec_addr.ngroups = 1; - config.sec_inst.ngroups = 1; - - /* main_input setup. */ - switch ((int) cmd) - { -#if VCDIFF_TOOLS - if (1) { case CMD_PRINTHDR: stream_flags = XD3_JUST_HDR; } - else if (1) { case CMD_PRINTHDRS: stream_flags = XD3_SKIP_WINDOW; } - else { case CMD_PRINTDELTA: stream_flags = XD3_SKIP_EMIT; } - ifile->flags |= RD_NONEXTERNAL; - input_func = xd3_decode_input; - output_func = main_print_func; - stream_flags |= XD3_ADLER32_NOVER; - break; -#endif -#if XD3_ENCODER - case CMD_ENCODE: - input_func = xd3_encode_input; - output_func = main_write_output; - - if (option_use_checksum) { stream_flags |= XD3_ADLER32; } - if (option_use_secondary) - { - /* The default secondary compressor is DJW, if it's compiled, being used, etc. */ - if (option_secondary == NULL) - { - if (SECONDARY_DJW) { stream_flags |= XD3_SEC_DJW; } - } - else - { - if (strcmp (option_secondary, "fgk") == 0 && SECONDARY_FGK) - { - stream_flags |= XD3_SEC_FGK; - } - else if (strcmp (option_secondary, "djw") == 0 && SECONDARY_DJW) - { - stream_flags |= XD3_SEC_DJW; - } - else - { - XPR(NT "unrecognized secondary compressor type: %s\n", option_secondary); - return EXIT_FAILURE; - } - } - } - if (option_no_compress) { stream_flags |= XD3_NOCOMPRESS; } - if (option_use_altcodetable) { stream_flags |= XD3_ALT_CODE_TABLE; } - if (option_smatch_config) - { - char *s = option_smatch_config, *e; - int values[XD3_SOFTCFG_VARCNT]; - int got; - - config.smatch_cfg = XD3_SMATCH_SOFT; - - for (got = 0; got < XD3_SOFTCFG_VARCNT; got += 1, s = e + 1) - { - values[got] = strtol (s, &e, 10); - - if ((values[got] < 0) || - (e == s) || - (got < XD3_SOFTCFG_VARCNT-1 && *e == 0) || - (got == XD3_SOFTCFG_VARCNT-1 && *e != 0)) - { - XPR(NT "invalid string match specifier (-C)\n"); - return EXIT_FAILURE; - } - } - - config.smatcher_soft.large_look = values[0]; - config.smatcher_soft.large_step = values[1]; - config.smatcher_soft.small_look = values[2]; - config.smatcher_soft.small_chain = values[3]; - config.smatcher_soft.small_lchain = values[4]; - config.smatcher_soft.ssmatch = values[5]; - config.smatcher_soft.try_lazy = values[6]; - config.smatcher_soft.max_lazy = values[7]; - config.smatcher_soft.long_enough = values[8]; - config.smatcher_soft.promote = values[9]; - } - else if (option_level < 5) { config.smatch_cfg = XD3_SMATCH_FAST; } - else { config.smatch_cfg = XD3_SMATCH_SLOW; } - break; -#endif - case CMD_DECODE: - if (option_use_checksum == 0) { stream_flags |= XD3_ADLER32_NOVER; } - stream_flags = 0; - ifile->flags |= RD_NONEXTERNAL; - input_func = xd3_decode_input; - output_func = main_write_output; - break; - default: - XPR(NT "internal error\n"); - return EXIT_FAILURE; - } - - start_time = get_millisecs_now (); - - if (main_file_stat (ifile, & input_size, 0) == 0) - { - option_winsize = min (input_size, (xoff_t) option_winsize); - } - - option_srcwinsz = max(option_srcwinsz, XD3_ALLOCSIZE); - option_winsize = max(option_winsize, XD3_ALLOCSIZE); - - source.blksize = (option_srcwinsz / 32) & ~(XD3_ALLOCSIZE-1); - source.blksize = max(XD3_DEFAULT_WINSIZE, source.blksize); - - config.srcwin_maxsz = option_srcwinsz; - config.winsize = option_winsize; - config.memsize = option_memsize; - config.getblk = main_getblk_func; - config.flags = stream_flags; - - if (option_verbose > 1) - { - XPR(NT "input buffer size: %u\n", option_winsize); - } - - if ((main_bdata = main_malloc (option_winsize)) == NULL) - { - return EXIT_FAILURE; - } - - if ((ret = xd3_config_stream (& stream, & config))) - { - XPR(NT XD3_LIB_ERRMSG (& stream, ret)); - return EXIT_FAILURE; - } - - if (IS_ENCODE (cmd)) - { - /* When encoding, open the source file, possibly decompress it. The decoder delays - * this step until XD3_GOTHEADER. */ - if (sfile->filename != NULL && (ret = main_set_source (& stream, cmd, sfile, & source))) - { - return EXIT_FAILURE; - } - } - - /* This times each window. */ - get_millisecs_since (); - - /* Main input loop. */ - do - { - xoff_t input_offset; - xoff_t input_remain; - usize_t try_read; - - input_offset = ifile->nread; - /*XD3_ASSERT (input_offset <= option_last_offset);*/ - - input_remain = /*option_last_offset*/ XOFF_T_MAX - input_offset; - - try_read = (usize_t) min ((xoff_t) config.winsize, input_remain); - - if ((ret = main_read_primary_input (ifile, main_bdata, try_read, & nread))) - { - return EXIT_FAILURE; - } - - /* If we've reached EOF tell the stream to flush. */ - if (nread < try_read) - { - stream_flags |= XD3_FLUSH; - xd3_set_flags (& stream, stream_flags); - } - -#if XD3_ENCODER - /* After the first main_read_primary_input completes, we know all the information - * needed to encode the application header. */ - if (cmd == CMD_ENCODE && (ret = main_set_appheader (& stream, ifile, sfile))) - { - return EXIT_FAILURE; - } -#endif - xd3_avail_input (& stream, main_bdata, nread); - - /* If we read zero bytes after encoding at least one window... */ - if (nread == 0 && stream.current_window > 0) { - break; - } - - again: - ret = input_func (& stream); - /*if (option_verbose > 1) { XPR(NT XD3_LIB_ERRMSG (& stream, ret)); }*/ - - switch (ret) - { - case XD3_INPUT: - continue; - - case XD3_GOTHEADER: - { - XD3_ASSERT (stream.current_window == 0); - - /* Need to process the appheader as soon as possible. It may contain a - * suggested default filename/decompression routine for the ofile, and it may - * contain default/decompression routine for the sources. */ - if (cmd == CMD_DECODE) - { - int have_src = sfile->filename != NULL; - int need_src = xd3_decoder_needs_source (& stream); - int recv_src; - - /* May need to set the sfile->filename if none was given. */ - main_get_appheader (& stream, ifile, ofile, sfile); - - recv_src = sfile->filename != NULL; - - /* Check if the user expected a source to be required although it was not. */ - if (have_src && ! need_src && ! option_quiet) - { - XPR(NT "warning: output window %"Q"u does not copy source\n", stream.current_window); - } - - /* Check if we have no source name and need one. */ - /* TODO: this doesn't fire due to cpyblocks_ calculation check */ - if (need_src && ! recv_src) - { - XPR(NT "input requires a source file, use -s\n"); - return EXIT_FAILURE; - } - - /* Now open the source file. */ - if (need_src && (ret = main_set_source (& stream, cmd, sfile, & source))) - { - return EXIT_FAILURE; - } - } - else if (cmd == CMD_PRINTHDR || - cmd == CMD_PRINTHDRS || - cmd == CMD_PRINTDELTA) - { - if (xd3_decoder_needs_source (& stream) && sfile->filename == NULL) - { - allow_fake_source = 1; - sfile->filename = ""; - main_set_source (& stream, cmd, sfile, & source); - } - } - } - /* FALLTHROUGH */ - case XD3_WINSTART: - { - /* Set or unset XD3_SKIP_WINDOW. */ - /*if (stream.current_window < option_first_window || stream.current_window > option_last_window) - { stream_flags |= XD3_SKIP_WINDOW; } - else - { stream_flags &= ~XD3_SKIP_WINDOW; }*/ - - xd3_set_flags (& stream, stream_flags); - goto again; - } - - case XD3_OUTPUT: - { - if (option_no_output == 0/* && - stream.current_window >= option_first_window && - stream.current_window <= option_last_window*/) - { - /* Defer opening the output file until the stream produces its first - * output for both encoder and decoder, this way we delay long enough for - * the decoder to receive the application header. (Or longer if there are - * skipped windows, but I can't think of any reason not to delay open.) */ - - if (! main_file_isopen (ofile) && (ret = main_open_output (& stream, ofile)) != 0) - { - return EXIT_FAILURE; - } - if ((ret = output_func (& stream, ofile)) && (ret != PRINTHDR_SPECIAL)) - { - return EXIT_FAILURE; - } - if (ret == PRINTHDR_SPECIAL) - { - xd3_abort_stream (& stream); - ret = EXIT_SUCCESS; - goto done; - } - ret = 0; - } - - xd3_consume_output (& stream); - goto again; - } - - case XD3_WINFINISH: - { - if (IS_ENCODE (cmd) || cmd == CMD_DECODE) - { - int used_source = xd3_encoder_used_source (& stream); - - if (! option_quiet && IS_ENCODE (cmd) && main_file_isopen (sfile) && ! used_source) - { - XPR(NT "warning: input position %"Q"u no source copies\n", - stream.current_window * option_winsize); - } - - if (option_verbose) - { - char rrateavg[32], wrateavg[32], tm[32]; - char rdb[32], wdb[32]; - char trdb[32], twdb[32]; - long millis = get_millisecs_since (); - usize_t this_read = stream.total_in - last_total_in; - usize_t this_write = stream.total_out - last_total_out; - last_total_in = stream.total_in; - last_total_out = stream.total_out; - - if (option_verbose > 1) - { - XPR(NT "%"Q"u: in %s (%s): out %s (%s): total in %s: out %s: %s\n", - stream.current_window, - main_format_bcnt (this_read, rdb), - main_format_rate (this_read, millis, rrateavg), - main_format_bcnt (this_write, wdb), - main_format_rate (this_write, millis, wrateavg), - main_format_bcnt (stream.total_in, trdb), - main_format_bcnt (stream.total_out, twdb), - main_format_millis (millis, tm)); - } - else - { - XPR(NT "%"Q"u: in %s: out %s: total in %s: out %s: %s\n", - stream.current_window, - main_format_bcnt (this_read, rdb), - main_format_bcnt (this_write, wdb), - main_format_bcnt (stream.total_in, trdb), - main_format_bcnt (stream.total_out, twdb), - main_format_millis (millis, tm)); - } - } - } - goto again; - } - - default: - /* input_func() error */ - XPR(NT XD3_LIB_ERRMSG (& stream, ret)); - return EXIT_FAILURE; - } - } - while (nread == config.winsize); -done: - /* Close the inputs. (ifile must be open, sfile may be open) */ - main_file_close (ifile); - main_file_close (sfile); - -#if EXTERNAL_COMPRESSION - if ((ret = main_external_compression_finish ())) - { - XPR(NT "external compression commands failed\n"); - return EXIT_FAILURE; - } -#endif - - /* If output file is not open yet because of delayed-open, it means we never encountered - * a window in the delta, but it could have had a VCDIFF header? TODO: solve this - * elsewhere. For now, it prints "nothing to output" below, but the check doesn't - * happen in case of option_no_output. */ - if (! option_no_output) - { - if (! main_file_isopen (ofile)) - { - XPR(NT "nothing to output: %s\n", ifile->filename); - return EXIT_FAILURE; - } - - /* Have to close the output before calling main_external_compression_finish, or else it hangs. */ - if (main_file_close (ofile) != 0) - { - return EXIT_FAILURE; - } - } - - if ((ret = xd3_close_stream (& stream))) - { - XPR(NT XD3_LIB_ERRMSG (& stream, ret)); - return EXIT_FAILURE; - } - - xd3_free_stream (& stream); - - if (option_verbose) - { - char tm[32]; - long end_time = get_millisecs_now (); - XPR(NT "command finished in %s\n", main_format_millis (end_time - start_time, tm)); - XPR(NT "input bytes: %"Q"u\n", ifile->nread); - XPR(NT "output bytes: %"Q"u\n", ofile->nwrite); - } - - return EXIT_SUCCESS; -} - -/* free memory before exit, reset single-use variables. */ -static void -main_cleanup (void) -{ - int i; - - if (option_appheader) { appheader_used = NULL; } - - main_free (appheader_used); - main_free (main_bdata); - -#if EXTERNAL_COMPRESSION - main_free (ext_tmpfile); -#endif - - for (i = 0; lru && i < lru_size; i += 1) - { - main_free (lru[i].blk); - } - - main_free (lru); - - lru_hits = 0; - lru_misses = 0; - lru_filled = 0; - - XD3_ASSERT (main_mallocs == 0); -} - -int -#if PYTHON_MODULE -xd3_main_cmdline (int argc, char **argv) -#else -main (int argc, char **argv) -#endif -{ - xd3_cmd cmd; - main_file ifile; - main_file ofile; - main_file sfile; - static char *flags = "0123456789cdefhnqvDJNORTVs:B:C:E:F:L:O:P:M:W:A::S::"; - int my_optind; - char *my_optarg; - char *my_optstr; - char *sfilename; - int orig_argc = argc; - char **orig_argv = argv; - int ret; - - main_file_init (& ifile); - main_file_init (& ofile); - main_file_init (& sfile); - - go: /* Go. */ - cmd = CMD_NONE; - sfilename = NULL; - my_optind = 1; - argv = orig_argv; - argc = orig_argc; - program_name = argv[0]; - extcomp_types[0].recomp_cmdname = program_name; - extcomp_types[0].decomp_cmdname = program_name; - takearg: - my_optarg = NULL; - my_optstr = argv[my_optind]; - /* This doesn't use getopt() because it makes trouble for -P & python which reenter - * main() and thus care about freeing all memory. I never had much trust for getopt - * anyway, it's too opaque. This implements a fairly standard non-long-option getopt - * with support for named operations (e.g., "xdelta3 [encode|decode|printhdr...] < in > - * out"). I'll probably add long options at some point. */ - if (my_optstr) - { - if (*my_optstr == '-') { my_optstr += 1; } - else if (cmd == CMD_NONE) { goto nonflag; } - else { my_optstr = NULL; } - } - while (my_optstr) - { - char *s; - my_optarg = NULL; - if ((ret = *my_optstr++) == 0) { my_optind += 1; goto takearg; } - - /* Option handling: first check for one ':' following the option in flags, then - * check for two. The syntax allows: - * - * 1. -Afoo defines optarg="foo" - * 2. -A foo defines optarg="foo" - * 3. -A "" defines optarg="" (allows optional empty-string) - * 4. -A [EOA or -moreargs] error (mandatory case) - * 5. -A [EOA -moreargs] defines optarg=NULL (optional case) - * 6. -A=foo defines optarg="foo" - * 7. -A= defines optarg="" (mandatory case) - * 8. -A= defines optarg=NULL (optional case) - * - * See tests in test_command_line_arguments(). - */ - s = strchr (flags, ret); - if (s && s[1] && s[1] == ':') - { - int eqcase = 0; - int option = s[2] && s[2] == ':'; - - /* Case 1, set optarg to the remaining characters. */ - my_optarg = my_optstr; - my_optstr = ""; - - /* Case 2-5 */ - if (*my_optarg == 0) - { - /* Condition 4-5 */ - int have_arg = my_optind < (argc - 1) && *argv[my_optind+1] != '-'; - - if (! have_arg) - { - if (! option) - { - /* Case 4 */ - XPR(NT "-%c: requires an argument\n", ret); - ret = EXIT_FAILURE; - goto cleanup; - } - /* Case 5. */ - my_optarg = NULL; - } - else - { - /* Case 2-3. */ - my_optarg = argv[++my_optind]; - } - } - /* Case 6-8. */ - else if (*my_optarg == '=') - { - /* Remove the = in all cases. */ - my_optarg += 1; - eqcase = 1; - - if (option && *my_optarg == 0) - { - /* Case 8. */ - my_optarg = NULL; - } - } - } - - switch (ret) - { - /* case: if no '-' was found, maybe check for a command name. */ - nonflag: - if (strcmp (my_optstr, "decode") == 0) { cmd = CMD_DECODE; } - else if (strcmp (my_optstr, "encode") == 0) - { -#if XD3_ENCODER - cmd = CMD_ENCODE; -#else - XPR(NT "encoder support not compiled\n"); - return EXIT_FAILURE; -#endif - } - else if (strcmp (my_optstr, "config") == 0) { cmd = CMD_CONFIG; } -#if REGRESSION_TEST - else if (strcmp (my_optstr, "test") == 0) { cmd = CMD_TEST; } -#endif -#if VCDIFF_TOOLS - else if (strcmp (my_optstr, "printhdr") == 0) { cmd = CMD_PRINTHDR; } - else if (strcmp (my_optstr, "printhdrs") == 0) { cmd = CMD_PRINTHDRS; } - else if (strcmp (my_optstr, "printdelta") == 0) { cmd = CMD_PRINTDELTA; } -#endif - - /* If no option was found and still no command, let the default command be - * encode. The remaining args are treated as filenames. */ - if (cmd == CMD_NONE) - { - cmd = CMD_DEFAULT; - my_optstr = NULL; - break; - } - else - { - /* But if we find a command name, continue the getopt loop. */ - my_optind += 1; - goto takearg; - } - - /* gzip-like options */ - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - option_level = ret - '0'; - break; - case 'f': option_force = 1; break; - case 'v': option_verbose += 1; option_quiet = 0; break; - case 'q': option_quiet = 1; option_verbose = 0; break; - case 'c': option_stdout = 1; break; - case 'd': - if (cmd == CMD_NONE) { cmd = CMD_DECODE; } - else { ret = main_help (); goto exit; } - break; - case 'e': -#if XD3_ENCODER - if (cmd == CMD_NONE) { cmd = CMD_ENCODE; } - else { ret = main_help (); goto exit; } - break; -#else - XPR(NT "encoder support not compiled\n"); - return EXIT_FAILURE; -#endif - case 'P': - /* only set profile count once, since... */ - if (option_profile_cnt == 0) - { - if ((ret = main_atou(my_optarg, (usize_t*) & option_profile_cnt, 0, 'P'))) { goto exit; } - - if (option_profile_cnt <= 0) - { - ret = EXIT_SUCCESS; - goto exit; - } - } - break; - - case 'n': option_use_checksum = 0; break; - case 'N': option_no_compress = 1; break; - case 'T': option_use_altcodetable = 1; break; - case 'C': option_smatch_config = my_optarg; break; - case 'J': option_no_output = 1; break; - case 'O': option_xdelta1 = my_optarg; break; - case 'S': if (my_optarg == NULL) { option_use_secondary = 0; } - else { option_use_secondary = 1; option_secondary = my_optarg; } break; - case 'A': if (my_optarg == NULL) { option_use_appheader = 0; } - else { option_appheader = (uint8_t*) my_optarg; } break; - case 'B': if ((ret = main_atou (my_optarg, & option_srcwinsz, XD3_ALLOCSIZE, 'B'))) - { - goto exit; - } - break; - case 'W': if ((ret = main_atou (my_optarg, & option_winsize, XD3_ALLOCSIZE, 'W'))) - { - goto exit; - } - break; - case 'M': if ((ret = main_atou (my_optarg, & option_memsize, XD3_ALLOCSIZE, 'M'))) - { - goto exit; - } - break; - case 'D': -#if EXTERNAL_COMPRESSION == 0 - if (! option_quiet) - { - XPR(NT "warning: -D option ignored, " - "external compression support was not compiled\n"); - } -#else - option_decompress_inputs = 0; -#endif - break; - case 'R': -#if EXTERNAL_COMPRESSION == 0 - if (! option_quiet) - { - XPR(NT "warning: -R option ignored, " - "external compression support was not compiled\n"); - } -#else - option_recompress_outputs = 0; -#endif - break; - case 's': - if (sfilename != NULL) - { - XPR(NT "specify only one source file\n"); - goto cleanup; - } - - sfilename = my_optarg; - break; - - case 'V': - ret = main_version (); goto exit; - default: - ret = main_help (); goto exit; - } - } - - option_source_filename = sfilename; - - /* In case there were no arguments, set the default command. */ - if (cmd == CMD_NONE) { cmd = CMD_DEFAULT; } - - argc -= my_optind; - argv += my_optind; - - /* There may be up to two more arguments. */ - if (argc > 2) - { - XPR(NT "too many filenames: %s ...\n", argv[2]); - ret = EXIT_FAILURE; - goto cleanup; - } - - extcomp_types[1].recomp_cmdname = option_xdelta1; - extcomp_types[1].decomp_cmdname = option_xdelta1; - - if (option_verbose > 1) - { - int l = 1; - int i; - char buf[1024]; - for (i = 0; i < orig_argc; i += 1) - { - l += strlen (orig_argv[i]) + 1; - } - buf[0] = 0; - for (i = 0; i < orig_argc; i += 1) - { - strcat (buf, orig_argv[i]); - strcat (buf, " "); - } - XPR(NT "command line: %s\n", buf); - } - - ifile.flags = RD_FIRST; - sfile.flags = RD_FIRST; - sfile.filename = option_source_filename; - - /* The infile takes the next argument, if there is one. But if not, infile is set to - * stdin. */ - if (argc > 0) - { - ifile.filename = argv[0]; - - if ((ret = main_file_open (& ifile, ifile.filename, XO_READ))) - { - goto cleanup; - } - } - else - { - XSTDIN_XF (& ifile); - } - - /* The ofile takes the following argument, if there is one. But if not, it is left NULL - * until the application header is processed. It will be set in main_open_output. */ - if (argc > 1) - { - /* Check for conflicting arguments. */ - if (option_stdout && ! option_quiet) - { - XPR(NT "warning: -c option overrides output filename: %s\n", argv[1]); - } - - if (! option_stdout) { ofile.filename = argv[1]; } - } - - switch (cmd) - { - case CMD_PRINTHDR: - case CMD_PRINTHDRS: - case CMD_PRINTDELTA: -#if XD3_ENCODER - case CMD_ENCODE: - if (cmd == CMD_ENCODE) - { - do_not_lru = 1; - } -#endif - case CMD_DECODE: - ret = main_input (cmd, & ifile, & ofile, & sfile); - break; - -#if REGRESSION_TEST - case CMD_TEST: - ret = xd3_selftest (); - break; -#endif - - case CMD_CONFIG: - ret = main_config (); - break; - - default: - ret = main_help (); - break; - } - -#if EXTERNAL_COMPRESSION - if (ext_tmpfile != NULL) { unlink (ext_tmpfile); } -#endif - - if (0) - { - cleanup: - ret = EXIT_FAILURE; - exit: - (void)0; - } - - main_file_cleanup (& ifile); - main_file_cleanup (& ofile); - main_file_cleanup (& sfile); - - main_cleanup (); - - if (--option_profile_cnt > 0 && ret == EXIT_SUCCESS) { goto go; } - - return ret; -} - -static int -main_help (void) -{ - /* TODO: update www/xdelta3-cmdline.html */ - main_version (); - P(RINT "usage: xdelta3 [command/options] [input [output]]\n"); - P(RINT "special command names:\n"); - P(RINT " config prints xdelta3 configuration\n"); - P(RINT " decode decompress the input\n"); - P(RINT " encode compress the input%s\n", XD3_ENCODER ? "" : " [Not compiled]"); -#if REGRESSION_TEST - P(RINT " test run the builtin tests\n"); -#endif -#if VCDIFF_TOOLS - P(RINT "special commands for VCDIFF inputs:\n"); - P(RINT " printdelta print information about the entire delta\n"); - P(RINT " printhdr print information about the first window\n"); - P(RINT " printhdrs print information about all windows\n"); -#endif - P(RINT "standard options:\n"); - P(RINT " -0 .. -9 compression level\n"); - P(RINT " -c use stdout\n"); - P(RINT " -d decompress\n"); - P(RINT " -e compress%s\n", XD3_ENCODER ? "" : " [Not compiled]"); - P(RINT " -f force overwrite\n"); - P(RINT " -h show help\n"); - P(RINT " -q be quiet\n"); - P(RINT " -v be verbose (max 2)\n"); - P(RINT " -V show version\n"); - - P(RINT "memory options:\n"); - P(RINT " -B blksize source file block size\n"); - P(RINT " -M memsize memory budget for hash tables\n"); - P(RINT " -W winsize input window buffer size\n"); - - P(RINT "compression options:\n"); - P(RINT " -s source source file to copy from (if any)\n"); - P(RINT " -S [djw|fgk] enable/disable secondary compression\n"); - P(RINT " -N disable small string-matching compression\n"); - P(RINT " -D disable external decompression (encode/decode)\n"); - P(RINT " -R disable external recompression (decode)\n"); - -#if XD3_DEBUG > 0 - P(RINT "developer options:\n"); - P(RINT " -A [apphead] disable/provide application header\n"); - P(RINT " -C soft config (see xdelta3-cfgs.h)\n"); - P(RINT " -J disable output (check/compute only)\n"); - P(RINT " -P repeat count (for profiling)\n"); - P(RINT " -T use alternate code table\n"); - P(RINT " -n disable checksum (encode/decode)\n"); -#endif - return EXIT_FAILURE; -} +/* xdelta 3 - delta compression tools and library + * Copyright (C) 2001 and onward. Joshua P. MacDonald + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* This is all the extra stuff you need for convenience to users in a command line + * application. It contains these major components: + * + * 1. VCDIFF tools + * 2. external compression support (this is POSIX-specific). + * 3. a general read/write loop that handles all of the Xdelta decode/encode/VCDIFF-print + * functions + * 4. command-line interpreter + * 5. an Xdelta application header which stores default filename, external compression settings + * 6. output/error printing + * 7. basic file support and OS interface + */ + +/* TODO list: + * 1. do exact gzip-like filename, stdout handling. make a .xz extension, refuse + * to encode to stdout without -cf, etc. + * 2. Allow the user to add a comment string to the app header without disturbing the default + * behavior. + */ + +/* On error handling and printing: + * + * The xdelta library sets stream->msg to indicate what condition caused an internal + * failure, but many failures originate here and are printed here. The return convention + * is 0 for success, as throughout Xdelta code, but special attention is required here for + * the operating system calls with different error handling. See the main_file_* routines. + * All errors in this file have a message printed at the time of occurance. Since some of + * these calls occur within calls to the library, the error may end up being printed again + * with a more general error message. + */ + +/******************************************************************************************/ + +#ifndef XD3_POSIX +#define XD3_POSIX 0 +#endif +#ifndef XD3_STDIO +#define XD3_STDIO 0 +#endif + +/* XPRINTX (used by main) prefixes an "xdelta3: " to the output. */ +#define XPR fprintf +#define NT stderr, "xdelta3: " + +#define VC fprintf +#define UT vcout, + +/* If none are set, default to posix. */ +#if (XD3_POSIX + XD3_STDIO) == 0 +#undef XD3_POSIX +#define XD3_POSIX 1 +#endif + +/* Handle externally-compressed inputs. */ +#ifndef EXTERNAL_COMPRESSION +#define EXTERNAL_COMPRESSION 1 +#endif + +#define PRINTHDR_SPECIAL -4378291 + +/* The number of soft-config variables. */ +#define XD3_SOFTCFG_VARCNT 1 + +/* this is used as in XPR(NT XD3_LIB_ERRMSG (stream, ret)) to print an error message + * from the library. */ +#define XD3_LIB_ERRMSG(stream, ret) "%s: %s\n", xd3_errstring (stream), xd3_strerror (ret) + +#include /* fprintf */ + +#if XD3_POSIX +#include /* close, read, write... */ +#include +#include +#endif + +#ifndef _WIN32 +#include /* lots */ +#include /* gettimeofday() */ +#include /* stat() and fstat() */ +#else +#define strtoll _strtoi64 +#include +#include +#ifndef WIFEXITED +# define WIFEXITED(stat) (((*((int *) &(stat))) & 0xff) == 0) +#endif +#ifndef WEXITSTATUS +# define WEXITSTATUS(stat) (((*((int *) &(stat))) >> 8) & 0xff) +#endif +#ifndef S_ISREG +# ifdef S_IFREG +# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) +# else +# define S_ISREG(m) 1 +# endif +#endif /* !S_ISREG */ +#endif + +/****************************************************************************************** + ENUMS and TYPES + ******************************************************************************************/ + +/* These flags (mainly pertaining to main_read() operations) are set in the + * main_file->flags variable. All are related to with external decompression support. + * + * RD_FIRST causes the external decompression check when the input is first read. + * + * RD_NONEXTERNAL disables external decompression for reading a compressed input, in the + * case of Xdelta inputs. Note: Xdelta is supported as an external compression type, + * which makes is the reason for this flag. An example to justify this is: to create a + * delta between two files that are VCDIFF-compressed. Two external Xdelta decoders are + * run to supply decompressed source and target inputs to the Xdelta encoder. */ +typedef enum +{ + RD_FIRST = (1 << 0), + RD_NONEXTERNAL = (1 << 1), + RD_EXTERNAL_V1 = (1 << 2), +} xd3_read_flags; + +/* main_file->mode values */ +typedef enum +{ + XO_READ = 0, + XO_WRITE = 1, +} main_file_modes; + +/* Main commands. For example, CMD_PRINTHDR is the "xdelta printhdr" command. */ +typedef enum +{ + CMD_NONE = 0, + CMD_PRINTHDR, + CMD_PRINTHDRS, + CMD_PRINTDELTA, +#if XD3_ENCODER + CMD_ENCODE, +#endif + CMD_DECODE, + CMD_TEST, + CMD_CONFIG, +} xd3_cmd; + +#if XD3_ENCODER +#define CMD_DEFAULT CMD_ENCODE +#define IS_ENCODE(cmd) (cmd == CMD_ENCODE) +#else +#define CMD_DEFAULT CMD_DECODE +#define IS_ENCODE(cmd) (0) +#endif + +typedef struct _main_file main_file; +typedef struct _main_extcomp main_extcomp; +typedef struct _main_blklru main_blklru; +typedef struct _main_blklru_list main_blklru_list; + +/* The main_file object supports abstract system calls like open, close, read, write, seek, + * stat. The program uses these to represent both seekable files and non-seekable files. + * Source files must be seekable, but the target input and any output file do not require + * seekability. + */ +struct _main_file +{ +#if XD3_STDIO + FILE *file; +#elif XD3_POSIX + int file; +#endif + + int mode; /* XO_READ and XO_WRITE */ + const char *filename; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */ + char *filename_copy; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */ + const char *realname; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */ + const main_extcomp *compressor; /* External compression struct. */ + int flags; /* RD_FIRST, RD_NONEXTERNAL, ... */ + xoff_t nread; /* for input position */ + xoff_t nwrite; /* for output position */ +}; + +/* Various strings and magic values used to detect and call external compression. See + * below for examples. */ +struct _main_extcomp +{ + const char *recomp_cmdname; + const char *recomp_options; + + const char *decomp_cmdname; + const char *decomp_options; + + const char *ident; + const char *magic; + int magic_size; + int flags; +}; + +/* This file implements a small LRU of source blocks. For encoding purposes, + * we prevent paging in blocks we've already scanned in the source (return + * XD3_NOTAVAIL). */ +struct _main_blklru_list +{ + main_blklru_list *next; + main_blklru_list *prev; +}; + +struct _main_blklru +{ + uint8_t *blk; + xoff_t blkno; + main_blklru_list link; +}; + +/* ... represented as a list (no cache index). */ +XD3_MAKELIST(main_blklru_list,main_blklru,link); + +/* Program options: various command line flags and options. */ +static int option_stdout = 0; +static int option_force = 0; +static int option_verbose = 0; +static int option_quiet = 0; +static int option_level = 6; +static int option_use_appheader = 1; +static uint8_t* option_appheader = NULL; +static int option_use_secondary = /* until-standardized, leave this off */ 0; +static char* option_secondary = NULL; +static int option_use_checksum = 1; +static int option_use_altcodetable = 0; +static char* option_smatch_config = NULL; +static int option_no_compress = 0; +static int option_no_output = 0; /* go through the motions, but do not open or write output */ +static const char *option_source_filename = NULL; +static const char *option_xdelta1 = "xdelta1"; + +static usize_t option_winsize = XD3_DEFAULT_WINSIZE; +static usize_t option_srcwinsz = XD3_DEFAULT_SRCWINSZ; +static usize_t option_memsize = XD3_DEFAULT_MEMSIZE; + +/* This controls the number of times main repeats itself, only for profiling. */ +static int option_profile_cnt = 0; + +/* These variables are supressed to avoid their use w/o support. main() warns + * appropriately. */ +#if EXTERNAL_COMPRESSION +static int option_decompress_inputs = 1; +static int option_recompress_outputs = 1; +#endif + +/* This is for comparing "printdelta" output without attention to + * copy-instruction modes, useful for reverse engineering. */ +#if VCDIFF_TOOLS +static int option_print_cpymode = 1; +#endif + +/* Static variables */ +IF_DEBUG(static int main_mallocs = 0;) + +static char* program_name = NULL; +static uint8_t* appheader_used = NULL; +static uint8_t* main_bdata = NULL; + +/* The LRU: obviously this is shared by all callers. */ +static int lru_size = 0; +static main_blklru *lru = NULL; /* array of lru_size elts */ +static main_blklru_list lru_list; +static main_blklru_list lru_free; +static int do_not_lru = 0; /* set to avoid lru, instead discard oldest */ + +static int lru_hits = 0; +static int lru_misses = 0; +static int lru_filled = 0; + +/* Hacks for VCDIFF tools */ +static int allow_fake_source = 0; + +/* This array of compressor types is compiled even if EXTERNAL_COMPRESSION is false just so + * the program knows the mapping of IDENT->NAME. */ +static main_extcomp extcomp_types[] = +{ + /* The entry for xdelta3/1 must be 0/1 because the program_names are set there. */ + { "xdelta3", "-cfq", "xdelta3", "-dcfq", "X", "\xd6\xc3\xc4", 3, RD_NONEXTERNAL }, + { "bzip2", "-cf", "bzip2", "-dcf", "B", "BZh", 3, 0 }, + { "gzip", "-cf", "gzip", "-dcf", "G", "\037\213", 2, 0 }, + { "compress", "-cf", "uncompress", "-cf", "Z", "\037\235", 2, 0 }, + + /* TODO: xdelta1 isn't working */ + /*{ "xdelta1", "delta", "xdelta1", "patch", "1", "%XD", 3, RD_EXTERNAL_V1 },*/ + + /* TODO: add commandline support for magic-less formats */ + /*{ "lzma", "-cf", "lzma", "-dcf", "M", "]\000", 2, 0 },*/ +}; + +static void main_get_appheader (xd3_stream *stream, main_file *ifile, + main_file *output, main_file *sfile); + +static int main_help (void); + +static int +main_version (void) +{ + /* $Format: " P(RINT \"VERSION=3.$Xdelta3Version$\\n\");" $ */ + P(RINT "VERSION=3.0h_pre0\n"); + return EXIT_SUCCESS; +} + +static int +main_config (void) +{ + main_version (); + + P(RINT "EXTERNAL_COMPRESSION=%d\n", EXTERNAL_COMPRESSION); + P(RINT "GENERIC_ENCODE_TABLES=%d\n", GENERIC_ENCODE_TABLES); + P(RINT "GENERIC_ENCODE_TABLES_COMPUTE=%d\n", GENERIC_ENCODE_TABLES_COMPUTE); + P(RINT "REGRESSION_TEST=%d\n", REGRESSION_TEST); + P(RINT "SECONDARY_DJW=%d\n", SECONDARY_DJW); + P(RINT "SECONDARY_FGK=%d\n", SECONDARY_FGK); + P(RINT "VCDIFF_TOOLS=%d\n", VCDIFF_TOOLS); + P(RINT "XD3_ALLOCSIZE=%d\n", XD3_ALLOCSIZE); + P(RINT "XD3_DEBUG=%d\n", XD3_DEBUG); + P(RINT "XD3_DEFAULT_CKSUM_SIZE=%d\n", XD3_DEFAULT_CKSUM_ADVANCE); + P(RINT "XD3_DEFAULT_IOPT_SIZE=%d\n", XD3_DEFAULT_IOPT_SIZE); + P(RINT "XD3_DEFAULT_MEMSIZE=%d\n", XD3_DEFAULT_MEMSIZE); + P(RINT "XD3_DEFAULT_SPREVSZ=%d\n", XD3_DEFAULT_SPREVSZ); + P(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ); + P(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE); + P(RINT "XD3_ENCODER=%d\n", XD3_ENCODER); + P(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE); + P(RINT "XD3_NODECOMPRESSSIZE=%d\n", XD3_NODECOMPRESSSIZE); + P(RINT "XD3_POSIX=%d\n", XD3_POSIX); + P(RINT "XD3_USE_LARGEFILE64=%d\n", XD3_USE_LARGEFILE64); + + return EXIT_SUCCESS; +} + +static void* +main_malloc1 (usize_t size) +{ + void* r = malloc (size); + if (r == NULL) { XPR(NT "malloc: %s\n", xd3_strerror (ENOMEM)); } + else if (option_verbose > 2) { XPR(NT "malloc: %u: %p\n", size, r); } + return r; +} + +static void* +main_malloc (usize_t size) +{ + void *r = main_malloc1 (size); + if (r) { IF_DEBUG (main_mallocs += 1); } + return r; +} + +static void* +main_alloc (void *opaque, + usize_t items, + usize_t size) +{ + return main_malloc1 (items * size); +} + +static void +main_free1 (void *opaque, void *ptr) +{ + if (option_verbose > 2) { XPR(NT "free: %p\n", ptr); } + free (ptr); +} + +static void +main_free (void *ptr) +{ + if (ptr) + { + IF_DEBUG (main_mallocs -= 1); + main_free1 (NULL, ptr); + } +} + +/* This ensures that (ret = errno) always indicates failure, in case errno was + * accidentally not set. If this prints there's a bug somewhere. */ +static int +get_errno (void) +{ + if (errno == 0) + { + XPR(NT "you found a bug: expected errno != 0\n"); + errno = XD3_INTERNAL; + } + return errno; +} + + + +static long +get_millisecs_now (void) +{ +#ifndef WIN32 + struct timeval tv; + + gettimeofday (& tv, NULL); + + return (tv.tv_sec) * 1000L + (tv.tv_usec) / 1000; +#else + // Found this in an example on www.codeproject.com + // It doesn't matter that the offset is Jan 1, 1601 + // Result is the numbre of 100 nanosecond units + // 100ns * 10,000 = 1ms + SYSTEMTIME st; + FILETIME ft; + __int64 *pi = (__int64*)&ft; + GetLocalTime(&st); + SystemTimeToFileTime(&st, &ft); + return (long)((*pi) / 10000); +#endif +} + +/* Always >= 1 millisec, right? */ +static long +get_millisecs_since (void) +{ + static long last = 0; + long now = get_millisecs_now(); + long diff = now - last; + last = now; + return diff; +} + +static char* +main_format_bcnt (xoff_t r, char *buf) +{ + static const char* fmts[] = { "B", "KB", "MB", "GB" }; + int i; + + for (i = 0; i < SIZEOF_ARRAY(fmts); i += 1) + { + if (r < 10 * 1e3 || i == (-1 + (int)SIZEOF_ARRAY(fmts))) + { + sprintf (buf, "%"Q"u %s", r, fmts[i]); + break; + } + r /= 1000; + } + return buf; +} + +static char* +main_format_rate (xoff_t bytes, long millis, char *buf) +{ + xoff_t r = (xoff_t)(1.0 * bytes / (1.0 * millis / 1000.0)); + static char lbuf[32]; + + main_format_bcnt (r, lbuf); + sprintf (buf, "%s/sec", lbuf); + return buf; +} + +static char* +main_format_millis (long millis, char *buf) +{ + if (millis < 1000) { sprintf (buf, "%lu ms", millis); } + else if (millis < 10000) { sprintf (buf, "%.1f sec", millis / 1000.0); } + else { sprintf (buf, "%lu sec", millis / 1000L); } + return buf; +} + +/* A safe version of strtol for xoff_t. */ +static int +main_strtoxoff (const char* s, xoff_t *xo, char which) +{ + char *e; + xoff_t x; + + XD3_ASSERT(s && *s != 0); + + { + /* Should check LONG_MIN, LONG_MAX, LLONG_MIN, LLONG_MAX? */ +#if SIZEOF_XOFF_T == 4 + long xx = strtol (s, &e, 0); +#else + long long xx = strtoll (s, &e, 0); +#endif + + if (xx < 0) + { + XPR(NT "-%c: negative integer: %s\n", which, s); + return EXIT_FAILURE; + } + + x = xx; + } + + if (*e != 0) + { + XPR(NT "-%c: invalid integer: %s\n", which, s); + return EXIT_FAILURE; + } + + (*xo) = x; + return 0; +} + +static int +main_atou (const char* arg, usize_t *xo, usize_t low, char which) +{ + xoff_t x; + int ret; + + if ((ret = main_strtoxoff (arg, & x, which))) { return ret; } + + if (x > USIZE_T_MAX || x < low) + { + XPR(NT "-%c: minimum value: %u", which, low); + return EXIT_FAILURE; + } + (*xo) = (usize_t)x; + return 0; +} + +/****************************************************************************************** + FILE BASICS + ******************************************************************************************/ + +/* With all the variation in file system-call semantics, arguments, return values and + * error-handling for the POSIX and STDIO file APIs, the insides of these functions make + * me sick, which is why these wrappers exist. */ + +#define XOPEN_OPNAME (xfile->mode == XO_READ ? "read" : "write") +#define XOPEN_STDIO (xfile->mode == XO_READ ? "rb" : "wb") +#define XOPEN_POSIX (xfile->mode == XO_READ ? O_RDONLY : O_WRONLY | O_CREAT | O_TRUNC) +#define XOPEN_MODE (xfile->mode == XO_READ ? 0 : 0666) + +#define XF_ERROR(op, name, ret) XPR(NT "file %s failed: %s: %s: %s\n", (op), XOPEN_OPNAME, (name), xd3_strerror (ret)) + +#if XD3_STDIO +#define XFNO(f) fileno(f->file) +#define XSTDOUT_XF(f) { (f)->file = stdout; (f)->filename = "/dev/stdout"; } +#define XSTDERR_XF(f) { (f)->file = stderr; (f)->filename = "/dev/stderr"; } +#define XSTDIN_XF(f) { (f)->file = stdin; (f)->filename = "/dev/stdin"; } + +#elif XD3_POSIX +#define XFNO(f) f->file +#define XSTDOUT_XF(f) { (f)->file = STDOUT_FILENO; (f)->filename = "/dev/stdout"; } +#define XSTDERR_XF(f) { (f)->file = STDERR_FILENO; (f)->filename = "/dev/stderr"; } +#define XSTDIN_XF(f) { (f)->file = STDIN_FILENO; (f)->filename = "/dev/stdin"; } +#endif + +static void +main_file_init (main_file *xfile) +{ + memset (xfile, 0, sizeof (*xfile)); + +#if XD3_POSIX + xfile->file = -1; +#endif +} + +static void +main_file_cleanup (main_file *xfile) +{ + if (xfile->filename_copy) { + main_free(xfile->filename_copy); + } +} + +static int +main_file_isopen (main_file *xfile) +{ +#if XD3_STDIO + return xfile->file != NULL; + +#elif XD3_POSIX + return xfile->file != -1; +#endif +} + +static int +main_file_close (main_file *xfile) +{ + int ret = 0; + + if (! main_file_isopen (xfile)) + { + return 0; + } + +#if XD3_STDIO + ret = fclose (xfile->file); + xfile->file = NULL; + +#elif XD3_POSIX + ret = close (xfile->file); + xfile->file = -1; +#endif + + if (ret != 0) { XF_ERROR ("close", xfile->filename, ret = get_errno ()); } + return ret; +} + +static int +main_file_open (main_file *xfile, const char* name, int mode) +{ + int ret = 0; + + xfile->mode = mode; + + XD3_ASSERT (! main_file_isopen (xfile)); + +#if XD3_STDIO + xfile->file = fopen (name, XOPEN_STDIO); + + ret = (xfile->file == NULL) ? get_errno () : 0; + +#elif XD3_POSIX + if ((ret = open (name, XOPEN_POSIX, XOPEN_MODE)) < 0) + { + ret = get_errno (); + } + else + { + xfile->file = ret; + ret = 0; + } +#endif + if (ret) { XF_ERROR ("open", name, ret); } + else { xfile->realname = name; xfile->nread = 0; } + return ret; +} + +static int +main_file_stat (main_file *xfile, xoff_t *size, int err_ifnoseek) +{ + int ret; + struct stat sbuf; + + XD3_ASSERT (main_file_isopen (xfile)); + + if (fstat (XFNO (xfile), & sbuf) < 0) + { + ret = get_errno (); + if (err_ifnoseek) { XF_ERROR ("stat", xfile->filename, ret); } + return ret; + } + + if (! S_ISREG (sbuf.st_mode)) + { + if (err_ifnoseek) { XPR(NT "source file must be seekable: %s\n", xfile->filename); } + return ESPIPE; + } + + (*size) = sbuf.st_size; + return 0; +} + +static int +main_file_exists (main_file *xfile) +{ + struct stat sbuf; + return stat (xfile->filename, & sbuf) == 0 && S_ISREG (sbuf.st_mode); +} + +#if (XD3_POSIX || EXTERNAL_COMPRESSION) +/* POSIX-generic code takes a function pointer to read() or write(). This calls the + * function repeatedly until the buffer is full or EOF. The NREAD parameter is not + * set for write, NULL is passed. Return is signed, < 0 indicate errors, otherwise + * byte count. */ +typedef int (xd3_posix_func) (int fd, uint8_t *buf, usize_t size); + +static int +xd3_posix_io (int fd, uint8_t *buf, usize_t size, xd3_posix_func *func, usize_t *nread) +{ + int ret; + usize_t nproc = 0; + + while (nproc < size) + { + int result = (*func) (fd, buf + nproc, size - nproc); + + if (result < 0) + { + ret = get_errno (); + if (ret != EAGAIN && ret != EINTR) + { + return ret; + } + result = 0; + } + + if (nread != NULL && result == 0) { break; } + + nproc += result; + } + if (nread != NULL) { (*nread) = nproc; } + return 0; +} +#endif + +/* POSIX is unbuffered, while STDIO is buffered. main_file_read() should always be called + * on blocks. */ +static int +main_file_read (main_file *ifile, + uint8_t *buf, + usize_t size, + usize_t *nread, + const char *msg) +{ + int ret = 0; + +#if XD3_STDIO + usize_t result; + + result = fread (buf, 1, size, ifile->file); + + if (result < size && ferror (ifile->file)) + { + ret = get_errno (); + } + else + { + *nread = result; + } + +#elif XD3_POSIX + ret = xd3_posix_io (ifile->file, buf, size, (xd3_posix_func*) &read, nread); +#endif + + if (ret) + { + XPR(NT "%s: %s: %s\n", msg, ifile->filename, xd3_strerror (ret)); + } + else + { + if (option_verbose > 2) { XPR(NT "main read: %s: %u\n", ifile->filename, (*nread)); } + ifile->nread += (*nread); + } + + return ret; +} + +static int +main_file_write (main_file *ofile, uint8_t *buf, usize_t size, const char *msg) +{ + int ret = 0; + +#if XD3_STDIO + usize_t result; + + result = fwrite (buf, 1, size, ofile->file); + + if (result != size) { ret = get_errno (); } + +#elif XD3_POSIX + ret = xd3_posix_io (ofile->file, buf, size, (xd3_posix_func*) &write, NULL); +#endif + + if (ret) + { + XPR(NT "%s: %s: %s\n", msg, ofile->filename, xd3_strerror (ret)); + } + else + { + if (option_verbose > 2) { XPR(NT "main write: %s: %u\n", ofile->filename, size); } + ofile->nwrite += size; + } + + return ret; +} + +static int +main_file_seek (main_file *xfile, xoff_t pos) +{ + int ret = 0; + +#if XD3_STDIO + if (fseek (xfile->file, pos, SEEK_SET) != 0) { ret = get_errno (); } +#else + if (lseek (xfile->file, pos, SEEK_SET) != pos) { ret = get_errno (); } +#endif + + if (ret) + { + XPR(NT "seek failed: %s: %s\n", xfile->filename, xd3_strerror (ret)); + } + + return ret; +} + +/****************************************************************************************** + VCDIFF TOOLS + ******************************************************************************************/ + +#if VCDIFF_TOOLS +/* This function prints a single VCDIFF window, mainly for debugging purposes. */ +static int +main_print_window (xd3_stream* stream, FILE *vcout) +{ + int ret; + usize_t size = 0; + + VC(UT " Offset Code Type1 Size1 @Addr1 + Type2 Size2 @Addr2\n"); + + while (stream->inst_sect.buf < stream->inst_sect.buf_max) + { + uint code = stream->inst_sect.buf[0]; + + if ((ret = xd3_decode_instruction (stream))) { return ret; } + + VC(UT " %06"Q"u %03u %s %3u", stream->dec_winstart + size, code, + xd3_rtype_to_string (stream->dec_current1.type, option_print_cpymode), + stream->dec_current1.size); + + if (stream->dec_current1.type != XD3_NOOP) + { + size += stream->dec_current1.size; + if (stream->dec_current1.type >= XD3_CPY) + { + VC(UT " @%-6u", stream->dec_current1.addr); + } + else + { + VC(UT " "); + } + } + + if (stream->dec_current2.type != XD3_NOOP) + { + size += stream->dec_current2.size; + VC(UT " %s %3u", + xd3_rtype_to_string (stream->dec_current2.type, option_print_cpymode), + stream->dec_current2.size); + + if (stream->dec_current2.type >= XD3_CPY) + { + VC(UT " @%-6u", stream->dec_current2.addr); + } + } + + VC(UT "\n"); + } + + if (stream->dec_tgtlen != size && (stream->flags & XD3_SKIP_WINDOW) == 0) + { + XPR(NT "target window size inconsistency"); + return XD3_INTERNAL; + } + + if (stream->dec_position != stream->dec_maxpos) + { + XPR(NT "target window position inconsistency"); + return XD3_INTERNAL; + } + + if (stream->addr_sect.buf != stream->addr_sect.buf_max) + { + XPR(NT "address section inconsistency"); + return XD3_INTERNAL; + } + + IF_DEBUG (VC(UT "SIZE=%u TGTLEN=%u\n", size, stream->dec_tgtlen)); + + return 0; +} + +static void +main_print_vcdiff_file (main_file *file, const char *type, FILE *vcout) +{ + if (file->filename) { VC(UT "XDELTA filename (%s): %s\n", type, file->filename); } + if (file->compressor) { VC(UT "XDELTA ext comp (%s): %s\n", type, file->compressor->recomp_cmdname); } +} + +/* This function prints a VCDIFF input, mainly for debugging purposes. */ +static int +main_print_func (xd3_stream* stream, main_file *xfile) +{ + int ret; + FILE *vcout; +#if XD3_POSIX + if (! (vcout = fdopen (dup(xfile->file), "w"))) + { + ret = get_errno (); + XPR(NT "fdopen: %s: %s\n", xfile->filename, xd3_strerror (ret)); + return ret; + } +#elif XD3_STDIO + vcout = xfile->file; +#endif + XD3_ASSERT (vcout); + if (stream->dec_winstart == 0) + { + VC(UT "VCDIFF version: 0\n"); + + VC(UT "VCDIFF header size: %d\n", stream->dec_hdrsize); + VC(UT "VCDIFF header indicator: "); + if ((stream->dec_hdr_ind & VCD_SECONDARY) != 0) VC(UT "VCD_SECONDARY "); + if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) VC(UT "VCD_CODETABLE "); + if ((stream->dec_hdr_ind & VCD_APPHEADER) != 0) VC(UT "VCD_APPHEADER "); + if (stream->dec_hdr_ind == 0) VC(UT "none"); + VC(UT "\n"); + + IF_SEC(VC(UT "VCDIFF secondary compressor: %s\n", stream->sec_type ? stream->sec_type->name : "none")); + IF_NSEC(VC(UT "VCDIFF secondary compressor: unsupported\n")); + + if (stream->dec_hdr_ind & VCD_APPHEADER) + { + uint8_t *apphead; + usize_t appheadsz; + ret = xd3_get_appheader (stream, & apphead, & appheadsz); + + if (ret == 0 && appheadsz > 0) + { + int sq = option_quiet; + main_file i, o, s; + XD3_ASSERT (apphead != NULL); + VC(UT "VCDIFF application header: "); + fwrite (apphead, 1, appheadsz, vcout); + VC(UT "\n"); + + main_file_init (& i); + main_file_init (& o); + main_file_init (& s); + option_quiet = 1; + main_get_appheader (stream, &i, & o, & s); + option_quiet = sq; + main_print_vcdiff_file (& o, "output", vcout); + main_print_vcdiff_file (& s, "source", vcout); + main_file_cleanup (& i); + main_file_cleanup (& o); + main_file_cleanup (& s); + } + } + } + else + { + VC(UT "\n"); + } + + VC(UT "VCDIFF window number: %"Q"u\n", stream->current_window); + VC(UT "VCDIFF window indicator: "); + if ((stream->dec_win_ind & VCD_SOURCE) != 0) VC(UT "VCD_SOURCE "); + if ((stream->dec_win_ind & VCD_TARGET) != 0) VC(UT "VCD_TARGET "); + if ((stream->dec_win_ind & VCD_ADLER32) != 0) VC(UT "VCD_ADLER32 "); + if (stream->dec_win_ind == 0) VC(UT "none"); + VC(UT "\n"); + + if ((stream->dec_win_ind & VCD_ADLER32) != 0) + { + VC(UT "VCDIFF adler32 checksum: %08X\n", stream->dec_adler32); + } + + if (stream->dec_del_ind != 0) + { + VC(UT "VCDIFF delta indicator: "); + if ((stream->dec_del_ind & VCD_DATACOMP) != 0) VC(UT "VCD_DATACOMP "); + if ((stream->dec_del_ind & VCD_INSTCOMP) != 0) VC(UT "VCD_INSTCOMP "); + if ((stream->dec_del_ind & VCD_ADDRCOMP) != 0) VC(UT "VCD_ADDRCOMP "); + if (stream->dec_del_ind == 0) VC(UT "none"); + VC(UT "\n"); + } + + if (stream->dec_winstart != 0) + { + VC(UT "VCDIFF window at offset: %"Q"u\n", stream->dec_winstart); + } + + if (SRCORTGT (stream->dec_win_ind)) + { + VC(UT "VCDIFF copy window length: %u\n", stream->dec_cpylen); + VC(UT "VCDIFF copy window offset: %"Q"u\n", stream->dec_cpyoff); + } + + VC(UT "VCDIFF delta encoding length: %u\n", stream->dec_enclen); + VC(UT "VCDIFF target window length: %u\n", stream->dec_tgtlen); + + VC(UT "VCDIFF data section length: %u\n", stream->data_sect.size); + VC(UT "VCDIFF inst section length: %u\n", stream->inst_sect.size); + VC(UT "VCDIFF addr section length: %u\n", stream->addr_sect.size); + + ret = 0; + if ((stream->flags & XD3_JUST_HDR) != 0) + { + /* Print a header -- finished! */ + ret = PRINTHDR_SPECIAL; + } + else if ((stream->flags & XD3_SKIP_WINDOW) == 0) + { + ret = main_print_window (stream, vcout); + } + + fclose (vcout); + return ret; +} +#endif /* VCDIFF_TOOLS */ + +/****************************************************************************************** + Input decompression, output recompression + ******************************************************************************************/ + +#if EXTERNAL_COMPRESSION +/* This is tricky POSIX-specific code with lots of fork(), pipe(), dup(), waitpid(), and + * exec() business. Most of this code originated in PRCS1, which did automatic + * package-file decompression. It works with both XD3_POSIX and XD3_STDIO file + * disciplines. + * + * To automatically detect compressed inputs requires a child process to reconstruct the + * input stream, which was advanced in order to detect compression, because it may not be + * seekable. In other words, the main program reads part of the input stream, and if it + * detects a compressed input it then forks a pipe copier process, which copies the + * first-read block out of the main-program's memory, then streams the remaining + * compressed input into the input-decompression pipe. + */ + +#include +#include +#include + +/* Remember which pipe FD is which. */ +#define PIPE_READ_FD 0 +#define PIPE_WRITE_FD 1 + +static pid_t ext_subprocs[2]; +static char* ext_tmpfile = NULL; + +/* Like write(), but makes repeated calls to empty the buffer. */ +static int +main_pipe_write (int outfd, const uint8_t *exist_buf, usize_t remain) +{ + int ret; + + if ((ret = xd3_posix_io (outfd, (uint8_t*) exist_buf, remain, (xd3_posix_func*) &write, NULL))) + { + XPR(NT "pipe write failed: %s", xd3_strerror (ret)); + return ret; + } + + return 0; +} + +/* A simple error-reporting waitpid interface. */ +static int +main_waitpid_check(pid_t pid) +{ + int status; + int ret = 0; + + if (waitpid (pid, & status, 0) < 0) + { + ret = get_errno (); + XPR(NT "compression subprocess: wait: %s\n", xd3_strerror (ret)); + } + else if (! WIFEXITED (status)) + { + ret = ECHILD; + XPR(NT "compression subprocess: signal %d\n", + WIFSIGNALED (status) ? WTERMSIG (status) : WSTOPSIG (status)); + } + else if (WEXITSTATUS (status) != 0) + { + ret = ECHILD; + XPR(NT "compression subprocess: exit %d\n", WEXITSTATUS (status)); + } + + return ret; +} + +/* Wait for any existing child processes to check for abnormal exit. */ +static int +main_external_compression_finish (void) +{ + int i; + int ret; + + for (i = 0; i < 2; i += 1) + { + if (! ext_subprocs[i]) { continue; } + + if ((ret = main_waitpid_check (ext_subprocs[i]))) + { + return ret; + } + } + + return 0; +} + +/* This runs as a forked process of main_input_decompress_setup() to copy input to the + * decompression process. First, the available input is copied out of the existing + * buffer, then the buffer is reused to continue reading from the compressed input + * file. */ +static int +main_pipe_copier (uint8_t *pipe_buf, + usize_t pipe_bufsize, + usize_t nread, + main_file *ifile, + int outfd) +{ + int ret; + + for (;;) + { + if (nread > 0 && (ret = main_pipe_write (outfd, pipe_buf, nread))) + { + return ret; + } + + if (nread < pipe_bufsize) + { + break; + } + + if ((ret = main_file_read (ifile, pipe_buf, pipe_bufsize, & nread, "pipe read failed")) < 0) + { + return ret; + } + } + + return 0; +} + +/* This function is called after we have read some amount of data from the input file and + * detected a compressed input. Here we start a decompression subprocess by forking + * twice. The first process runs the decompression command, the second process copies + * data to the input of the first. */ +static int +main_input_decompress_setup (const main_extcomp *decomp, + main_file *ifile, + uint8_t *input_buf, + usize_t input_bufsize, + uint8_t *pipe_buf, + usize_t pipe_bufsize, + usize_t pipe_avail, + usize_t *nread) +{ + int outpipefd[2], inpipefd[2]; /* The two pipes: input and output file descriptors. */ + int input_fd = -1; /* The resulting input_fd (output of decompression). */ + pid_t decomp_id, copier_id; /* The two subprocs. */ + int ret; + + outpipefd[0] = outpipefd[1] = -1; + inpipefd[0] = inpipefd[1] = -1; + + if (pipe (outpipefd) || pipe (inpipefd)) + { + XPR(NT "pipe failed: %s\n", xd3_strerror (ret = get_errno ())); + goto pipe_cleanup; + } + + if ((decomp_id = fork ()) < 0) + { + XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ())); + goto pipe_cleanup; + } + + /* The first child runs the decompression process: */ + if (decomp_id == 0) + { + /* Setup pipes: write to the outpipe, read from the inpipe. */ + if (dup2 (outpipefd[PIPE_WRITE_FD], STDOUT_FILENO) < 0 || + dup2 (inpipefd[PIPE_READ_FD], STDIN_FILENO) < 0 || + close (outpipefd[PIPE_READ_FD]) || + close (outpipefd[PIPE_WRITE_FD]) || + close (inpipefd[PIPE_READ_FD]) || + close (inpipefd[PIPE_WRITE_FD]) || + execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, decomp->decomp_options, NULL)) + { + XPR(NT "child process %s failed to execute: %s\n", decomp->decomp_cmdname, xd3_strerror (get_errno ())); + } + + _exit (127); + } + + ext_subprocs[0] = decomp_id; + + if ((copier_id = fork ()) < 0) + { + XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ())); + goto pipe_cleanup; + } + + /* The second child runs the copier process: */ + if (copier_id == 0) + { + int exitval = 0; + + if (close (inpipefd[PIPE_READ_FD]) || + main_pipe_copier (pipe_buf, pipe_bufsize, pipe_avail, ifile, inpipefd[PIPE_WRITE_FD]) || + close (inpipefd[PIPE_WRITE_FD])) + { + XPR(NT "child copier process failed: %s\n", xd3_strerror (get_errno ())); + exitval = 1; + } + + _exit (exitval); + } + + ext_subprocs[1] = copier_id; + + /* The parent closes both pipes after duplicating the output of compression. */ + input_fd = dup (outpipefd[PIPE_READ_FD]); + + if (input_fd < 0 || + main_file_close (ifile) || + close (outpipefd[PIPE_READ_FD]) || + close (outpipefd[PIPE_WRITE_FD]) || + close (inpipefd[PIPE_READ_FD]) || + close (inpipefd[PIPE_WRITE_FD])) + { + XPR(NT "dup/close failed: %s\n", xd3_strerror (ret = get_errno ())); + goto pipe_cleanup; + } + +#if XD3_STDIO + /* Note: fdopen() acquires the fd, closes it when finished. */ + if ((ifile->file = fdopen (input_fd, "r")) == NULL) + { + XPR(NT "fdopen failed: %s\n", xd3_strerror (ret = get_errno ())); + goto pipe_cleanup; + } + +#elif XD3_POSIX + ifile->file = input_fd; +#endif + + ifile->compressor = decomp; + + /* Now the input file is decompressed. */ + return main_file_read (ifile, input_buf, input_bufsize, nread, "input decompression failed"); + + pipe_cleanup: + close (input_fd); + close (outpipefd[PIPE_READ_FD]); + close (outpipefd[PIPE_WRITE_FD]); + close (inpipefd[PIPE_READ_FD]); + close (inpipefd[PIPE_WRITE_FD]); + return ret; +} + + +/* This routine is called when the first buffer of input data is read by the main program + * (unless input decompression is disabled by command-line option). If it recognizes the + * magic number of a known input type it invokes decompression. + * + * Skips decompression if the decompression type or the file type is RD_NONEXTERNAL. + * + * Behaves exactly like main_file_read, otherwise. + * + * This function uses a separate buffer to read the first small block of input. If a + * compressed input is detected, the separate buffer is passed to the pipe copier. This + * avoids using the same size buffer in both cases. */ +static int +main_decompress_input_check (main_file *ifile, + uint8_t *input_buf, + usize_t input_size, + usize_t *nread) +{ + int i; + int ret; + uint8_t check_buf[XD3_ALLOCSIZE]; + usize_t check_nread; + + if ((ret = main_file_read (ifile, check_buf, min (input_size, XD3_ALLOCSIZE), & check_nread, "input read failed"))) + { + return ret; + } + + for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) + { + const main_extcomp *decomp = & extcomp_types[i]; + + if ((check_nread > decomp->magic_size) && + /* The following expr skips decompression if we are trying to read a VCDIFF + * input and that is the magic number. */ + !((decomp->flags & RD_NONEXTERNAL) && (ifile->flags & RD_NONEXTERNAL)) && + memcmp (check_buf, decomp->magic, decomp->magic_size) == 0) + { + if (! option_quiet) + { + XPR(NT "%s | %s %s\n", + ifile->filename, + decomp->decomp_cmdname, + decomp->decomp_options); + } + + return main_input_decompress_setup (decomp, ifile, + input_buf, input_size, + check_buf, XD3_ALLOCSIZE, + check_nread, nread); + } + } + + /* Now read the rest of the input block. */ + (*nread) = 0; + + if (check_nread == XD3_ALLOCSIZE) + { + ret = main_file_read (ifile, input_buf + XD3_ALLOCSIZE, + input_size - XD3_ALLOCSIZE, nread, + "input read failed"); + } + + memcpy (input_buf, check_buf, check_nread); + + (*nread) += check_nread; + + return 0; +} + +/* This is called when the source file needs to be decompressed. We fork/exec a + * decompression command with the proper input and output to a temporary file. */ +static int +main_decompress_source (main_file *sfile, xd3_source *source) +{ + const main_extcomp *decomp = sfile->compressor; + pid_t decomp_id; /* One subproc. */ + int input_fd = -1; + int output_fd = -1; + int ret; + char *tmpname = NULL; + char *tmpdir = getenv ("TMPDIR"); + static const char tmpl[] = "/xd3src.XXXXXX"; + + /* Make a template for mkstmp() */ + if (tmpdir == NULL) { tmpdir = "/tmp"; } + if ((tmpname = main_malloc (strlen (tmpdir) + sizeof (tmpl) + 1)) == NULL) { return ENOMEM; } + sprintf (tmpname, "%s%s", tmpdir, tmpl); + + XD3_ASSERT (ext_tmpfile == NULL); + ext_tmpfile = tmpname; + + /* Open the output FD. */ + if ((output_fd = mkstemp (tmpname)) < 0) + { + XPR(NT "mkstemp failed: %s: %s", tmpname, xd3_strerror (ret = get_errno ())); + goto cleanup; + } + + /* Copy the input FD, reset file position. */ + XD3_ASSERT (main_file_isopen (sfile)); +#if XD3_STDIO + if ((input_fd = dup (fileno (sfile->file))) < 0) + { + XPR(NT "dup failed: %s", xd3_strerror (ret = get_errno ())); + goto cleanup; + } + main_file_close (sfile); + sfile->file = NULL; +#elif XD3_POSIX + input_fd = sfile->file; + sfile->file = -1; +#endif + + if ((ret = lseek (input_fd, SEEK_SET, 0)) != 0) + { + XPR(NT "lseek failed: : %s", xd3_strerror (ret = get_errno ())); + goto cleanup; + } + + if ((decomp_id = fork ()) < 0) + { + XPR(NT "fork failed: %s", xd3_strerror (ret = get_errno ())); + goto cleanup; + } + + /* The child runs the decompression process: */ + if (decomp_id == 0) + { + /* Setup pipes: write to the output file, read from the pipe. */ + if (dup2 (input_fd, STDIN_FILENO) < 0 || + dup2 (output_fd, STDOUT_FILENO) < 0 || + execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, decomp->decomp_options, NULL)) + { + XPR(NT "child process %s failed to execute: %s\n", + decomp->decomp_cmdname, xd3_strerror (get_errno ())); + } + + _exit (127); + } + + close (input_fd); + close (output_fd); + input_fd = -1; + output_fd = -1; + + /* Then wait for completion. */ + if ((ret = main_waitpid_check (decomp_id))) + { + goto cleanup; + } + + /* Open/stat the decompressed source file. */ + if ((ret = main_file_open (sfile, tmpname, XO_READ))) { goto cleanup; } + if ((ret = main_file_stat (sfile, & source->size, 1))) { goto cleanup; } + return 0; + + cleanup: + close (input_fd); + close (output_fd); + if (tmpname) { free (tmpname); } + ext_tmpfile = NULL; + return ret; +} + +/* Initiate re-compression of the output stream. This is easier than input decompression + * because we know beforehand that the stream will be compressed, whereas the input has + * already been read when we decide it should be decompressed. Thus, it only requires one + * subprocess and one pipe. */ +static int +main_recompress_output (main_file *ofile) +{ + pid_t recomp_id; /* One subproc. */ + int pipefd[2]; /* One pipe. */ + int output_fd = -1; + int ret; + const main_extcomp *recomp = ofile->compressor; + + pipefd[0] = pipefd[1] = -1; + + if (pipe (pipefd)) + { + XPR(NT "pipe failed: %s\n", xd3_strerror (ret = get_errno ())); + goto pipe_cleanup; + } + + if ((recomp_id = fork ()) < 0) + { + XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ())); + goto pipe_cleanup; + } + + /* The child runs the recompression process: */ + if (recomp_id == 0) + { + /* Setup pipes: write to the output file, read from the pipe. */ + if (dup2 (XFNO (ofile), STDOUT_FILENO) < 0 || + dup2 (pipefd[PIPE_READ_FD], STDIN_FILENO) < 0 || + close (pipefd[PIPE_READ_FD]) || + close (pipefd[PIPE_WRITE_FD]) || + execlp (recomp->recomp_cmdname, recomp->recomp_cmdname, recomp->recomp_options, NULL)) + { + XPR(NT "child process %s failed to execute: %s\n", recomp->recomp_cmdname, xd3_strerror (get_errno ())); + } + + _exit (127); + } + + ext_subprocs[0] = recomp_id; + + /* The parent closes both pipes after duplicating the output-fd for writing to the + * compression pipe. */ + output_fd = dup (pipefd[PIPE_WRITE_FD]); + + if (output_fd < 0 || + main_file_close (ofile) || + close (pipefd[PIPE_READ_FD]) || + close (pipefd[PIPE_WRITE_FD])) + { + XPR(NT "close failed: %s\n", xd3_strerror (ret = get_errno ())); + goto pipe_cleanup; + } + +#if XD3_STDIO + /* Note: fdopen() acquires the fd, closes it when finished. */ + if ((ofile->file = fdopen (output_fd, "w")) == NULL) + { + XPR(NT "fdopen failed: %s\n", xd3_strerror (ret = get_errno ())); + goto pipe_cleanup; + } + +#elif XD3_POSIX + ofile->file = output_fd; +#endif + + /* Now the output file will be compressed. */ + return 0; + + pipe_cleanup: + close (output_fd); + close (pipefd[PIPE_READ_FD]); + close (pipefd[PIPE_WRITE_FD]); + return ret; +} +#endif /* EXTERNAL_COMPRESSION */ + +/* Identify the compressor that was used based on its ident string, which is passed in the + * application header. */ +static const main_extcomp* +main_ident_compressor (const char *ident) +{ + int i; + + for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) + { + if (strcmp (extcomp_types[i].ident, ident) == 0) + { + return & extcomp_types[i]; + } + } + + return NULL; +} + +/* Return the main_extcomp record to use for this identifier, if possible. */ +static const main_extcomp* +main_get_compressor (const char *ident) +{ + const main_extcomp *ext = main_ident_compressor (ident); + + if (ext == NULL) + { + if (! option_quiet) + { + XPR(NT "warning: cannot recompress output: " + "unrecognized external compression ID: %s\n", ident); + } + return NULL; + } + else if (! EXTERNAL_COMPRESSION) + { + if (! option_quiet) + { + XPR(NT "warning: external support not compiled: " + "original input was compressed: %s\n", ext->recomp_cmdname); + } + return NULL; + } + else + { + return ext; + } +} + +/****************************************************************************************** + APPLICATION HEADER + ******************************************************************************************/ + +#if XD3_ENCODER +static const char* +main_apphead_string (const char* x) +{ + const char *y; + + if (x == NULL) { return ""; } + + if (strcmp (x, "/dev/stdin") == 0 || + strcmp (x, "/dev/stdout") == 0 || + strcmp (x, "/dev/stderr") == 0) { return "-"; } + + // TODO: this is not portable + return (y = strrchr (x, '/')) == NULL ? x : y + 1; +} + +static int +main_set_appheader (xd3_stream *stream, main_file *input, main_file *sfile) +{ + /* The user may disable the application header. Once the appheader is set, this + * disables setting it again. */ + if (appheader_used || ! option_use_appheader) { return 0; } + + /* The user may specify the application header, otherwise format the default header. */ + if (option_appheader) + { + appheader_used = option_appheader; + } + else + { + const char *iname; + const char *icomp; + const char *sname; + const char *scomp; + int len; + + iname = main_apphead_string (input->filename); + icomp = (input->compressor == NULL) ? "" : input->compressor->ident; + len = strlen (iname) + strlen (icomp) + 2; + + if (sfile->filename != NULL) + { + sname = main_apphead_string (sfile->filename); + scomp = (sfile->compressor == NULL) ? "" : sfile->compressor->ident; + len += strlen (sname) + strlen (scomp) + 2; + } + else + { + sname = scomp = ""; + } + + if ((appheader_used = main_malloc (len)) == NULL) + { + return ENOMEM; + } + + if (sfile->filename == NULL) + { + sprintf ((char*)appheader_used, "%s/%s", iname, icomp); + } + else + { + sprintf ((char*)appheader_used, "%s/%s/%s/%s", iname, icomp, sname, scomp); + } + } + + xd3_set_appheader (stream, appheader_used, strlen ((char*)appheader_used)); + + return 0; +} +#endif + +static void +main_get_appheader_params (main_file *file, char **parsed, int output, const char *type, + main_file *other) +{ + /* Set the filename if it was not specified. If output, option_stdout (-c) overrides. */ + if (file->filename == NULL && ! (output && option_stdout) && strcmp (parsed[0], "-") != 0) + { + file->filename = parsed[0]; + + if (other->filename != NULL) { + /* Take directory from the other file, if it has one. */ + char *last_slash = strrchr(other->filename, '/'); + + if (last_slash != NULL) { + int dlen = last_slash - other->filename; + file->filename_copy = main_malloc(dlen + 2 + strlen(file->filename)); + + strncpy(file->filename_copy, other->filename, dlen); + file->filename_copy[dlen] = '/'; + strcpy(file->filename_copy + dlen + 1, parsed[0]); + + file->filename = file->filename_copy; + } + } + + if (! option_quiet) + { + XPR(NT "using default %s filename: %s\n", type, file->filename); + } + } + + /* Set the compressor, initiate de/recompression later. */ + if (file->compressor == NULL && *parsed[1] != 0) + { + file->compressor = main_get_compressor (parsed[1]); + } +} + +static void +main_get_appheader (xd3_stream *stream, main_file *ifile, main_file *output, main_file *sfile) +{ + uint8_t *apphead; + usize_t appheadsz; + int ret; + + /* The user may disable the application header. Once the appheader is set, this + * disables setting it again. */ + if (! option_use_appheader) { return; } + + ret = xd3_get_appheader (stream, & apphead, & appheadsz); + + /* Ignore failure, it only means we haven't received a header yet. */ + if (ret != 0) { return; } + + if (appheadsz > 0) + { + char *start = (char*)apphead; + char *slash; + int place = 0; + char *parsed[4]; + + memset (parsed, 0, sizeof (parsed)); + + while ((slash = strchr (start, '/')) != NULL) + { + *slash = 0; + parsed[place++] = start; + start = slash + 1; + } + + parsed[place++] = start; + + /* First take the output parameters. */ + if (place == 2 || place == 4) + { + main_get_appheader_params (output, parsed, 1, "output", ifile); + } + + /* Then take the source parameters. */ + if (place == 4) + { + main_get_appheader_params (sfile, parsed+2, 0, "source", ifile); + } + } + + option_use_appheader = 0; + return; +} + +/****************************************************************************************** + Main I/O routines + ******************************************************************************************/ + +/* This function acts like the above except it may also try to recognize a compressed + * input when the first buffer of data is read. The EXTERNAL_COMPRESSION code is called + * to search for magic numbers. */ +static int +main_read_primary_input (main_file *ifile, + uint8_t *buf, + usize_t size, + usize_t *nread) +{ +#if EXTERNAL_COMPRESSION + if (option_decompress_inputs && ifile->flags & RD_FIRST) + { + ifile->flags &= ~RD_FIRST; + + return main_decompress_input_check (ifile, buf, size, nread); + } +#endif + + return main_file_read (ifile, buf, size, nread, "input read failed"); +} + +/* This function simply writes the stream output buffer, if there is any. This is used + * for both encode and decode commands. (The VCDIFF tools use main_print_func()). */ +static int +main_write_output (xd3_stream* stream, main_file *ofile) +{ + int ret; + + if (stream->avail_out > 0 && (ret = main_file_write (ofile, stream->next_out, stream->avail_out, "write failed"))) + { + return ret; + } + + return 0; +} + +/* Open the main output file, sets a default file name, initiate recompression. This + * function is expected to fprint any error messages. */ +static int +main_open_output (xd3_stream *stream, main_file *ofile) +{ + int ret; + + if (ofile->filename == NULL) + { + XSTDOUT_XF (ofile); + + if (option_verbose > 1) { XPR(NT "using standard output: %s\n", ofile->filename); } + } + else + { + /* Stat the file to check for overwrite. */ + if (option_force == 0 && main_file_exists (ofile)) + { + XPR(NT "to overwrite output file specify -f: %s\n", ofile->filename); + return EEXIST; + } + + if ((ret = main_file_open (ofile, ofile->filename, XO_WRITE))) + { + return ret; + } + + if (option_verbose > 1) { XPR(NT "open output: %s\n", ofile->filename); } + } + +#if EXTERNAL_COMPRESSION + /* Do output recompression. */ + if (ofile->compressor != NULL && option_recompress_outputs == 1) + { + if (! option_quiet) + { + XPR(NT "%s %s | %s\n", + ofile->compressor->recomp_cmdname, + ofile->compressor->recomp_options, + ofile->filename); + } + + if ((ret = main_recompress_output (ofile))) + { + return ret; + } + } +#endif + + return 0; +} + +/* This is called at different times for encoding and decoding. The encoder calls it + * immediately, the decoder delays until the application header is received. */ +static int +main_set_source (xd3_stream *stream, int cmd, main_file *sfile, xd3_source *source) +{ + int ret, i; + + /* Open it, check for seekability, set required xd3_source fields. */ + if (allow_fake_source) + { + sfile->mode = XO_READ; + sfile->realname = sfile->filename; + sfile->nread = 0; + source->size = UINT64_MAX; + } + else if ((ret = main_file_open (sfile, sfile->filename, XO_READ)) || + (ret = main_file_stat (sfile, & source->size, 1))) + { + return ret; + } + + source->name = sfile->filename; + source->ioh = sfile; + source->curblkno = (xoff_t) -1; + source->curblk = NULL; + + /* Source block LRU init. */ + main_blklru_list_init (& lru_list); + main_blklru_list_init (& lru_free); + + // Note: to avoid unnecessary allocs for small files: problem + // was with --decompress_inputs, we don't know the true size + // option_srcwinsz = min(source->size, (xoff_t) option_srcwinsz); + + if (option_verbose > 1) { XPR(NT "source window size: %u\n", option_srcwinsz); } + if (option_verbose > 1) { XPR(NT "source block size: %u\n", source->blksize); } + if (option_verbose > 1) { XPR(NT "memory size: %u\n", option_memsize); } + + lru_size = (option_srcwinsz / source->blksize); + lru_size = max(1, lru_size); + + if ((lru = main_malloc (sizeof (main_blklru) * lru_size)) == NULL) + { + return ENOMEM; + } + + for (i = 0; i < lru_size; i += 1) + { + lru[i].blkno = (xoff_t) -1; + + if ((lru[i].blk = main_malloc (source->blksize)) == NULL) + { + return ENOMEM; + } + + main_blklru_list_push_back (& lru_free, & lru[i]); + } + +#if EXTERNAL_COMPRESSION + if (option_decompress_inputs) + { + if (IS_ENCODE (cmd)) + { + usize_t nread; + + source->curblk = lru[0].blk; + + /* If encoding, read the first block now to check for decompression. */ + if ((ret = main_file_read (sfile, (uint8_t*) source->curblk, source->blksize, & nread, "source read failed"))) + { + return ret; + } + + /* Check known magic numbers. */ + for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) + { + const main_extcomp *decomp = & extcomp_types[i]; + + if ((nread > decomp->magic_size) && memcmp (source->curblk, decomp->magic, decomp->magic_size) == 0) + { + sfile->compressor = decomp; + break; + } + } + + /* If no decompression, the current buffer is now a valid source->curblock. */ + if (sfile->compressor == NULL) + { + main_blklru_list_remove (& lru[0]); + main_blklru_list_push_back (& lru_list, & lru[0]); + + lru[0].blkno = 0; + source->curblkno = 0; + source->onblk = nread; + + if (option_verbose > 2) + { + XPR(NT "source block 0 read (not compressed)\n"); + } + } + } + + /* In either the encoder or decoder, start decompression. */ + if (sfile->compressor) + { + xoff_t osize = source->size; + + if (osize > XD3_NODECOMPRESSSIZE) + { + XPR(NT "source file too large for external decompression: %s: %"Q"u\n", + sfile->filename, osize); + return XD3_INTERNAL; + } + + if ((ret = main_decompress_source (sfile, source))) + { + return ret; + } + + if (! option_quiet) + { + char s1[32], s2[32]; + XPR(NT "%s | %s %s => %s %.1f%% [ %s , %s ]\n", + sfile->filename, + sfile->compressor->decomp_cmdname, + sfile->compressor->decomp_options, + sfile->realname, + 100.0 * source->size / osize, + main_format_bcnt (osize, s1), + main_format_bcnt (source->size, s2)); + } + } + } +#endif + + if (option_verbose > 1) { XPR(NT "source file: %s: %"Q"u bytes\n", sfile->realname, source->size); } + + if ((ret = xd3_set_source (stream, source))) + { + XPR(NT XD3_LIB_ERRMSG (stream, ret)); + return EXIT_FAILURE; + } + + return 0; +} + +/****************************************************************************************** + Source routines + ******************************************************************************************/ + +/* This is the callback for reading a block of source. This function is blocking and it + * implements a small LRU. + * + * Note that it is possible for main_input() to handle getblk requests in a non-blocking + * manner. If the callback is NULL then the caller of xd3_*_input() must handle the + * XD3_GETSRCBLK return value and fill the source in the same way. See xd3_getblk for + * details. To see an example of non-blocking getblk, see xdelta-test.h. */ +static int +main_getblk_func (xd3_stream *stream, + xd3_source *source, + xoff_t blkno) +{ + xoff_t pos = blkno * source->blksize; + main_file *sfile = (main_file*) source->ioh; + main_blklru *blru = NULL; + usize_t onblk = xd3_bytes_on_srcblk (source, blkno); + usize_t nread; + int ret; + int i; + + if (allow_fake_source) + { + source->curblkno = blkno; + source->onblk = onblk; + source->curblk = lru[0].blk; + return 0; + } + + if (do_not_lru) + { + /* Direct lookup assumes sequential scan w/o skipping blocks. */ + int idx = blkno % lru_size; + if (lru[idx].blkno == blkno) + { + source->curblkno = blkno; + source->onblk = onblk; + source->curblk = lru[idx].blk; + lru_hits += 1; + return 0; + } + + if (lru[idx].blkno != -1LL && + lru[idx].blkno != blkno - lru_size) + { + return XD3_TOOFARBACK; + } + } + else + { + /* Sequential search through LRU. */ + for (i = 0; i < lru_size; i += 1) + { + if (lru[i].blkno == blkno) + { + main_blklru_list_remove (& lru[i]); + main_blklru_list_push_back (& lru_list, & lru[i]); + + source->curblkno = blkno; + source->onblk = onblk; + source->curblk = lru[i].blk; + lru_hits += 1; + return 0; + } + } + } + + if (! main_blklru_list_empty (& lru_free)) + { + blru = main_blklru_list_pop_front (& lru_free); + } + else if (! main_blklru_list_empty (& lru_list)) + { + if (do_not_lru) { + blru = & lru[blkno % lru_size]; + main_blklru_list_remove(blru); + } else { + blru = main_blklru_list_pop_front (& lru_list); + } + lru_misses += 1; + } + + lru_filled += 1; + + if ((ret = main_file_seek (sfile, pos))) + { + return ret; + } + + if ((ret = main_file_read (sfile, (uint8_t*) blru->blk, source->blksize, + & nread, "source read failed"))) + { + return ret; + } + + if (nread != onblk) + { + XPR(NT "source file size change: %s\n", sfile->filename); + return XD3_INTERNAL; + } + + main_blklru_list_push_back (& lru_list, blru); + + if (option_verbose > 2) + { + if (blru->blkno != -1LL) + { + XPR(NT "source block %"Q"u ejects %"Q"u (lru_hits=%u, lru_misses=%u, lru_filled=%u)\n", + blkno, blru->blkno, lru_hits, lru_misses, lru_filled); + } + else + { + XPR(NT "source block %"Q"u read (lru_hits=%u, lru_misses=%u, lru_filled=%u)\n", + blkno, lru_hits, lru_misses, lru_filled); + } + } + + blru->blkno = blkno; + source->curblk = blru->blk; + source->curblkno = blkno; + source->onblk = onblk; + + return 0; +} + +/****************************************************************************************** + Main routines + ******************************************************************************************/ + +/* This is a generic input function. It calls the xd3_encode_input or xd3_decode_input + * functions and makes calls to the various input handling routines above, which + * coordinate external decompression. + */ +static int +main_input (xd3_cmd cmd, + main_file *ifile, + main_file *ofile, + main_file *sfile) +{ + int ret; + xd3_stream stream; + usize_t nread; + int stream_flags = 0; + xd3_config config; + xd3_source source; + xoff_t last_total_in = 0; + xoff_t last_total_out = 0; + long start_time; + xoff_t input_size = 0; + + int (*input_func) (xd3_stream*); + int (*output_func) (xd3_stream*, main_file *); + + memset (& source, 0, sizeof (source)); + memset (& config, 0, sizeof (config)); + + config.alloc = main_alloc; + config.freef = main_free1; + config.sec_data.ngroups = 1; + config.sec_addr.ngroups = 1; + config.sec_inst.ngroups = 1; + + /* main_input setup. */ + switch ((int) cmd) + { +#if VCDIFF_TOOLS + if (1) { case CMD_PRINTHDR: stream_flags = XD3_JUST_HDR; } + else if (1) { case CMD_PRINTHDRS: stream_flags = XD3_SKIP_WINDOW; } + else { case CMD_PRINTDELTA: stream_flags = XD3_SKIP_EMIT; } + ifile->flags |= RD_NONEXTERNAL; + input_func = xd3_decode_input; + output_func = main_print_func; + stream_flags |= XD3_ADLER32_NOVER; + break; +#endif +#if XD3_ENCODER + case CMD_ENCODE: + input_func = xd3_encode_input; + output_func = main_write_output; + + if (option_use_checksum) { stream_flags |= XD3_ADLER32; } + if (option_use_secondary) + { + /* The default secondary compressor is DJW, if it's compiled, being used, etc. */ + if (option_secondary == NULL) + { + if (SECONDARY_DJW) { stream_flags |= XD3_SEC_DJW; } + } + else + { + if (strcmp (option_secondary, "fgk") == 0 && SECONDARY_FGK) + { + stream_flags |= XD3_SEC_FGK; + } + else if (strcmp (option_secondary, "djw") == 0 && SECONDARY_DJW) + { + stream_flags |= XD3_SEC_DJW; + } + else + { + XPR(NT "unrecognized secondary compressor type: %s\n", option_secondary); + return EXIT_FAILURE; + } + } + } + if (option_no_compress) { stream_flags |= XD3_NOCOMPRESS; } + if (option_use_altcodetable) { stream_flags |= XD3_ALT_CODE_TABLE; } + if (option_smatch_config) + { + char *s = option_smatch_config, *e; + int values[XD3_SOFTCFG_VARCNT]; + int got; + + config.smatch_cfg = XD3_SMATCH_SOFT; + + for (got = 0; got < XD3_SOFTCFG_VARCNT; got += 1, s = e + 1) + { + values[got] = strtol (s, &e, 10); + + if ((values[got] < 0) || + (e == s) || + (got < XD3_SOFTCFG_VARCNT-1 && *e == 0) || + (got == XD3_SOFTCFG_VARCNT-1 && *e != 0)) + { + XPR(NT "invalid string match specifier (-C)\n"); + return EXIT_FAILURE; + } + } + + config.smatcher_soft.large_look = values[0]; + config.smatcher_soft.large_step = values[1]; + config.smatcher_soft.small_look = values[2]; + config.smatcher_soft.small_chain = values[3]; + config.smatcher_soft.small_lchain = values[4]; + config.smatcher_soft.ssmatch = values[5]; + config.smatcher_soft.try_lazy = values[6]; + config.smatcher_soft.max_lazy = values[7]; + config.smatcher_soft.long_enough = values[8]; + config.smatcher_soft.promote = values[9]; + } + else if (option_level < 5) { config.smatch_cfg = XD3_SMATCH_FAST; } + else { config.smatch_cfg = XD3_SMATCH_SLOW; } + break; +#endif + case CMD_DECODE: + if (option_use_checksum == 0) { stream_flags |= XD3_ADLER32_NOVER; } + stream_flags = 0; + ifile->flags |= RD_NONEXTERNAL; + input_func = xd3_decode_input; + output_func = main_write_output; + break; + default: + XPR(NT "internal error\n"); + return EXIT_FAILURE; + } + + start_time = get_millisecs_now (); + + if (main_file_stat (ifile, & input_size, 0) == 0) + { + // Note: to avoid unnecessary allocs for small files: problem + // was with --decompress_inputs, we don't know the true size + // option_winsize = min (input_size, (xoff_t) option_winsize); + } + + option_srcwinsz = max(option_srcwinsz, XD3_ALLOCSIZE); + option_winsize = max(option_winsize, XD3_ALLOCSIZE); + + source.blksize = (option_srcwinsz / 32) & ~(XD3_ALLOCSIZE-1); + source.blksize = max(XD3_DEFAULT_WINSIZE, source.blksize); + + config.srcwin_maxsz = option_srcwinsz; + config.winsize = option_winsize; + config.memsize = option_memsize; + config.getblk = main_getblk_func; + config.flags = stream_flags; + + if (option_verbose > 1) + { + XPR(NT "input buffer size: %u\n", option_winsize); + } + + if ((main_bdata = main_malloc (option_winsize)) == NULL) + { + return EXIT_FAILURE; + } + + if ((ret = xd3_config_stream (& stream, & config))) + { + XPR(NT XD3_LIB_ERRMSG (& stream, ret)); + return EXIT_FAILURE; + } + + if (IS_ENCODE (cmd)) + { + /* When encoding, open the source file, possibly decompress it. The decoder delays + * this step until XD3_GOTHEADER. */ + if (sfile->filename != NULL && (ret = main_set_source (& stream, cmd, sfile, & source))) + { + return EXIT_FAILURE; + } + } + + /* This times each window. */ + get_millisecs_since (); + + /* Main input loop. */ + do + { + xoff_t input_offset; + xoff_t input_remain; + usize_t try_read; + + input_offset = ifile->nread; + /*XD3_ASSERT (input_offset <= option_last_offset);*/ + + input_remain = /*option_last_offset*/ XOFF_T_MAX - input_offset; + + try_read = (usize_t) min ((xoff_t) config.winsize, input_remain); + + if ((ret = main_read_primary_input (ifile, main_bdata, try_read, & nread))) + { + return EXIT_FAILURE; + } + + /* If we've reached EOF tell the stream to flush. */ + if (nread < try_read) + { + stream_flags |= XD3_FLUSH; + xd3_set_flags (& stream, stream_flags); + } + +#if XD3_ENCODER + /* After the first main_read_primary_input completes, we know all the information + * needed to encode the application header. */ + if (cmd == CMD_ENCODE && (ret = main_set_appheader (& stream, ifile, sfile))) + { + return EXIT_FAILURE; + } +#endif + xd3_avail_input (& stream, main_bdata, nread); + + /* If we read zero bytes after encoding at least one window... */ + if (nread == 0 && stream.current_window > 0) { + break; + } + + again: + ret = input_func (& stream); + /*if (option_verbose > 1) { XPR(NT XD3_LIB_ERRMSG (& stream, ret)); }*/ + + switch (ret) + { + case XD3_INPUT: + continue; + + case XD3_GOTHEADER: + { + XD3_ASSERT (stream.current_window == 0); + + /* Need to process the appheader as soon as possible. It may contain a + * suggested default filename/decompression routine for the ofile, and it may + * contain default/decompression routine for the sources. */ + if (cmd == CMD_DECODE) + { + int have_src = sfile->filename != NULL; + int need_src = xd3_decoder_needs_source (& stream); + int recv_src; + + /* May need to set the sfile->filename if none was given. */ + main_get_appheader (& stream, ifile, ofile, sfile); + + recv_src = sfile->filename != NULL; + + /* Check if the user expected a source to be required although it was not. */ + if (have_src && ! need_src && ! option_quiet) + { + XPR(NT "warning: output window %"Q"u does not copy source\n", stream.current_window); + } + + /* Check if we have no source name and need one. */ + /* TODO: this doesn't fire due to cpyblocks_ calculation check */ + if (need_src && ! recv_src) + { + XPR(NT "input requires a source file, use -s\n"); + return EXIT_FAILURE; + } + + /* Now open the source file. */ + if (need_src && (ret = main_set_source (& stream, cmd, sfile, & source))) + { + return EXIT_FAILURE; + } + } + else if (cmd == CMD_PRINTHDR || + cmd == CMD_PRINTHDRS || + cmd == CMD_PRINTDELTA) + { + if (xd3_decoder_needs_source (& stream) && sfile->filename == NULL) + { + allow_fake_source = 1; + sfile->filename = ""; + main_set_source (& stream, cmd, sfile, & source); + } + } + } + /* FALLTHROUGH */ + case XD3_WINSTART: + { + /* Set or unset XD3_SKIP_WINDOW. */ + /*if (stream.current_window < option_first_window || stream.current_window > option_last_window) + { stream_flags |= XD3_SKIP_WINDOW; } + else + { stream_flags &= ~XD3_SKIP_WINDOW; }*/ + + xd3_set_flags (& stream, stream_flags); + goto again; + } + + case XD3_OUTPUT: + { + if (option_no_output == 0/* && + stream.current_window >= option_first_window && + stream.current_window <= option_last_window*/) + { + /* Defer opening the output file until the stream produces its first + * output for both encoder and decoder, this way we delay long enough for + * the decoder to receive the application header. (Or longer if there are + * skipped windows, but I can't think of any reason not to delay open.) */ + + if (! main_file_isopen (ofile) && (ret = main_open_output (& stream, ofile)) != 0) + { + return EXIT_FAILURE; + } + if ((ret = output_func (& stream, ofile)) && (ret != PRINTHDR_SPECIAL)) + { + return EXIT_FAILURE; + } + if (ret == PRINTHDR_SPECIAL) + { + xd3_abort_stream (& stream); + ret = EXIT_SUCCESS; + goto done; + } + ret = 0; + } + + xd3_consume_output (& stream); + goto again; + } + + case XD3_WINFINISH: + { + if (IS_ENCODE (cmd) || cmd == CMD_DECODE) + { + int used_source = xd3_encoder_used_source (& stream); + + if (! option_quiet && IS_ENCODE (cmd) && main_file_isopen (sfile) && ! used_source) + { + XPR(NT "warning: input position %"Q"u no source copies\n", + stream.current_window * option_winsize); + } + + if (option_verbose) + { + char rrateavg[32], wrateavg[32], tm[32]; + char rdb[32], wdb[32]; + char trdb[32], twdb[32]; + long millis = get_millisecs_since (); + usize_t this_read = stream.total_in - last_total_in; + usize_t this_write = stream.total_out - last_total_out; + last_total_in = stream.total_in; + last_total_out = stream.total_out; + + if (option_verbose > 1) + { + XPR(NT "%"Q"u: in %s (%s): out %s (%s): total in %s: out %s: %s\n", + stream.current_window, + main_format_bcnt (this_read, rdb), + main_format_rate (this_read, millis, rrateavg), + main_format_bcnt (this_write, wdb), + main_format_rate (this_write, millis, wrateavg), + main_format_bcnt (stream.total_in, trdb), + main_format_bcnt (stream.total_out, twdb), + main_format_millis (millis, tm)); + } + else + { + XPR(NT "%"Q"u: in %s: out %s: total in %s: out %s: %s\n", + stream.current_window, + main_format_bcnt (this_read, rdb), + main_format_bcnt (this_write, wdb), + main_format_bcnt (stream.total_in, trdb), + main_format_bcnt (stream.total_out, twdb), + main_format_millis (millis, tm)); + } + } + } + goto again; + } + + default: + /* input_func() error */ + XPR(NT XD3_LIB_ERRMSG (& stream, ret)); + return EXIT_FAILURE; + } + } + while (nread == config.winsize); +done: + /* Close the inputs. (ifile must be open, sfile may be open) */ + main_file_close (ifile); + main_file_close (sfile); + +#if EXTERNAL_COMPRESSION + if ((ret = main_external_compression_finish ())) + { + XPR(NT "external compression commands failed\n"); + return EXIT_FAILURE; + } +#endif + + /* If output file is not open yet because of delayed-open, it means we never encountered + * a window in the delta, but it could have had a VCDIFF header? TODO: solve this + * elsewhere. For now, it prints "nothing to output" below, but the check doesn't + * happen in case of option_no_output. */ + if (! option_no_output) + { + if (! main_file_isopen (ofile)) + { + XPR(NT "nothing to output: %s\n", ifile->filename); + return EXIT_FAILURE; + } + + /* Have to close the output before calling main_external_compression_finish, or else it hangs. */ + if (main_file_close (ofile) != 0) + { + return EXIT_FAILURE; + } + } + + if ((ret = xd3_close_stream (& stream))) + { + XPR(NT XD3_LIB_ERRMSG (& stream, ret)); + return EXIT_FAILURE; + } + + xd3_free_stream (& stream); + + if (option_verbose) + { + char tm[32]; + long end_time = get_millisecs_now (); + XPR(NT "command finished in %s\n", main_format_millis (end_time - start_time, tm)); + XPR(NT "input bytes: %"Q"u\n", ifile->nread); + XPR(NT "output bytes: %"Q"u\n", ofile->nwrite); + } + + return EXIT_SUCCESS; +} + +/* free memory before exit, reset single-use variables. */ +static void +main_cleanup (void) +{ + int i; + + if (option_appheader) { appheader_used = NULL; } + + main_free (appheader_used); + main_free (main_bdata); + +#if EXTERNAL_COMPRESSION + main_free (ext_tmpfile); +#endif + + for (i = 0; lru && i < lru_size; i += 1) + { + main_free (lru[i].blk); + } + + main_free (lru); + + lru_hits = 0; + lru_misses = 0; + lru_filled = 0; + + XD3_ASSERT (main_mallocs == 0); +} + +int +#if PYTHON_MODULE +xd3_main_cmdline (int argc, char **argv) +#else +main (int argc, char **argv) +#endif +{ + xd3_cmd cmd; + main_file ifile; + main_file ofile; + main_file sfile; + static char *flags = "0123456789cdefhnqvDJNORTVs:B:C:E:F:L:O:P:M:W:A::S::"; + int my_optind; + char *my_optarg; + char *my_optstr; + char *sfilename; + int orig_argc = argc; + char **orig_argv = argv; + int ret; + + main_file_init (& ifile); + main_file_init (& ofile); + main_file_init (& sfile); + + go: /* Go. */ + cmd = CMD_NONE; + sfilename = NULL; + my_optind = 1; + argv = orig_argv; + argc = orig_argc; + program_name = argv[0]; + extcomp_types[0].recomp_cmdname = program_name; + extcomp_types[0].decomp_cmdname = program_name; + takearg: + my_optarg = NULL; + my_optstr = argv[my_optind]; + /* This doesn't use getopt() because it makes trouble for -P & python which reenter + * main() and thus care about freeing all memory. I never had much trust for getopt + * anyway, it's too opaque. This implements a fairly standard non-long-option getopt + * with support for named operations (e.g., "xdelta3 [encode|decode|printhdr...] < in > + * out"). I'll probably add long options at some point. */ + if (my_optstr) + { + if (*my_optstr == '-') { my_optstr += 1; } + else if (cmd == CMD_NONE) { goto nonflag; } + else { my_optstr = NULL; } + } + while (my_optstr) + { + char *s; + my_optarg = NULL; + if ((ret = *my_optstr++) == 0) { my_optind += 1; goto takearg; } + + /* Option handling: first check for one ':' following the option in flags, then + * check for two. The syntax allows: + * + * 1. -Afoo defines optarg="foo" + * 2. -A foo defines optarg="foo" + * 3. -A "" defines optarg="" (allows optional empty-string) + * 4. -A [EOA or -moreargs] error (mandatory case) + * 5. -A [EOA -moreargs] defines optarg=NULL (optional case) + * 6. -A=foo defines optarg="foo" + * 7. -A= defines optarg="" (mandatory case) + * 8. -A= defines optarg=NULL (optional case) + * + * See tests in test_command_line_arguments(). + */ + s = strchr (flags, ret); + if (s && s[1] && s[1] == ':') + { + int eqcase = 0; + int option = s[2] && s[2] == ':'; + + /* Case 1, set optarg to the remaining characters. */ + my_optarg = my_optstr; + my_optstr = ""; + + /* Case 2-5 */ + if (*my_optarg == 0) + { + /* Condition 4-5 */ + int have_arg = my_optind < (argc - 1) && *argv[my_optind+1] != '-'; + + if (! have_arg) + { + if (! option) + { + /* Case 4 */ + XPR(NT "-%c: requires an argument\n", ret); + ret = EXIT_FAILURE; + goto cleanup; + } + /* Case 5. */ + my_optarg = NULL; + } + else + { + /* Case 2-3. */ + my_optarg = argv[++my_optind]; + } + } + /* Case 6-8. */ + else if (*my_optarg == '=') + { + /* Remove the = in all cases. */ + my_optarg += 1; + eqcase = 1; + + if (option && *my_optarg == 0) + { + /* Case 8. */ + my_optarg = NULL; + } + } + } + + switch (ret) + { + /* case: if no '-' was found, maybe check for a command name. */ + nonflag: + if (strcmp (my_optstr, "decode") == 0) { cmd = CMD_DECODE; } + else if (strcmp (my_optstr, "encode") == 0) + { +#if XD3_ENCODER + cmd = CMD_ENCODE; +#else + XPR(NT "encoder support not compiled\n"); + return EXIT_FAILURE; +#endif + } + else if (strcmp (my_optstr, "config") == 0) { cmd = CMD_CONFIG; } +#if REGRESSION_TEST + else if (strcmp (my_optstr, "test") == 0) { cmd = CMD_TEST; } +#endif +#if VCDIFF_TOOLS + else if (strcmp (my_optstr, "printhdr") == 0) { cmd = CMD_PRINTHDR; } + else if (strcmp (my_optstr, "printhdrs") == 0) { cmd = CMD_PRINTHDRS; } + else if (strcmp (my_optstr, "printdelta") == 0) { cmd = CMD_PRINTDELTA; } +#endif + + /* If no option was found and still no command, let the default command be + * encode. The remaining args are treated as filenames. */ + if (cmd == CMD_NONE) + { + cmd = CMD_DEFAULT; + my_optstr = NULL; + break; + } + else + { + /* But if we find a command name, continue the getopt loop. */ + my_optind += 1; + goto takearg; + } + + /* gzip-like options */ + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + option_level = ret - '0'; + break; + case 'f': option_force = 1; break; + case 'v': option_verbose += 1; option_quiet = 0; break; + case 'q': option_quiet = 1; option_verbose = 0; break; + case 'c': option_stdout = 1; break; + case 'd': + if (cmd == CMD_NONE) { cmd = CMD_DECODE; } + else { ret = main_help (); goto exit; } + break; + case 'e': +#if XD3_ENCODER + if (cmd == CMD_NONE) { cmd = CMD_ENCODE; } + else { ret = main_help (); goto exit; } + break; +#else + XPR(NT "encoder support not compiled\n"); + return EXIT_FAILURE; +#endif + case 'P': + /* only set profile count once, since... */ + if (option_profile_cnt == 0) + { + if ((ret = main_atou(my_optarg, (usize_t*) & option_profile_cnt, 0, 'P'))) { goto exit; } + + if (option_profile_cnt <= 0) + { + ret = EXIT_SUCCESS; + goto exit; + } + } + break; + + case 'n': option_use_checksum = 0; break; + case 'N': option_no_compress = 1; break; + case 'T': option_use_altcodetable = 1; break; + case 'C': option_smatch_config = my_optarg; break; + case 'J': option_no_output = 1; break; + case 'O': option_xdelta1 = my_optarg; break; + case 'S': if (my_optarg == NULL) { option_use_secondary = 0; } + else { option_use_secondary = 1; option_secondary = my_optarg; } break; + case 'A': if (my_optarg == NULL) { option_use_appheader = 0; } + else { option_appheader = (uint8_t*) my_optarg; } break; + case 'B': if ((ret = main_atou (my_optarg, & option_srcwinsz, XD3_ALLOCSIZE, 'B'))) + { + goto exit; + } + break; + case 'W': if ((ret = main_atou (my_optarg, & option_winsize, XD3_ALLOCSIZE, 'W'))) + { + goto exit; + } + break; + case 'M': if ((ret = main_atou (my_optarg, & option_memsize, XD3_ALLOCSIZE, 'M'))) + { + goto exit; + } + break; + case 'D': +#if EXTERNAL_COMPRESSION == 0 + if (! option_quiet) + { + XPR(NT "warning: -D option ignored, " + "external compression support was not compiled\n"); + } +#else + option_decompress_inputs = 0; +#endif + break; + case 'R': +#if EXTERNAL_COMPRESSION == 0 + if (! option_quiet) + { + XPR(NT "warning: -R option ignored, " + "external compression support was not compiled\n"); + } +#else + option_recompress_outputs = 0; +#endif + break; + case 's': + if (sfilename != NULL) + { + XPR(NT "specify only one source file\n"); + goto cleanup; + } + + sfilename = my_optarg; + break; + + case 'V': + ret = main_version (); goto exit; + default: + ret = main_help (); goto exit; + } + } + + option_source_filename = sfilename; + + /* In case there were no arguments, set the default command. */ + if (cmd == CMD_NONE) { cmd = CMD_DEFAULT; } + + argc -= my_optind; + argv += my_optind; + + /* There may be up to two more arguments. */ + if (argc > 2) + { + XPR(NT "too many filenames: %s ...\n", argv[2]); + ret = EXIT_FAILURE; + goto cleanup; + } + + extcomp_types[1].recomp_cmdname = option_xdelta1; + extcomp_types[1].decomp_cmdname = option_xdelta1; + + if (option_verbose > 1) + { + int l = 1; + int i; + char buf[1024]; + for (i = 0; i < orig_argc; i += 1) + { + l += strlen (orig_argv[i]) + 1; + } + buf[0] = 0; + for (i = 0; i < orig_argc; i += 1) + { + strcat (buf, orig_argv[i]); + strcat (buf, " "); + } + XPR(NT "command line: %s\n", buf); + } + + ifile.flags = RD_FIRST; + sfile.flags = RD_FIRST; + sfile.filename = option_source_filename; + + /* The infile takes the next argument, if there is one. But if not, infile is set to + * stdin. */ + if (argc > 0) + { + ifile.filename = argv[0]; + + if ((ret = main_file_open (& ifile, ifile.filename, XO_READ))) + { + goto cleanup; + } + } + else + { + XSTDIN_XF (& ifile); + } + + /* The ofile takes the following argument, if there is one. But if not, it is left NULL + * until the application header is processed. It will be set in main_open_output. */ + if (argc > 1) + { + /* Check for conflicting arguments. */ + if (option_stdout && ! option_quiet) + { + XPR(NT "warning: -c option overrides output filename: %s\n", argv[1]); + } + + if (! option_stdout) { ofile.filename = argv[1]; } + } + + switch (cmd) + { + case CMD_PRINTHDR: + case CMD_PRINTHDRS: + case CMD_PRINTDELTA: +#if XD3_ENCODER + case CMD_ENCODE: + if (cmd == CMD_ENCODE) + { + do_not_lru = 1; + } +#endif + case CMD_DECODE: + ret = main_input (cmd, & ifile, & ofile, & sfile); + break; + +#if REGRESSION_TEST + case CMD_TEST: + ret = xd3_selftest (); + break; +#endif + + case CMD_CONFIG: + ret = main_config (); + break; + + default: + ret = main_help (); + break; + } + +#if EXTERNAL_COMPRESSION + if (ext_tmpfile != NULL) { unlink (ext_tmpfile); } +#endif + + if (0) + { + cleanup: + ret = EXIT_FAILURE; + exit: + (void)0; + } + + main_file_cleanup (& ifile); + main_file_cleanup (& ofile); + main_file_cleanup (& sfile); + + main_cleanup (); + + if (--option_profile_cnt > 0 && ret == EXIT_SUCCESS) { goto go; } + + return ret; +} + +static int +main_help (void) +{ + /* TODO: update www/xdelta3-cmdline.html */ + main_version (); + P(RINT "usage: xdelta3 [command/options] [input [output]]\n"); + P(RINT "special command names:\n"); + P(RINT " config prints xdelta3 configuration\n"); + P(RINT " decode decompress the input\n"); + P(RINT " encode compress the input%s\n", XD3_ENCODER ? "" : " [Not compiled]"); +#if REGRESSION_TEST + P(RINT " test run the builtin tests\n"); +#endif +#if VCDIFF_TOOLS + P(RINT "special commands for VCDIFF inputs:\n"); + P(RINT " printdelta print information about the entire delta\n"); + P(RINT " printhdr print information about the first window\n"); + P(RINT " printhdrs print information about all windows\n"); +#endif + P(RINT "standard options:\n"); + P(RINT " -0 .. -9 compression level\n"); + P(RINT " -c use stdout\n"); + P(RINT " -d decompress\n"); + P(RINT " -e compress%s\n", XD3_ENCODER ? "" : " [Not compiled]"); + P(RINT " -f force overwrite\n"); + P(RINT " -h show help\n"); + P(RINT " -q be quiet\n"); + P(RINT " -v be verbose (max 2)\n"); + P(RINT " -V show version\n"); + + P(RINT "memory options:\n"); + P(RINT " -B blksize source file block size\n"); + P(RINT " -M memsize memory budget for hash tables\n"); + P(RINT " -W winsize input window buffer size\n"); + + P(RINT "compression options:\n"); + P(RINT " -s source source file to copy from (if any)\n"); + P(RINT " -S [djw|fgk] enable/disable secondary compression\n"); + P(RINT " -N disable small string-matching compression\n"); + P(RINT " -D disable external decompression (encode/decode)\n"); + P(RINT " -R disable external recompression (decode)\n"); + +#if XD3_DEBUG > 0 + P(RINT "developer options:\n"); + P(RINT " -A [apphead] disable/provide application header\n"); + P(RINT " -C soft config (see xdelta3-cfgs.h)\n"); + P(RINT " -J disable output (check/compute only)\n"); + P(RINT " -P repeat count (for profiling)\n"); + P(RINT " -T use alternate code table\n"); + P(RINT " -n disable checksum (encode/decode)\n"); +#endif + return EXIT_FAILURE; +} diff --git a/xdelta3/xdelta3-test.h b/xdelta3/xdelta3-test.h index 30d8629..c3cbd4d 100755 --- a/xdelta3/xdelta3-test.h +++ b/xdelta3/xdelta3-test.h @@ -32,7 +32,7 @@ static const usize_t TEST_FILE_MEAN = 16384; static const double TEST_ADD_MEAN = 16; static const double TEST_ADD_MAX = 256; static const double TEST_ADD_RATIO = 0.1; -static const double TEST_EPSILON = 0.5; +static const double TEST_EPSILON = 0.55; #define TESTBUFSIZE (1024 * 16) @@ -1522,8 +1522,8 @@ test_command_line_arguments (xd3_stream *stream, int ignore) /* Check that it is not too small, not too large. */ if (ratio >= TEST_ADD_RATIO + TEST_EPSILON) { - P(RINT "xdelta3: test encode with size ratio %.3f, expected < %.3f\n", - ratio, TEST_ADD_RATIO + TEST_EPSILON); + P(RINT "xdelta3: test encode with size ratio %.3f, expected < %.3f (%"Q"u, %"Q"u)\n", + ratio, TEST_ADD_RATIO + TEST_EPSILON, dsize, tsize); stream->msg = "strange encoding"; return XD3_INTERNAL; } -- cgit v1.2.3