summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xxdelta3/Makefile2
-rwxr-xr-xxdelta3/xdelta3-main.h5927
-rwxr-xr-xxdelta3/xdelta3-test.h6
3 files changed, 2972 insertions, 2963 deletions
diff --git a/xdelta3/Makefile b/xdelta3/Makefile
index a23ce28..dcbbace 100755
--- a/xdelta3/Makefile
+++ b/xdelta3/Makefile
@@ -62,7 +62,7 @@ xdelta3: $(SOURCES)
62 62
63xdelta3-debug: $(SOURCES) 63xdelta3-debug: $(SOURCES)
64 $(CC) -g -Wall -Wshadow xdelta3.c -o xdelta3-debug -DXD3_MAIN=1 -DGENERIC_ENCODE_TABLES=1 \ 64 $(CC) -g -Wall -Wshadow xdelta3.c -o xdelta3-debug -DXD3_MAIN=1 -DGENERIC_ENCODE_TABLES=1 \
65 -DXD3_USE_LARGEFILE64=1 -DXD3_STDIO=1 -DREGRESSION_TEST=1 -DXD3_DEBUG=2 -DSECONDARY_DJW=1 -lm 65 -DXD3_USE_LARGEFILE64=1 -DXD3_STDIO=1 -DREGRESSION_TEST=1 -DXD3_DEBUG=1 -DSECONDARY_DJW=1 -DSECONDARY_FGK=1 -lm
66 66
67$(PYTGT): $(SOURCES) 67$(PYTGT): $(SOURCES)
68 $(PYTHON) setup.py install --verbose --compile --force 68 $(PYTHON) setup.py install --verbose --compile --force
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h
index eb6ca9c..0b21c2f 100755
--- a/xdelta3/xdelta3-main.h
+++ b/xdelta3/xdelta3-main.h
@@ -1,2959 +1,2968 @@
1/* xdelta 3 - delta compression tools and library 1/* xdelta 3 - delta compression tools and library
2 * Copyright (C) 2001 and onward. Joshua P. MacDonald 2 * Copyright (C) 2001 and onward. Joshua P. MacDonald
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by 5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or 6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version. 7 * (at your option) any later version.
8 * 8 *
9 * This program is distributed in the hope that it will be useful, 9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software 15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 */ 17 */
18 18
19/* This is all the extra stuff you need for convenience to users in a command line 19/* This is all the extra stuff you need for convenience to users in a command line
20 * application. It contains these major components: 20 * application. It contains these major components:
21 * 21 *
22 * 1. VCDIFF tools 22 * 1. VCDIFF tools
23 * 2. external compression support (this is POSIX-specific). 23 * 2. external compression support (this is POSIX-specific).
24 * 3. a general read/write loop that handles all of the Xdelta decode/encode/VCDIFF-print 24 * 3. a general read/write loop that handles all of the Xdelta decode/encode/VCDIFF-print
25 * functions 25 * functions
26 * 4. command-line interpreter 26 * 4. command-line interpreter
27 * 5. an Xdelta application header which stores default filename, external compression settings 27 * 5. an Xdelta application header which stores default filename, external compression settings
28 * 6. output/error printing 28 * 6. output/error printing
29 * 7. basic file support and OS interface 29 * 7. basic file support and OS interface
30 */ 30 */
31 31
32/* TODO list: 32/* TODO list:
33 * 1. do exact gzip-like filename, stdout handling. make a .xz extension, refuse 33 * 1. do exact gzip-like filename, stdout handling. make a .xz extension, refuse
34 * to encode to stdout without -cf, etc. 34 * to encode to stdout without -cf, etc.
35 * 2. Allow the user to add a comment string to the app header without disturbing the default 35 * 2. Allow the user to add a comment string to the app header without disturbing the default
36 * behavior. 36 * behavior.
37 */ 37 */
38 38
39/* On error handling and printing: 39/* On error handling and printing:
40 * 40 *
41 * The xdelta library sets stream->msg to indicate what condition caused an internal 41 * The xdelta library sets stream->msg to indicate what condition caused an internal
42 * failure, but many failures originate here and are printed here. The return convention 42 * failure, but many failures originate here and are printed here. The return convention
43 * is 0 for success, as throughout Xdelta code, but special attention is required here for 43 * is 0 for success, as throughout Xdelta code, but special attention is required here for
44 * the operating system calls with different error handling. See the main_file_* routines. 44 * the operating system calls with different error handling. See the main_file_* routines.
45 * All errors in this file have a message printed at the time of occurance. Since some of 45 * All errors in this file have a message printed at the time of occurance. Since some of
46 * these calls occur within calls to the library, the error may end up being printed again 46 * these calls occur within calls to the library, the error may end up being printed again
47 * with a more general error message. 47 * with a more general error message.
48 */ 48 */
49 49
50/******************************************************************************************/ 50/******************************************************************************************/
51 51
52#ifndef XD3_POSIX 52#ifndef XD3_POSIX
53#define XD3_POSIX 0 53#define XD3_POSIX 0
54#endif 54#endif
55#ifndef XD3_STDIO 55#ifndef XD3_STDIO
56#define XD3_STDIO 0 56#define XD3_STDIO 0
57#endif 57#endif
58 58
59/* XPRINTX (used by main) prefixes an "xdelta3: " to the output. */ 59/* XPRINTX (used by main) prefixes an "xdelta3: " to the output. */
60#define XPR fprintf 60#define XPR fprintf
61#define NT stderr, "xdelta3: " 61#define NT stderr, "xdelta3: "
62 62
63#define VC fprintf 63#define VC fprintf
64#define UT vcout, 64#define UT vcout,
65 65
66/* If none are set, default to posix. */ 66/* If none are set, default to posix. */
67#if (XD3_POSIX + XD3_STDIO) == 0 67#if (XD3_POSIX + XD3_STDIO) == 0
68#undef XD3_POSIX 68#undef XD3_POSIX
69#define XD3_POSIX 1 69#define XD3_POSIX 1
70#endif 70#endif
71 71
72/* Handle externally-compressed inputs. */ 72/* Handle externally-compressed inputs. */
73#ifndef EXTERNAL_COMPRESSION 73#ifndef EXTERNAL_COMPRESSION
74#define EXTERNAL_COMPRESSION 1 74#define EXTERNAL_COMPRESSION 1
75#endif 75#endif
76 76
77#define PRINTHDR_SPECIAL -4378291 77#define PRINTHDR_SPECIAL -4378291
78 78
79/* The number of soft-config variables. */ 79/* The number of soft-config variables. */
80#define XD3_SOFTCFG_VARCNT 1 80#define XD3_SOFTCFG_VARCNT 1
81 81
82/* this is used as in XPR(NT XD3_LIB_ERRMSG (stream, ret)) to print an error message 82/* this is used as in XPR(NT XD3_LIB_ERRMSG (stream, ret)) to print an error message
83 * from the library. */ 83 * from the library. */
84#define XD3_LIB_ERRMSG(stream, ret) "%s: %s\n", xd3_errstring (stream), xd3_strerror (ret) 84#define XD3_LIB_ERRMSG(stream, ret) "%s: %s\n", xd3_errstring (stream), xd3_strerror (ret)
85 85
86#include <stdio.h> /* fprintf */ 86#include <stdio.h> /* fprintf */
87 87
88#if XD3_POSIX 88#if XD3_POSIX
89#include <unistd.h> /* close, read, write... */ 89#include <unistd.h> /* close, read, write... */
90#include <sys/types.h> 90#include <sys/types.h>
91#include <fcntl.h> 91#include <fcntl.h>
92#endif 92#endif
93 93
94#ifndef _WIN32 94#ifndef _WIN32
95#include <unistd.h> /* lots */ 95#include <unistd.h> /* lots */
96#include <sys/time.h> /* gettimeofday() */ 96#include <sys/time.h> /* gettimeofday() */
97#include <sys/stat.h> /* stat() and fstat() */ 97#include <sys/stat.h> /* stat() and fstat() */
98#else 98#else
99#define strtoll _strtoi64 99#define strtoll _strtoi64
100#include <sys/types.h> 100#include <sys/types.h>
101#include <sys/stat.h> 101#include <sys/stat.h>
102#ifndef WIFEXITED 102#ifndef WIFEXITED
103# define WIFEXITED(stat) (((*((int *) &(stat))) & 0xff) == 0) 103# define WIFEXITED(stat) (((*((int *) &(stat))) & 0xff) == 0)
104#endif 104#endif
105#ifndef WEXITSTATUS 105#ifndef WEXITSTATUS
106# define WEXITSTATUS(stat) (((*((int *) &(stat))) >> 8) & 0xff) 106# define WEXITSTATUS(stat) (((*((int *) &(stat))) >> 8) & 0xff)
107#endif 107#endif
108#ifndef S_ISREG 108#ifndef S_ISREG
109# ifdef S_IFREG 109# ifdef S_IFREG
110# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) 110# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
111# else 111# else
112# define S_ISREG(m) 1 112# define S_ISREG(m) 1
113# endif 113# endif
114#endif /* !S_ISREG */ 114#endif /* !S_ISREG */
115#endif 115#endif
116 116
117/****************************************************************************************** 117/******************************************************************************************
118 ENUMS and TYPES 118 ENUMS and TYPES
119 ******************************************************************************************/ 119 ******************************************************************************************/
120 120
121/* These flags (mainly pertaining to main_read() operations) are set in the 121/* These flags (mainly pertaining to main_read() operations) are set in the
122 * main_file->flags variable. All are related to with external decompression support. 122 * main_file->flags variable. All are related to with external decompression support.
123 * 123 *
124 * RD_FIRST causes the external decompression check when the input is first read. 124 * RD_FIRST causes the external decompression check when the input is first read.
125 * 125 *
126 * RD_NONEXTERNAL disables external decompression for reading a compressed input, in the 126 * RD_NONEXTERNAL disables external decompression for reading a compressed input, in the
127 * case of Xdelta inputs. Note: Xdelta is supported as an external compression type, 127 * case of Xdelta inputs. Note: Xdelta is supported as an external compression type,
128 * which makes is the reason for this flag. An example to justify this is: to create a 128 * which makes is the reason for this flag. An example to justify this is: to create a
129 * delta between two files that are VCDIFF-compressed. Two external Xdelta decoders are 129 * delta between two files that are VCDIFF-compressed. Two external Xdelta decoders are
130 * run to supply decompressed source and target inputs to the Xdelta encoder. */ 130 * run to supply decompressed source and target inputs to the Xdelta encoder. */
131typedef enum 131typedef enum
132{ 132{
133 RD_FIRST = (1 << 0), 133 RD_FIRST = (1 << 0),
134 RD_NONEXTERNAL = (1 << 1), 134 RD_NONEXTERNAL = (1 << 1),
135 RD_EXTERNAL_V1 = (1 << 2), 135 RD_EXTERNAL_V1 = (1 << 2),
136} xd3_read_flags; 136} xd3_read_flags;
137 137
138/* main_file->mode values */ 138/* main_file->mode values */
139typedef enum 139typedef enum
140{ 140{
141 XO_READ = 0, 141 XO_READ = 0,
142 XO_WRITE = 1, 142 XO_WRITE = 1,
143} main_file_modes; 143} main_file_modes;
144 144
145/* Main commands. For example, CMD_PRINTHDR is the "xdelta printhdr" command. */ 145/* Main commands. For example, CMD_PRINTHDR is the "xdelta printhdr" command. */
146typedef enum 146typedef enum
147{ 147{
148 CMD_NONE = 0, 148 CMD_NONE = 0,
149 CMD_PRINTHDR, 149 CMD_PRINTHDR,
150 CMD_PRINTHDRS, 150 CMD_PRINTHDRS,
151 CMD_PRINTDELTA, 151 CMD_PRINTDELTA,
152#if XD3_ENCODER 152#if XD3_ENCODER
153 CMD_ENCODE, 153 CMD_ENCODE,
154#endif 154#endif
155 CMD_DECODE, 155 CMD_DECODE,
156 CMD_TEST, 156 CMD_TEST,
157 CMD_CONFIG, 157 CMD_CONFIG,
158} xd3_cmd; 158} xd3_cmd;
159 159
160#if XD3_ENCODER 160#if XD3_ENCODER
161#define CMD_DEFAULT CMD_ENCODE 161#define CMD_DEFAULT CMD_ENCODE
162#define IS_ENCODE(cmd) (cmd == CMD_ENCODE) 162#define IS_ENCODE(cmd) (cmd == CMD_ENCODE)
163#else 163#else
164#define CMD_DEFAULT CMD_DECODE 164#define CMD_DEFAULT CMD_DECODE
165#define IS_ENCODE(cmd) (0) 165#define IS_ENCODE(cmd) (0)
166#endif 166#endif
167 167
168typedef struct _main_file main_file; 168typedef struct _main_file main_file;
169typedef struct _main_extcomp main_extcomp; 169typedef struct _main_extcomp main_extcomp;
170typedef struct _main_blklru main_blklru; 170typedef struct _main_blklru main_blklru;
171typedef struct _main_blklru_list main_blklru_list; 171typedef struct _main_blklru_list main_blklru_list;
172 172
173/* The main_file object supports abstract system calls like open, close, read, write, seek, 173/* The main_file object supports abstract system calls like open, close, read, write, seek,
174 * stat. The program uses these to represent both seekable files and non-seekable files. 174 * stat. The program uses these to represent both seekable files and non-seekable files.
175 * Source files must be seekable, but the target input and any output file do not require 175 * Source files must be seekable, but the target input and any output file do not require
176 * seekability. 176 * seekability.
177 */ 177 */
178struct _main_file 178struct _main_file
179{ 179{
180#if XD3_STDIO 180#if XD3_STDIO
181 FILE *file; 181 FILE *file;
182#elif XD3_POSIX 182#elif XD3_POSIX
183 int file; 183 int file;
184#endif 184#endif
185 185
186 int mode; /* XO_READ and XO_WRITE */ 186 int mode; /* XO_READ and XO_WRITE */
187 const char *filename; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */ 187 const char *filename; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */
188 char *filename_copy; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */ 188 char *filename_copy; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */
189 const char *realname; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */ 189 const char *realname; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */
190 const main_extcomp *compressor; /* External compression struct. */ 190 const main_extcomp *compressor; /* External compression struct. */
191 int flags; /* RD_FIRST, RD_NONEXTERNAL, ... */ 191 int flags; /* RD_FIRST, RD_NONEXTERNAL, ... */
192 xoff_t nread; /* for input position */ 192 xoff_t nread; /* for input position */
193 xoff_t nwrite; /* for output position */ 193 xoff_t nwrite; /* for output position */
194}; 194};
195 195
196/* Various strings and magic values used to detect and call external compression. See 196/* Various strings and magic values used to detect and call external compression. See
197 * below for examples. */ 197 * below for examples. */
198struct _main_extcomp 198struct _main_extcomp
199{ 199{
200 const char *recomp_cmdname; 200 const char *recomp_cmdname;
201 const char *recomp_options; 201 const char *recomp_options;
202 202
203 const char *decomp_cmdname; 203 const char *decomp_cmdname;
204 const char *decomp_options; 204 const char *decomp_options;
205 205
206 const char *ident; 206 const char *ident;
207 const char *magic; 207 const char *magic;
208 int magic_size; 208 int magic_size;
209 int flags; 209 int flags;
210}; 210};
211 211
212/* This file implements a small LRU of source blocks. For encoding purposes, 212/* This file implements a small LRU of source blocks. For encoding purposes,
213 * we prevent paging in blocks we've already scanned in the source (return 213 * we prevent paging in blocks we've already scanned in the source (return
214 * XD3_NOTAVAIL). */ 214 * XD3_NOTAVAIL). */
215struct _main_blklru_list 215struct _main_blklru_list
216{ 216{
217 main_blklru_list *next; 217 main_blklru_list *next;
218 main_blklru_list *prev; 218 main_blklru_list *prev;
219}; 219};
220 220
221struct _main_blklru 221struct _main_blklru
222{ 222{
223 uint8_t *blk; 223 uint8_t *blk;
224 xoff_t blkno; 224 xoff_t blkno;
225 main_blklru_list link; 225 main_blklru_list link;
226}; 226};
227 227
228/* ... represented as a list (no cache index). */ 228/* ... represented as a list (no cache index). */
229XD3_MAKELIST(main_blklru_list,main_blklru,link); 229XD3_MAKELIST(main_blklru_list,main_blklru,link);
230 230
231/* Program options: various command line flags and options. */ 231/* Program options: various command line flags and options. */
232static int option_stdout = 0; 232static int option_stdout = 0;
233static int option_force = 0; 233static int option_force = 0;
234static int option_verbose = 0; 234static int option_verbose = 0;
235static int option_quiet = 0; 235static int option_quiet = 0;
236static int option_level = 6; 236static int option_level = 6;
237static int option_use_appheader = 1; 237static int option_use_appheader = 1;
238static uint8_t* option_appheader = NULL; 238static uint8_t* option_appheader = NULL;
239static int option_use_secondary = /* until-standardized, leave this off */ 0; 239static int option_use_secondary = /* until-standardized, leave this off */ 0;
240static char* option_secondary = NULL; 240static char* option_secondary = NULL;
241static int option_use_checksum = 1; 241static int option_use_checksum = 1;
242static int option_use_altcodetable = 0; 242static int option_use_altcodetable = 0;
243static char* option_smatch_config = NULL; 243static char* option_smatch_config = NULL;
244static int option_no_compress = 0; 244static int option_no_compress = 0;
245static int option_no_output = 0; /* go through the motions, but do not open or write output */ 245static int option_no_output = 0; /* go through the motions, but do not open or write output */
246static const char *option_source_filename = NULL; 246static const char *option_source_filename = NULL;
247static const char *option_xdelta1 = "xdelta1"; 247static const char *option_xdelta1 = "xdelta1";
248 248
249static usize_t option_winsize = XD3_DEFAULT_WINSIZE; 249static usize_t option_winsize = XD3_DEFAULT_WINSIZE;
250static usize_t option_srcwinsz = XD3_DEFAULT_SRCWINSZ; 250static usize_t option_srcwinsz = XD3_DEFAULT_SRCWINSZ;
251static usize_t option_memsize = XD3_DEFAULT_MEMSIZE; 251static usize_t option_memsize = XD3_DEFAULT_MEMSIZE;
252 252
253/* This controls the number of times main repeats itself, only for profiling. */ 253/* This controls the number of times main repeats itself, only for profiling. */
254static int option_profile_cnt = 0; 254static int option_profile_cnt = 0;
255 255
256/* These variables are supressed to avoid their use w/o support. main() warns 256/* These variables are supressed to avoid their use w/o support. main() warns
257 * appropriately. */ 257 * appropriately. */
258#if EXTERNAL_COMPRESSION 258#if EXTERNAL_COMPRESSION
259static int option_decompress_inputs = 1; 259static int option_decompress_inputs = 1;
260static int option_recompress_outputs = 1; 260static int option_recompress_outputs = 1;
261#endif 261#endif
262 262
263/* This is for comparing "printdelta" output without attention to 263/* This is for comparing "printdelta" output without attention to
264 * copy-instruction modes, useful for reverse engineering. */ 264 * copy-instruction modes, useful for reverse engineering. */
265#if VCDIFF_TOOLS 265#if VCDIFF_TOOLS
266static int option_print_cpymode = 1; 266static int option_print_cpymode = 1;
267#endif 267#endif
268 268
269/* Static variables */ 269/* Static variables */
270IF_DEBUG(static int main_mallocs = 0;) 270IF_DEBUG(static int main_mallocs = 0;)
271 271
272static char* program_name = NULL; 272static char* program_name = NULL;
273static uint8_t* appheader_used = NULL; 273static uint8_t* appheader_used = NULL;
274static uint8_t* main_bdata = NULL; 274static uint8_t* main_bdata = NULL;
275 275
276/* The LRU: obviously this is shared by all callers. */ 276/* The LRU: obviously this is shared by all callers. */
277static int lru_size = 0; 277static int lru_size = 0;
278static main_blklru *lru = NULL; /* array of lru_size elts */ 278static main_blklru *lru = NULL; /* array of lru_size elts */
279static main_blklru_list lru_list; 279static main_blklru_list lru_list;
280static main_blklru_list lru_free; 280static main_blklru_list lru_free;
281static int do_not_lru = 0; /* set to avoid lru, instead discard oldest */ 281static int do_not_lru = 0; /* set to avoid lru, instead discard oldest */
282 282
283static int lru_hits = 0; 283static int lru_hits = 0;
284static int lru_misses = 0; 284static int lru_misses = 0;
285static int lru_filled = 0; 285static int lru_filled = 0;
286 286
287/* Hacks for VCDIFF tools */ 287/* Hacks for VCDIFF tools */
288static int allow_fake_source = 0; 288static int allow_fake_source = 0;
289 289
290/* This array of compressor types is compiled even if EXTERNAL_COMPRESSION is false just so 290/* This array of compressor types is compiled even if EXTERNAL_COMPRESSION is false just so
291 * the program knows the mapping of IDENT->NAME. */ 291 * the program knows the mapping of IDENT->NAME. */
292static main_extcomp extcomp_types[] = 292static main_extcomp extcomp_types[] =
293{ 293{
294 /* The entry for xdelta3/1 must be 0/1 because the program_names are set there. */ 294 /* The entry for xdelta3/1 must be 0/1 because the program_names are set there. */
295 { "xdelta3", "-cfq", "xdelta3", "-dcfq", "X", "\xd6\xc3\xc4", 3, RD_NONEXTERNAL }, 295 { "xdelta3", "-cfq", "xdelta3", "-dcfq", "X", "\xd6\xc3\xc4", 3, RD_NONEXTERNAL },
296 { "xdelta1", "delta", "xdelta1", "patch", "1", "%XD", 3, RD_EXTERNAL_V1 }, 296 { "bzip2", "-cf", "bzip2", "-dcf", "B", "BZh", 3, 0 },
297 { "bzip2", "-cf", "bzip2", "-dcf", "B", "BZh", 3, 0 }, 297 { "gzip", "-cf", "gzip", "-dcf", "G", "\037\213", 2, 0 },
298 { "gzip", "-cf", "gzip", "-dcf", "G", "\037\213", 2, 0 }, 298 { "compress", "-cf", "uncompress", "-cf", "Z", "\037\235", 2, 0 },
299 { "compress", "-cf", "uncompress", "-cf", "Z", "\037\235", 2, 0 }, 299
300}; 300 /* TODO: xdelta1 isn't working */
301 301 /*{ "xdelta1", "delta", "xdelta1", "patch", "1", "%XD", 3, RD_EXTERNAL_V1 },*/
302static void main_get_appheader (xd3_stream *stream, main_file *ifile, 302
303 main_file *output, main_file *sfile); 303 /* TODO: add commandline support for magic-less formats */
304 304 /*{ "lzma", "-cf", "lzma", "-dcf", "M", "]\000", 2, 0 },*/
305static int main_help (void); 305};
306 306
307static int 307static void main_get_appheader (xd3_stream *stream, main_file *ifile,
308main_version (void) 308 main_file *output, main_file *sfile);
309{ 309
310 /* $Format: " P(RINT \"VERSION=3.$Xdelta3Version$\\n\");" $ */ 310static int main_help (void);
311 P(RINT "VERSION=3.0h_pre0\n"); 311
312 return EXIT_SUCCESS; 312static int
313} 313main_version (void)
314 314{
315static int 315 /* $Format: " P(RINT \"VERSION=3.$Xdelta3Version$\\n\");" $ */
316main_config (void) 316 P(RINT "VERSION=3.0h_pre0\n");
317{ 317 return EXIT_SUCCESS;
318 main_version (); 318}
319 319
320 P(RINT "EXTERNAL_COMPRESSION=%d\n", EXTERNAL_COMPRESSION); 320static int
321 P(RINT "GENERIC_ENCODE_TABLES=%d\n", GENERIC_ENCODE_TABLES); 321main_config (void)
322 P(RINT "GENERIC_ENCODE_TABLES_COMPUTE=%d\n", GENERIC_ENCODE_TABLES_COMPUTE); 322{
323 P(RINT "REGRESSION_TEST=%d\n", REGRESSION_TEST); 323 main_version ();
324 P(RINT "SECONDARY_DJW=%d\n", SECONDARY_DJW); 324
325 P(RINT "SECONDARY_FGK=%d\n", SECONDARY_FGK); 325 P(RINT "EXTERNAL_COMPRESSION=%d\n", EXTERNAL_COMPRESSION);
326 P(RINT "VCDIFF_TOOLS=%d\n", VCDIFF_TOOLS); 326 P(RINT "GENERIC_ENCODE_TABLES=%d\n", GENERIC_ENCODE_TABLES);
327 P(RINT "XD3_ALLOCSIZE=%d\n", XD3_ALLOCSIZE); 327 P(RINT "GENERIC_ENCODE_TABLES_COMPUTE=%d\n", GENERIC_ENCODE_TABLES_COMPUTE);
328 P(RINT "XD3_DEBUG=%d\n", XD3_DEBUG); 328 P(RINT "REGRESSION_TEST=%d\n", REGRESSION_TEST);
329 P(RINT "XD3_DEFAULT_CKSUM_SIZE=%d\n", XD3_DEFAULT_CKSUM_ADVANCE); 329 P(RINT "SECONDARY_DJW=%d\n", SECONDARY_DJW);
330 P(RINT "XD3_DEFAULT_IOPT_SIZE=%d\n", XD3_DEFAULT_IOPT_SIZE); 330 P(RINT "SECONDARY_FGK=%d\n", SECONDARY_FGK);
331 P(RINT "XD3_DEFAULT_MEMSIZE=%d\n", XD3_DEFAULT_MEMSIZE); 331 P(RINT "VCDIFF_TOOLS=%d\n", VCDIFF_TOOLS);
332 P(RINT "XD3_DEFAULT_SPREVSZ=%d\n", XD3_DEFAULT_SPREVSZ); 332 P(RINT "XD3_ALLOCSIZE=%d\n", XD3_ALLOCSIZE);
333 P(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ); 333 P(RINT "XD3_DEBUG=%d\n", XD3_DEBUG);
334 P(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE); 334 P(RINT "XD3_DEFAULT_CKSUM_SIZE=%d\n", XD3_DEFAULT_CKSUM_ADVANCE);
335 P(RINT "XD3_ENCODER=%d\n", XD3_ENCODER); 335 P(RINT "XD3_DEFAULT_IOPT_SIZE=%d\n", XD3_DEFAULT_IOPT_SIZE);
336 P(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE); 336 P(RINT "XD3_DEFAULT_MEMSIZE=%d\n", XD3_DEFAULT_MEMSIZE);
337 P(RINT "XD3_NODECOMPRESSSIZE=%d\n", XD3_NODECOMPRESSSIZE); 337 P(RINT "XD3_DEFAULT_SPREVSZ=%d\n", XD3_DEFAULT_SPREVSZ);
338 P(RINT "XD3_POSIX=%d\n", XD3_POSIX); 338 P(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ);
339 P(RINT "XD3_USE_LARGEFILE64=%d\n", XD3_USE_LARGEFILE64); 339 P(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE);
340 340 P(RINT "XD3_ENCODER=%d\n", XD3_ENCODER);
341 return EXIT_SUCCESS; 341 P(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE);
342} 342 P(RINT "XD3_NODECOMPRESSSIZE=%d\n", XD3_NODECOMPRESSSIZE);
343 343 P(RINT "XD3_POSIX=%d\n", XD3_POSIX);
344static void* 344 P(RINT "XD3_USE_LARGEFILE64=%d\n", XD3_USE_LARGEFILE64);
345main_malloc1 (usize_t size) 345
346{ 346 return EXIT_SUCCESS;
347 void* r = malloc (size); 347}
348 if (r == NULL) { XPR(NT "malloc: %s\n", xd3_strerror (ENOMEM)); } 348
349 else if (option_verbose > 2) { XPR(NT "malloc: %u: %p\n", size, r); } 349static void*
350 return r; 350main_malloc1 (usize_t size)
351} 351{
352 352 void* r = malloc (size);
353static void* 353 if (r == NULL) { XPR(NT "malloc: %s\n", xd3_strerror (ENOMEM)); }
354main_malloc (usize_t size) 354 else if (option_verbose > 2) { XPR(NT "malloc: %u: %p\n", size, r); }
355{ 355 return r;
356 void *r = main_malloc1 (size); 356}
357 if (r) { IF_DEBUG (main_mallocs += 1); } 357
358 return r; 358static void*
359} 359main_malloc (usize_t size)
360 360{
361static void* 361 void *r = main_malloc1 (size);
362main_alloc (void *opaque, 362 if (r) { IF_DEBUG (main_mallocs += 1); }
363 usize_t items, 363 return r;
364 usize_t size) 364}
365{ 365
366 return main_malloc1 (items * size); 366static void*
367} 367main_alloc (void *opaque,
368 368 usize_t items,
369static void 369 usize_t size)
370main_free1 (void *opaque, void *ptr) 370{
371{ 371 return main_malloc1 (items * size);
372 if (option_verbose > 2) { XPR(NT "free: %p\n", ptr); } 372}
373 free (ptr); 373
374} 374static void
375 375main_free1 (void *opaque, void *ptr)
376static void 376{
377main_free (void *ptr) 377 if (option_verbose > 2) { XPR(NT "free: %p\n", ptr); }
378{ 378 free (ptr);
379 if (ptr) 379}
380 { 380
381 IF_DEBUG (main_mallocs -= 1); 381static void
382 main_free1 (NULL, ptr); 382main_free (void *ptr)
383 } 383{
384} 384 if (ptr)
385 385 {
386/* This ensures that (ret = errno) always indicates failure, in case errno was 386 IF_DEBUG (main_mallocs -= 1);
387 * accidentally not set. If this prints there's a bug somewhere. */ 387 main_free1 (NULL, ptr);
388static int 388 }
389get_errno (void) 389}
390{ 390
391 if (errno == 0) 391/* This ensures that (ret = errno) always indicates failure, in case errno was
392 { 392 * accidentally not set. If this prints there's a bug somewhere. */
393 XPR(NT "you found a bug: expected errno != 0\n"); 393static int
394 errno = XD3_INTERNAL; 394get_errno (void)
395 } 395{
396 return errno; 396 if (errno == 0)
397} 397 {
398 398 XPR(NT "you found a bug: expected errno != 0\n");
399 399 errno = XD3_INTERNAL;
400 400 }
401static long 401 return errno;
402get_millisecs_now (void) 402}
403{ 403
404#ifndef WIN32 404
405 struct timeval tv; 405
406 406static long
407 gettimeofday (& tv, NULL); 407get_millisecs_now (void)
408 408{
409 return (tv.tv_sec) * 1000L + (tv.tv_usec) / 1000; 409#ifndef WIN32
410#else 410 struct timeval tv;
411 // Found this in an example on www.codeproject.com 411
412 // It doesn't matter that the offset is Jan 1, 1601 412 gettimeofday (& tv, NULL);
413 // Result is the numbre of 100 nanosecond units 413
414 // 100ns * 10,000 = 1ms 414 return (tv.tv_sec) * 1000L + (tv.tv_usec) / 1000;
415 SYSTEMTIME st; 415#else
416 FILETIME ft; 416 // Found this in an example on www.codeproject.com
417 __int64 *pi = (__int64*)&ft; 417 // It doesn't matter that the offset is Jan 1, 1601
418 GetLocalTime(&st); 418 // Result is the numbre of 100 nanosecond units
419 SystemTimeToFileTime(&st, &ft); 419 // 100ns * 10,000 = 1ms
420 return (long)((*pi) / 10000); 420 SYSTEMTIME st;
421#endif 421 FILETIME ft;
422} 422 __int64 *pi = (__int64*)&ft;
423 423 GetLocalTime(&st);
424/* Always >= 1 millisec, right? */ 424 SystemTimeToFileTime(&st, &ft);
425static long 425 return (long)((*pi) / 10000);
426get_millisecs_since (void) 426#endif
427{ 427}
428 static long last = 0; 428
429 long now = get_millisecs_now(); 429/* Always >= 1 millisec, right? */
430 long diff = now - last; 430static long
431 last = now; 431get_millisecs_since (void)
432 return diff; 432{
433} 433 static long last = 0;
434 434 long now = get_millisecs_now();
435static char* 435 long diff = now - last;
436main_format_bcnt (xoff_t r, char *buf) 436 last = now;
437{ 437 return diff;
438 static const char* fmts[] = { "B", "KB", "MB", "GB" }; 438}
439 int i; 439
440 440static char*
441 for (i = 0; i < SIZEOF_ARRAY(fmts); i += 1) 441main_format_bcnt (xoff_t r, char *buf)
442 { 442{
443 if (r < 10 * 1e3 || i == (-1 + (int)SIZEOF_ARRAY(fmts))) 443 static const char* fmts[] = { "B", "KB", "MB", "GB" };
444 { 444 int i;
445 sprintf (buf, "%"Q"u %s", r, fmts[i]); 445
446 break; 446 for (i = 0; i < SIZEOF_ARRAY(fmts); i += 1)
447 } 447 {
448 r /= 1000; 448 if (r < 10 * 1e3 || i == (-1 + (int)SIZEOF_ARRAY(fmts)))
449 } 449 {
450 return buf; 450 sprintf (buf, "%"Q"u %s", r, fmts[i]);
451} 451 break;
452 452 }
453static char* 453 r /= 1000;
454main_format_rate (xoff_t bytes, long millis, char *buf) 454 }
455{ 455 return buf;
456 xoff_t r = (xoff_t)(1.0 * bytes / (1.0 * millis / 1000.0)); 456}
457 static char lbuf[32]; 457
458 458static char*
459 main_format_bcnt (r, lbuf); 459main_format_rate (xoff_t bytes, long millis, char *buf)
460 sprintf (buf, "%s/sec", lbuf); 460{
461 return buf; 461 xoff_t r = (xoff_t)(1.0 * bytes / (1.0 * millis / 1000.0));
462} 462 static char lbuf[32];
463 463
464static char* 464 main_format_bcnt (r, lbuf);
465main_format_millis (long millis, char *buf) 465 sprintf (buf, "%s/sec", lbuf);
466{ 466 return buf;
467 if (millis < 1000) { sprintf (buf, "%lu ms", millis); } 467}
468 else if (millis < 10000) { sprintf (buf, "%.1f sec", millis / 1000.0); } 468
469 else { sprintf (buf, "%lu sec", millis / 1000L); } 469static char*
470 return buf; 470main_format_millis (long millis, char *buf)
471} 471{
472 472 if (millis < 1000) { sprintf (buf, "%lu ms", millis); }
473/* A safe version of strtol for xoff_t. */ 473 else if (millis < 10000) { sprintf (buf, "%.1f sec", millis / 1000.0); }
474static int 474 else { sprintf (buf, "%lu sec", millis / 1000L); }
475main_strtoxoff (const char* s, xoff_t *xo, char which) 475 return buf;
476{ 476}
477 char *e; 477
478 xoff_t x; 478/* A safe version of strtol for xoff_t. */
479 479static int
480 XD3_ASSERT(s && *s != 0); 480main_strtoxoff (const char* s, xoff_t *xo, char which)
481 481{
482 { 482 char *e;
483 /* Should check LONG_MIN, LONG_MAX, LLONG_MIN, LLONG_MAX? */ 483 xoff_t x;
484#if SIZEOF_XOFF_T == 4 484
485 long xx = strtol (s, &e, 0); 485 XD3_ASSERT(s && *s != 0);
486#else 486
487 long long xx = strtoll (s, &e, 0); 487 {
488#endif 488 /* Should check LONG_MIN, LONG_MAX, LLONG_MIN, LLONG_MAX? */
489 489#if SIZEOF_XOFF_T == 4
490 if (xx < 0) 490 long xx = strtol (s, &e, 0);
491 { 491#else
492 XPR(NT "-%c: negative integer: %s\n", which, s); 492 long long xx = strtoll (s, &e, 0);
493 return EXIT_FAILURE; 493#endif
494 } 494
495 495 if (xx < 0)
496 x = xx; 496 {
497 } 497 XPR(NT "-%c: negative integer: %s\n", which, s);
498 498 return EXIT_FAILURE;
499 if (*e != 0) 499 }
500 { 500
501 XPR(NT "-%c: invalid integer: %s\n", which, s); 501 x = xx;
502 return EXIT_FAILURE; 502 }
503 } 503
504 504 if (*e != 0)
505 (*xo) = x; 505 {
506 return 0; 506 XPR(NT "-%c: invalid integer: %s\n", which, s);
507} 507 return EXIT_FAILURE;
508 508 }
509static int 509
510main_atou (const char* arg, usize_t *xo, usize_t low, char which) 510 (*xo) = x;
511{ 511 return 0;
512 xoff_t x; 512}
513 int ret; 513
514 514static int
515 if ((ret = main_strtoxoff (arg, & x, which))) { return ret; } 515main_atou (const char* arg, usize_t *xo, usize_t low, char which)
516 516{
517 if (x > USIZE_T_MAX || x < low) 517 xoff_t x;
518 { 518 int ret;
519 XPR(NT "-%c: minimum value: %u", which, low); 519
520 return EXIT_FAILURE; 520 if ((ret = main_strtoxoff (arg, & x, which))) { return ret; }
521 } 521
522 (*xo) = (usize_t)x; 522 if (x > USIZE_T_MAX || x < low)
523 return 0; 523 {
524} 524 XPR(NT "-%c: minimum value: %u", which, low);
525 525 return EXIT_FAILURE;
526/****************************************************************************************** 526 }
527 FILE BASICS 527 (*xo) = (usize_t)x;
528 ******************************************************************************************/ 528 return 0;
529 529}
530/* With all the variation in file system-call semantics, arguments, return values and 530
531 * error-handling for the POSIX and STDIO file APIs, the insides of these functions make 531/******************************************************************************************
532 * me sick, which is why these wrappers exist. */ 532 FILE BASICS
533 533 ******************************************************************************************/
534#define XOPEN_OPNAME (xfile->mode == XO_READ ? "read" : "write") 534
535#define XOPEN_STDIO (xfile->mode == XO_READ ? "rb" : "wb") 535/* With all the variation in file system-call semantics, arguments, return values and
536#define XOPEN_POSIX (xfile->mode == XO_READ ? O_RDONLY : O_WRONLY | O_CREAT | O_TRUNC) 536 * error-handling for the POSIX and STDIO file APIs, the insides of these functions make
537#define XOPEN_MODE (xfile->mode == XO_READ ? 0 : 0666) 537 * me sick, which is why these wrappers exist. */
538 538
539#define XF_ERROR(op, name, ret) XPR(NT "file %s failed: %s: %s: %s\n", (op), XOPEN_OPNAME, (name), xd3_strerror (ret)) 539#define XOPEN_OPNAME (xfile->mode == XO_READ ? "read" : "write")
540 540#define XOPEN_STDIO (xfile->mode == XO_READ ? "rb" : "wb")
541#if XD3_STDIO 541#define XOPEN_POSIX (xfile->mode == XO_READ ? O_RDONLY : O_WRONLY | O_CREAT | O_TRUNC)
542#define XFNO(f) fileno(f->file) 542#define XOPEN_MODE (xfile->mode == XO_READ ? 0 : 0666)
543#define XSTDOUT_XF(f) { (f)->file = stdout; (f)->filename = "/dev/stdout"; } 543
544#define XSTDERR_XF(f) { (f)->file = stderr; (f)->filename = "/dev/stderr"; } 544#define XF_ERROR(op, name, ret) XPR(NT "file %s failed: %s: %s: %s\n", (op), XOPEN_OPNAME, (name), xd3_strerror (ret))
545#define XSTDIN_XF(f) { (f)->file = stdin; (f)->filename = "/dev/stdin"; } 545
546 546#if XD3_STDIO
547#elif XD3_POSIX 547#define XFNO(f) fileno(f->file)
548#define XFNO(f) f->file 548#define XSTDOUT_XF(f) { (f)->file = stdout; (f)->filename = "/dev/stdout"; }
549#define XSTDOUT_XF(f) { (f)->file = STDOUT_FILENO; (f)->filename = "/dev/stdout"; } 549#define XSTDERR_XF(f) { (f)->file = stderr; (f)->filename = "/dev/stderr"; }
550#define XSTDERR_XF(f) { (f)->file = STDERR_FILENO; (f)->filename = "/dev/stderr"; } 550#define XSTDIN_XF(f) { (f)->file = stdin; (f)->filename = "/dev/stdin"; }
551#define XSTDIN_XF(f) { (f)->file = STDIN_FILENO; (f)->filename = "/dev/stdin"; } 551
552#endif 552#elif XD3_POSIX
553 553#define XFNO(f) f->file
554static void 554#define XSTDOUT_XF(f) { (f)->file = STDOUT_FILENO; (f)->filename = "/dev/stdout"; }
555main_file_init (main_file *xfile) 555#define XSTDERR_XF(f) { (f)->file = STDERR_FILENO; (f)->filename = "/dev/stderr"; }
556{ 556#define XSTDIN_XF(f) { (f)->file = STDIN_FILENO; (f)->filename = "/dev/stdin"; }
557 memset (xfile, 0, sizeof (*xfile)); 557#endif
558 558
559#if XD3_POSIX 559static void
560 xfile->file = -1; 560main_file_init (main_file *xfile)
561#endif 561{
562} 562 memset (xfile, 0, sizeof (*xfile));
563 563
564static void 564#if XD3_POSIX
565main_file_cleanup (main_file *xfile) 565 xfile->file = -1;
566{ 566#endif
567 if (xfile->filename_copy) { 567}
568 main_free(xfile->filename_copy); 568
569 } 569static void
570} 570main_file_cleanup (main_file *xfile)
571 571{
572static int 572 if (xfile->filename_copy) {
573main_file_isopen (main_file *xfile) 573 main_free(xfile->filename_copy);
574{ 574 }
575#if XD3_STDIO 575}
576 return xfile->file != NULL; 576
577 577static int
578#elif XD3_POSIX 578main_file_isopen (main_file *xfile)
579 return xfile->file != -1; 579{
580#endif 580#if XD3_STDIO
581} 581 return xfile->file != NULL;
582 582
583static int 583#elif XD3_POSIX
584main_file_close (main_file *xfile) 584 return xfile->file != -1;
585{ 585#endif
586 int ret = 0; 586}
587 587
588 if (! main_file_isopen (xfile)) 588static int
589 { 589main_file_close (main_file *xfile)
590 return 0; 590{
591 } 591 int ret = 0;
592 592
593#if XD3_STDIO 593 if (! main_file_isopen (xfile))
594 ret = fclose (xfile->file); 594 {
595 xfile->file = NULL; 595 return 0;
596 596 }
597#elif XD3_POSIX 597
598 ret = close (xfile->file); 598#if XD3_STDIO
599 xfile->file = -1; 599 ret = fclose (xfile->file);
600#endif 600 xfile->file = NULL;
601 601
602 if (ret != 0) { XF_ERROR ("close", xfile->filename, ret = get_errno ()); } 602#elif XD3_POSIX
603 return ret; 603 ret = close (xfile->file);
604} 604 xfile->file = -1;
605 605#endif
606static int 606
607main_file_open (main_file *xfile, const char* name, int mode) 607 if (ret != 0) { XF_ERROR ("close", xfile->filename, ret = get_errno ()); }
608{ 608 return ret;
609 int ret = 0; 609}
610 610
611 xfile->mode = mode; 611static int
612 612main_file_open (main_file *xfile, const char* name, int mode)
613 XD3_ASSERT (! main_file_isopen (xfile)); 613{
614 614 int ret = 0;
615#if XD3_STDIO 615
616 xfile->file = fopen (name, XOPEN_STDIO); 616 xfile->mode = mode;
617 617
618 ret = (xfile->file == NULL) ? get_errno () : 0; 618 XD3_ASSERT (! main_file_isopen (xfile));
619 619
620#elif XD3_POSIX 620#if XD3_STDIO
621 if ((ret = open (name, XOPEN_POSIX, XOPEN_MODE)) < 0) 621 xfile->file = fopen (name, XOPEN_STDIO);
622 { 622
623 ret = get_errno (); 623 ret = (xfile->file == NULL) ? get_errno () : 0;
624 } 624
625 else 625#elif XD3_POSIX
626 { 626 if ((ret = open (name, XOPEN_POSIX, XOPEN_MODE)) < 0)
627 xfile->file = ret; 627 {
628 ret = 0; 628 ret = get_errno ();
629 } 629 }
630#endif 630 else
631 if (ret) { XF_ERROR ("open", name, ret); } 631 {
632 else { xfile->realname = name; xfile->nread = 0; } 632 xfile->file = ret;
633 return ret; 633 ret = 0;
634} 634 }
635 635#endif
636static int 636 if (ret) { XF_ERROR ("open", name, ret); }
637main_file_stat (main_file *xfile, xoff_t *size, int err_ifnoseek) 637 else { xfile->realname = name; xfile->nread = 0; }
638{ 638 return ret;
639 int ret; 639}
640 struct stat sbuf; 640
641 641static int
642 XD3_ASSERT (main_file_isopen (xfile)); 642main_file_stat (main_file *xfile, xoff_t *size, int err_ifnoseek)
643 643{
644 if (fstat (XFNO (xfile), & sbuf) < 0) 644 int ret;
645 { 645 struct stat sbuf;
646 ret = get_errno (); 646
647 if (err_ifnoseek) { XF_ERROR ("stat", xfile->filename, ret); } 647 XD3_ASSERT (main_file_isopen (xfile));
648 return ret; 648
649 } 649 if (fstat (XFNO (xfile), & sbuf) < 0)
650 650 {
651 if (! S_ISREG (sbuf.st_mode)) 651 ret = get_errno ();
652 { 652 if (err_ifnoseek) { XF_ERROR ("stat", xfile->filename, ret); }
653 if (err_ifnoseek) { XPR(NT "source file must be seekable: %s\n", xfile->filename); } 653 return ret;
654 return ESPIPE; 654 }
655 } 655
656 656 if (! S_ISREG (sbuf.st_mode))
657 (*size) = sbuf.st_size; 657 {
658 return 0; 658 if (err_ifnoseek) { XPR(NT "source file must be seekable: %s\n", xfile->filename); }
659} 659 return ESPIPE;
660 660 }
661static int 661
662main_file_exists (main_file *xfile) 662 (*size) = sbuf.st_size;
663{ 663 return 0;
664 struct stat sbuf; 664}
665 return stat (xfile->filename, & sbuf) == 0 && S_ISREG (sbuf.st_mode); 665
666} 666static int
667 667main_file_exists (main_file *xfile)
668#if (XD3_POSIX || EXTERNAL_COMPRESSION) 668{
669/* POSIX-generic code takes a function pointer to read() or write(). This calls the 669 struct stat sbuf;
670 * function repeatedly until the buffer is full or EOF. The NREAD parameter is not 670 return stat (xfile->filename, & sbuf) == 0 && S_ISREG (sbuf.st_mode);
671 * set for write, NULL is passed. Return is signed, < 0 indicate errors, otherwise 671}
672 * byte count. */ 672
673typedef int (xd3_posix_func) (int fd, uint8_t *buf, usize_t size); 673#if (XD3_POSIX || EXTERNAL_COMPRESSION)
674 674/* POSIX-generic code takes a function pointer to read() or write(). This calls the
675static int 675 * function repeatedly until the buffer is full or EOF. The NREAD parameter is not
676xd3_posix_io (int fd, uint8_t *buf, usize_t size, xd3_posix_func *func, usize_t *nread) 676 * set for write, NULL is passed. Return is signed, < 0 indicate errors, otherwise
677{ 677 * byte count. */
678 int ret; 678typedef int (xd3_posix_func) (int fd, uint8_t *buf, usize_t size);
679 usize_t nproc = 0; 679
680 680static int
681 while (nproc < size) 681xd3_posix_io (int fd, uint8_t *buf, usize_t size, xd3_posix_func *func, usize_t *nread)
682 { 682{
683 int result = (*func) (fd, buf + nproc, size - nproc); 683 int ret;
684 684 usize_t nproc = 0;
685 if (result < 0) 685
686 { 686 while (nproc < size)
687 ret = get_errno (); 687 {
688 if (ret != EAGAIN && ret != EINTR) 688 int result = (*func) (fd, buf + nproc, size - nproc);
689 { 689
690 return ret; 690 if (result < 0)
691 } 691 {
692 result = 0; 692 ret = get_errno ();
693 } 693 if (ret != EAGAIN && ret != EINTR)
694 694 {
695 if (nread != NULL && result == 0) { break; } 695 return ret;
696 696 }
697 nproc += result; 697 result = 0;
698 } 698 }
699 if (nread != NULL) { (*nread) = nproc; } 699
700 return 0; 700 if (nread != NULL && result == 0) { break; }
701} 701
702#endif 702 nproc += result;
703 703 }
704/* POSIX is unbuffered, while STDIO is buffered. main_file_read() should always be called 704 if (nread != NULL) { (*nread) = nproc; }
705 * on blocks. */ 705 return 0;
706static int 706}
707main_file_read (main_file *ifile, 707#endif
708 uint8_t *buf, 708
709 usize_t size, 709/* POSIX is unbuffered, while STDIO is buffered. main_file_read() should always be called
710 usize_t *nread, 710 * on blocks. */
711 const char *msg) 711static int
712{ 712main_file_read (main_file *ifile,
713 int ret = 0; 713 uint8_t *buf,
714 714 usize_t size,
715#if XD3_STDIO 715 usize_t *nread,
716 usize_t result; 716 const char *msg)
717 717{
718 result = fread (buf, 1, size, ifile->file); 718 int ret = 0;
719 719
720 if (result < size && ferror (ifile->file)) 720#if XD3_STDIO
721 { 721 usize_t result;
722 ret = get_errno (); 722
723 } 723 result = fread (buf, 1, size, ifile->file);
724 else 724
725 { 725 if (result < size && ferror (ifile->file))
726 *nread = result; 726 {
727 } 727 ret = get_errno ();
728 728 }
729#elif XD3_POSIX 729 else
730 ret = xd3_posix_io (ifile->file, buf, size, (xd3_posix_func*) &read, nread); 730 {
731#endif 731 *nread = result;
732 732 }
733 if (ret) 733
734 { 734#elif XD3_POSIX
735 XPR(NT "%s: %s: %s\n", msg, ifile->filename, xd3_strerror (ret)); 735 ret = xd3_posix_io (ifile->file, buf, size, (xd3_posix_func*) &read, nread);
736 } 736#endif
737 else 737
738 { 738 if (ret)
739 if (option_verbose > 2) { XPR(NT "main read: %s: %u\n", ifile->filename, (*nread)); } 739 {
740 ifile->nread += (*nread); 740 XPR(NT "%s: %s: %s\n", msg, ifile->filename, xd3_strerror (ret));
741 } 741 }
742 742 else
743 return ret; 743 {
744} 744 if (option_verbose > 2) { XPR(NT "main read: %s: %u\n", ifile->filename, (*nread)); }
745 745 ifile->nread += (*nread);
746static int 746 }
747main_file_write (main_file *ofile, uint8_t *buf, usize_t size, const char *msg) 747
748{ 748 return ret;
749 int ret = 0; 749}
750 750
751#if XD3_STDIO 751static int
752 usize_t result; 752main_file_write (main_file *ofile, uint8_t *buf, usize_t size, const char *msg)
753 753{
754 result = fwrite (buf, 1, size, ofile->file); 754 int ret = 0;
755 755
756 if (result != size) { ret = get_errno (); } 756#if XD3_STDIO
757 757 usize_t result;
758#elif XD3_POSIX 758
759 ret = xd3_posix_io (ofile->file, buf, size, (xd3_posix_func*) &write, NULL); 759 result = fwrite (buf, 1, size, ofile->file);
760#endif 760
761 761 if (result != size) { ret = get_errno (); }
762 if (ret) 762
763 { 763#elif XD3_POSIX
764 XPR(NT "%s: %s: %s\n", msg, ofile->filename, xd3_strerror (ret)); 764 ret = xd3_posix_io (ofile->file, buf, size, (xd3_posix_func*) &write, NULL);
765 } 765#endif
766 else 766
767 { 767 if (ret)
768 if (option_verbose > 2) { XPR(NT "main write: %s: %u\n", ofile->filename, size); } 768 {
769 ofile->nwrite += size; 769 XPR(NT "%s: %s: %s\n", msg, ofile->filename, xd3_strerror (ret));
770 } 770 }
771 771 else
772 return ret; 772 {
773} 773 if (option_verbose > 2) { XPR(NT "main write: %s: %u\n", ofile->filename, size); }
774 774 ofile->nwrite += size;
775static int 775 }
776main_file_seek (main_file *xfile, xoff_t pos) 776
777{ 777 return ret;
778 int ret = 0; 778}
779 779
780#if XD3_STDIO 780static int
781 if (fseek (xfile->file, pos, SEEK_SET) != 0) { ret = get_errno (); } 781main_file_seek (main_file *xfile, xoff_t pos)
782#else 782{
783 if (lseek (xfile->file, pos, SEEK_SET) != pos) { ret = get_errno (); } 783 int ret = 0;
784#endif 784
785 785#if XD3_STDIO
786 if (ret) 786 if (fseek (xfile->file, pos, SEEK_SET) != 0) { ret = get_errno (); }
787 { 787#else
788 XPR(NT "seek failed: %s: %s\n", xfile->filename, xd3_strerror (ret)); 788 if (lseek (xfile->file, pos, SEEK_SET) != pos) { ret = get_errno (); }
789 } 789#endif
790 790
791 return ret; 791 if (ret)
792} 792 {
793 793 XPR(NT "seek failed: %s: %s\n", xfile->filename, xd3_strerror (ret));
794/****************************************************************************************** 794 }
795 VCDIFF TOOLS 795
796 ******************************************************************************************/ 796 return ret;
797 797}
798#if VCDIFF_TOOLS 798
799/* This function prints a single VCDIFF window, mainly for debugging purposes. */ 799/******************************************************************************************
800static int 800 VCDIFF TOOLS
801main_print_window (xd3_stream* stream, FILE *vcout) 801 ******************************************************************************************/
802{ 802
803 int ret; 803#if VCDIFF_TOOLS
804 usize_t size = 0; 804/* This function prints a single VCDIFF window, mainly for debugging purposes. */
805 805static int
806 VC(UT " Offset Code Type1 Size1 @Addr1 + Type2 Size2 @Addr2\n"); 806main_print_window (xd3_stream* stream, FILE *vcout)
807 807{
808 while (stream->inst_sect.buf < stream->inst_sect.buf_max) 808 int ret;
809 { 809 usize_t size = 0;
810 uint code = stream->inst_sect.buf[0]; 810
811 811 VC(UT " Offset Code Type1 Size1 @Addr1 + Type2 Size2 @Addr2\n");
812 if ((ret = xd3_decode_instruction (stream))) { return ret; } 812
813 813 while (stream->inst_sect.buf < stream->inst_sect.buf_max)
814 VC(UT " %06"Q"u %03u %s %3u", stream->dec_winstart + size, code, 814 {
815 xd3_rtype_to_string (stream->dec_current1.type, option_print_cpymode), 815 uint code = stream->inst_sect.buf[0];
816 stream->dec_current1.size); 816
817 817 if ((ret = xd3_decode_instruction (stream))) { return ret; }
818 if (stream->dec_current1.type != XD3_NOOP) 818
819 { 819 VC(UT " %06"Q"u %03u %s %3u", stream->dec_winstart + size, code,
820 size += stream->dec_current1.size; 820 xd3_rtype_to_string (stream->dec_current1.type, option_print_cpymode),
821 if (stream->dec_current1.type >= XD3_CPY) 821 stream->dec_current1.size);
822 { 822
823 VC(UT " @%-6u", stream->dec_current1.addr); 823 if (stream->dec_current1.type != XD3_NOOP)
824 } 824 {
825 else 825 size += stream->dec_current1.size;
826 { 826 if (stream->dec_current1.type >= XD3_CPY)
827 VC(UT " "); 827 {
828 } 828 VC(UT " @%-6u", stream->dec_current1.addr);
829 } 829 }
830 830 else
831 if (stream->dec_current2.type != XD3_NOOP) 831 {
832 { 832 VC(UT " ");
833 size += stream->dec_current2.size; 833 }
834 VC(UT " %s %3u", 834 }
835 xd3_rtype_to_string (stream->dec_current2.type, option_print_cpymode), 835
836 stream->dec_current2.size); 836 if (stream->dec_current2.type != XD3_NOOP)
837 837 {
838 if (stream->dec_current2.type >= XD3_CPY) 838 size += stream->dec_current2.size;
839 { 839 VC(UT " %s %3u",
840 VC(UT " @%-6u", stream->dec_current2.addr); 840 xd3_rtype_to_string (stream->dec_current2.type, option_print_cpymode),
841 } 841 stream->dec_current2.size);
842 } 842
843 843 if (stream->dec_current2.type >= XD3_CPY)
844 VC(UT "\n"); 844 {
845 } 845 VC(UT " @%-6u", stream->dec_current2.addr);
846 846 }
847 if (stream->dec_tgtlen != size && (stream->flags & XD3_SKIP_WINDOW) == 0) 847 }
848 { 848
849 XPR(NT "target window size inconsistency"); 849 VC(UT "\n");
850 return XD3_INTERNAL; 850 }
851 } 851
852 852 if (stream->dec_tgtlen != size && (stream->flags & XD3_SKIP_WINDOW) == 0)
853 if (stream->dec_position != stream->dec_maxpos) 853 {
854 { 854 XPR(NT "target window size inconsistency");
855 XPR(NT "target window position inconsistency"); 855 return XD3_INTERNAL;
856 return XD3_INTERNAL; 856 }
857 } 857
858 858 if (stream->dec_position != stream->dec_maxpos)
859 if (stream->addr_sect.buf != stream->addr_sect.buf_max) 859 {
860 { 860 XPR(NT "target window position inconsistency");
861 XPR(NT "address section inconsistency"); 861 return XD3_INTERNAL;
862 return XD3_INTERNAL; 862 }
863 } 863
864 864 if (stream->addr_sect.buf != stream->addr_sect.buf_max)
865 IF_DEBUG (VC(UT "SIZE=%u TGTLEN=%u\n", size, stream->dec_tgtlen)); 865 {
866 866 XPR(NT "address section inconsistency");
867 return 0; 867 return XD3_INTERNAL;
868} 868 }
869 869
870static void 870 IF_DEBUG (VC(UT "SIZE=%u TGTLEN=%u\n", size, stream->dec_tgtlen));
871main_print_vcdiff_file (main_file *file, const char *type, FILE *vcout) 871
872{ 872 return 0;
873 if (file->filename) { VC(UT "XDELTA filename (%s): %s\n", type, file->filename); } 873}
874 if (file->compressor) { VC(UT "XDELTA ext comp (%s): %s\n", type, file->compressor->recomp_cmdname); } 874
875} 875static void
876 876main_print_vcdiff_file (main_file *file, const char *type, FILE *vcout)
877/* This function prints a VCDIFF input, mainly for debugging purposes. */ 877{
878static int 878 if (file->filename) { VC(UT "XDELTA filename (%s): %s\n", type, file->filename); }
879main_print_func (xd3_stream* stream, main_file *xfile) 879 if (file->compressor) { VC(UT "XDELTA ext comp (%s): %s\n", type, file->compressor->recomp_cmdname); }
880{ 880}
881 int ret; 881
882 FILE *vcout; 882/* This function prints a VCDIFF input, mainly for debugging purposes. */
883#if XD3_POSIX 883static int
884 if (! (vcout = fdopen (dup(xfile->file), "w"))) 884main_print_func (xd3_stream* stream, main_file *xfile)
885 { 885{
886 ret = get_errno (); 886 int ret;
887 XPR(NT "fdopen: %s: %s\n", xfile->filename, xd3_strerror (ret)); 887 FILE *vcout;
888 return ret; 888#if XD3_POSIX
889 } 889 if (! (vcout = fdopen (dup(xfile->file), "w")))
890#elif XD3_STDIO 890 {
891 vcout = xfile->file; 891 ret = get_errno ();
892#endif 892 XPR(NT "fdopen: %s: %s\n", xfile->filename, xd3_strerror (ret));
893 XD3_ASSERT (vcout); 893 return ret;
894 if (stream->dec_winstart == 0) 894 }
895 { 895#elif XD3_STDIO
896 VC(UT "VCDIFF version: 0\n"); 896 vcout = xfile->file;
897 897#endif
898 VC(UT "VCDIFF header size: %d\n", stream->dec_hdrsize); 898 XD3_ASSERT (vcout);
899 VC(UT "VCDIFF header indicator: "); 899 if (stream->dec_winstart == 0)
900 if ((stream->dec_hdr_ind & VCD_SECONDARY) != 0) VC(UT "VCD_SECONDARY "); 900 {
901 if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) VC(UT "VCD_CODETABLE "); 901 VC(UT "VCDIFF version: 0\n");
902 if ((stream->dec_hdr_ind & VCD_APPHEADER) != 0) VC(UT "VCD_APPHEADER "); 902
903 if (stream->dec_hdr_ind == 0) VC(UT "none"); 903 VC(UT "VCDIFF header size: %d\n", stream->dec_hdrsize);
904 VC(UT "\n"); 904 VC(UT "VCDIFF header indicator: ");
905 905 if ((stream->dec_hdr_ind & VCD_SECONDARY) != 0) VC(UT "VCD_SECONDARY ");
906 IF_SEC(VC(UT "VCDIFF secondary compressor: %s\n", stream->sec_type ? stream->sec_type->name : "none")); 906 if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) VC(UT "VCD_CODETABLE ");
907 IF_NSEC(VC(UT "VCDIFF secondary compressor: unsupported\n")); 907 if ((stream->dec_hdr_ind & VCD_APPHEADER) != 0) VC(UT "VCD_APPHEADER ");
908 908 if (stream->dec_hdr_ind == 0) VC(UT "none");
909 if (stream->dec_hdr_ind & VCD_APPHEADER) 909 VC(UT "\n");
910 { 910
911 uint8_t *apphead; 911 IF_SEC(VC(UT "VCDIFF secondary compressor: %s\n", stream->sec_type ? stream->sec_type->name : "none"));
912 usize_t appheadsz; 912 IF_NSEC(VC(UT "VCDIFF secondary compressor: unsupported\n"));
913 ret = xd3_get_appheader (stream, & apphead, & appheadsz); 913
914 914 if (stream->dec_hdr_ind & VCD_APPHEADER)
915 if (ret == 0 && appheadsz > 0) 915 {
916 { 916 uint8_t *apphead;
917 int sq = option_quiet; 917 usize_t appheadsz;
918 main_file i, o, s; 918 ret = xd3_get_appheader (stream, & apphead, & appheadsz);
919 XD3_ASSERT (apphead != NULL); 919
920 VC(UT "VCDIFF application header: "); 920 if (ret == 0 && appheadsz > 0)
921 fwrite (apphead, 1, appheadsz, vcout); 921 {
922 VC(UT "\n"); 922 int sq = option_quiet;
923 923 main_file i, o, s;
924 main_file_init (& i); 924 XD3_ASSERT (apphead != NULL);
925 main_file_init (& o); 925 VC(UT "VCDIFF application header: ");
926 main_file_init (& s); 926 fwrite (apphead, 1, appheadsz, vcout);
927 option_quiet = 1; 927 VC(UT "\n");
928 main_get_appheader (stream, &i, & o, & s); 928
929 option_quiet = sq; 929 main_file_init (& i);
930 main_print_vcdiff_file (& o, "output", vcout); 930 main_file_init (& o);
931 main_print_vcdiff_file (& s, "source", vcout); 931 main_file_init (& s);
932 main_file_cleanup (& i); 932 option_quiet = 1;
933 main_file_cleanup (& o); 933 main_get_appheader (stream, &i, & o, & s);
934 main_file_cleanup (& s); 934 option_quiet = sq;
935 } 935 main_print_vcdiff_file (& o, "output", vcout);
936 } 936 main_print_vcdiff_file (& s, "source", vcout);
937 } 937 main_file_cleanup (& i);
938 else 938 main_file_cleanup (& o);
939 { 939 main_file_cleanup (& s);
940 VC(UT "\n"); 940 }
941 } 941 }
942 942 }
943 VC(UT "VCDIFF window number: %"Q"u\n", stream->current_window); 943 else
944 VC(UT "VCDIFF window indicator: "); 944 {
945 if ((stream->dec_win_ind & VCD_SOURCE) != 0) VC(UT "VCD_SOURCE "); 945 VC(UT "\n");
946 if ((stream->dec_win_ind & VCD_TARGET) != 0) VC(UT "VCD_TARGET "); 946 }
947 if ((stream->dec_win_ind & VCD_ADLER32) != 0) VC(UT "VCD_ADLER32 "); 947
948 if (stream->dec_win_ind == 0) VC(UT "none"); 948 VC(UT "VCDIFF window number: %"Q"u\n", stream->current_window);
949 VC(UT "\n"); 949 VC(UT "VCDIFF window indicator: ");
950 950 if ((stream->dec_win_ind & VCD_SOURCE) != 0) VC(UT "VCD_SOURCE ");
951 if ((stream->dec_win_ind & VCD_ADLER32) != 0) 951 if ((stream->dec_win_ind & VCD_TARGET) != 0) VC(UT "VCD_TARGET ");
952 { 952 if ((stream->dec_win_ind & VCD_ADLER32) != 0) VC(UT "VCD_ADLER32 ");
953 VC(UT "VCDIFF adler32 checksum: %08X\n", stream->dec_adler32); 953 if (stream->dec_win_ind == 0) VC(UT "none");
954 } 954 VC(UT "\n");
955 955
956 if (stream->dec_del_ind != 0) 956 if ((stream->dec_win_ind & VCD_ADLER32) != 0)
957 { 957 {
958 VC(UT "VCDIFF delta indicator: "); 958 VC(UT "VCDIFF adler32 checksum: %08X\n", stream->dec_adler32);
959 if ((stream->dec_del_ind & VCD_DATACOMP) != 0) VC(UT "VCD_DATACOMP "); 959 }
960 if ((stream->dec_del_ind & VCD_INSTCOMP) != 0) VC(UT "VCD_INSTCOMP "); 960
961 if ((stream->dec_del_ind & VCD_ADDRCOMP) != 0) VC(UT "VCD_ADDRCOMP "); 961 if (stream->dec_del_ind != 0)
962 if (stream->dec_del_ind == 0) VC(UT "none"); 962 {
963 VC(UT "\n"); 963 VC(UT "VCDIFF delta indicator: ");
964 } 964 if ((stream->dec_del_ind & VCD_DATACOMP) != 0) VC(UT "VCD_DATACOMP ");
965 965 if ((stream->dec_del_ind & VCD_INSTCOMP) != 0) VC(UT "VCD_INSTCOMP ");
966 if (stream->dec_winstart != 0) 966 if ((stream->dec_del_ind & VCD_ADDRCOMP) != 0) VC(UT "VCD_ADDRCOMP ");
967 { 967 if (stream->dec_del_ind == 0) VC(UT "none");
968 VC(UT "VCDIFF window at offset: %"Q"u\n", stream->dec_winstart); 968 VC(UT "\n");
969 } 969 }
970 970
971 if (SRCORTGT (stream->dec_win_ind)) 971 if (stream->dec_winstart != 0)
972 { 972 {
973 VC(UT "VCDIFF copy window length: %u\n", stream->dec_cpylen); 973 VC(UT "VCDIFF window at offset: %"Q"u\n", stream->dec_winstart);
974 VC(UT "VCDIFF copy window offset: %"Q"u\n", stream->dec_cpyoff); 974 }
975 } 975
976 976 if (SRCORTGT (stream->dec_win_ind))
977 VC(UT "VCDIFF delta encoding length: %u\n", stream->dec_enclen); 977 {
978 VC(UT "VCDIFF target window length: %u\n", stream->dec_tgtlen); 978 VC(UT "VCDIFF copy window length: %u\n", stream->dec_cpylen);
979 979 VC(UT "VCDIFF copy window offset: %"Q"u\n", stream->dec_cpyoff);
980 VC(UT "VCDIFF data section length: %u\n", stream->data_sect.size); 980 }
981 VC(UT "VCDIFF inst section length: %u\n", stream->inst_sect.size); 981
982 VC(UT "VCDIFF addr section length: %u\n", stream->addr_sect.size); 982 VC(UT "VCDIFF delta encoding length: %u\n", stream->dec_enclen);
983 983 VC(UT "VCDIFF target window length: %u\n", stream->dec_tgtlen);
984 ret = 0; 984
985 if ((stream->flags & XD3_JUST_HDR) != 0) 985 VC(UT "VCDIFF data section length: %u\n", stream->data_sect.size);
986 { 986 VC(UT "VCDIFF inst section length: %u\n", stream->inst_sect.size);
987 /* Print a header -- finished! */ 987 VC(UT "VCDIFF addr section length: %u\n", stream->addr_sect.size);
988 ret = PRINTHDR_SPECIAL; 988
989 } 989 ret = 0;
990 else if ((stream->flags & XD3_SKIP_WINDOW) == 0) 990 if ((stream->flags & XD3_JUST_HDR) != 0)
991 { 991 {
992 ret = main_print_window (stream, vcout); 992 /* Print a header -- finished! */
993 } 993 ret = PRINTHDR_SPECIAL;
994 994 }
995 fclose (vcout); 995 else if ((stream->flags & XD3_SKIP_WINDOW) == 0)
996 return ret; 996 {
997} 997 ret = main_print_window (stream, vcout);
998#endif /* VCDIFF_TOOLS */ 998 }
999 999
1000/****************************************************************************************** 1000 fclose (vcout);
1001 Input decompression, output recompression 1001 return ret;
1002 ******************************************************************************************/ 1002}
1003 1003#endif /* VCDIFF_TOOLS */
1004#if EXTERNAL_COMPRESSION 1004
1005/* This is tricky POSIX-specific code with lots of fork(), pipe(), dup(), waitpid(), and 1005/******************************************************************************************
1006 * exec() business. Most of this code originated in PRCS1, which did automatic 1006 Input decompression, output recompression
1007 * package-file decompression. It works with both XD3_POSIX and XD3_STDIO file 1007 ******************************************************************************************/
1008 * disciplines. 1008
1009 * 1009#if EXTERNAL_COMPRESSION
1010 * To automatically detect compressed inputs requires a child process to reconstruct the 1010/* This is tricky POSIX-specific code with lots of fork(), pipe(), dup(), waitpid(), and
1011 * input stream, which was advanced in order to detect compression, because it may not be 1011 * exec() business. Most of this code originated in PRCS1, which did automatic
1012 * seekable. In other words, the main program reads part of the input stream, and if it 1012 * package-file decompression. It works with both XD3_POSIX and XD3_STDIO file
1013 * detects a compressed input it then forks a pipe copier process, which copies the 1013 * disciplines.
1014 * first-read block out of the main-program's memory, then streams the remaining 1014 *
1015 * compressed input into the input-decompression pipe. 1015 * To automatically detect compressed inputs requires a child process to reconstruct the
1016 */ 1016 * input stream, which was advanced in order to detect compression, because it may not be
1017 1017 * seekable. In other words, the main program reads part of the input stream, and if it
1018#include <unistd.h> 1018 * detects a compressed input it then forks a pipe copier process, which copies the
1019#include <sys/stat.h> 1019 * first-read block out of the main-program's memory, then streams the remaining
1020#include <sys/wait.h> 1020 * compressed input into the input-decompression pipe.
1021 1021 */
1022/* Remember which pipe FD is which. */ 1022
1023#define PIPE_READ_FD 0 1023#include <unistd.h>
1024#define PIPE_WRITE_FD 1 1024#include <sys/stat.h>
1025 1025#include <sys/wait.h>
1026static pid_t ext_subprocs[2]; 1026
1027static char* ext_tmpfile = NULL; 1027/* Remember which pipe FD is which. */
1028 1028#define PIPE_READ_FD 0
1029/* Like write(), but makes repeated calls to empty the buffer. */ 1029#define PIPE_WRITE_FD 1
1030static int 1030
1031main_pipe_write (int outfd, const uint8_t *exist_buf, usize_t remain) 1031static pid_t ext_subprocs[2];
1032{ 1032static char* ext_tmpfile = NULL;
1033 int ret; 1033
1034 1034/* Like write(), but makes repeated calls to empty the buffer. */
1035 if ((ret = xd3_posix_io (outfd, (uint8_t*) exist_buf, remain, (xd3_posix_func*) &write, NULL))) 1035static int
1036 { 1036main_pipe_write (int outfd, const uint8_t *exist_buf, usize_t remain)
1037 XPR(NT "pipe write failed: %s", xd3_strerror (ret)); 1037{
1038 return ret; 1038 int ret;
1039 } 1039
1040 1040 if ((ret = xd3_posix_io (outfd, (uint8_t*) exist_buf, remain, (xd3_posix_func*) &write, NULL)))
1041 return 0; 1041 {
1042} 1042 XPR(NT "pipe write failed: %s", xd3_strerror (ret));
1043 1043 return ret;
1044/* A simple error-reporting waitpid interface. */ 1044 }
1045static int 1045
1046main_waitpid_check(pid_t pid) 1046 return 0;
1047{ 1047}
1048 int status; 1048
1049 int ret = 0; 1049/* A simple error-reporting waitpid interface. */
1050 1050static int
1051 if (waitpid (pid, & status, 0) < 0) 1051main_waitpid_check(pid_t pid)
1052 { 1052{
1053 ret = get_errno (); 1053 int status;
1054 XPR(NT "compression subprocess: wait: %s\n", xd3_strerror (ret)); 1054 int ret = 0;
1055 } 1055
1056 else if (! WIFEXITED (status)) 1056 if (waitpid (pid, & status, 0) < 0)
1057 { 1057 {
1058 ret = ECHILD; 1058 ret = get_errno ();
1059 XPR(NT "compression subprocess: signal %d\n", 1059 XPR(NT "compression subprocess: wait: %s\n", xd3_strerror (ret));
1060 WIFSIGNALED (status) ? WTERMSIG (status) : WSTOPSIG (status)); 1060 }
1061 } 1061 else if (! WIFEXITED (status))
1062 else if (WEXITSTATUS (status) != 0) 1062 {
1063 { 1063 ret = ECHILD;
1064 ret = ECHILD; 1064 XPR(NT "compression subprocess: signal %d\n",
1065 XPR(NT "compression subprocess: exit %d\n", WEXITSTATUS (status)); 1065 WIFSIGNALED (status) ? WTERMSIG (status) : WSTOPSIG (status));
1066 } 1066 }
1067 1067 else if (WEXITSTATUS (status) != 0)
1068 return ret; 1068 {
1069} 1069 ret = ECHILD;
1070 1070 XPR(NT "compression subprocess: exit %d\n", WEXITSTATUS (status));
1071/* Wait for any existing child processes to check for abnormal exit. */ 1071 }
1072static int 1072
1073main_external_compression_finish (void) 1073 return ret;
1074{ 1074}
1075 int i; 1075
1076 int ret; 1076/* Wait for any existing child processes to check for abnormal exit. */
1077 1077static int
1078 for (i = 0; i < 2; i += 1) 1078main_external_compression_finish (void)
1079 { 1079{
1080 if (! ext_subprocs[i]) { continue; } 1080 int i;
1081 1081 int ret;
1082 if ((ret = main_waitpid_check (ext_subprocs[i]))) 1082
1083 { 1083 for (i = 0; i < 2; i += 1)
1084 return ret; 1084 {
1085 } 1085 if (! ext_subprocs[i]) { continue; }
1086 } 1086
1087 1087 if ((ret = main_waitpid_check (ext_subprocs[i])))
1088 return 0; 1088 {
1089} 1089 return ret;
1090 1090 }
1091/* This runs as a forked process of main_input_decompress_setup() to copy input to the 1091 }
1092 * decompression process. First, the available input is copied out of the existing 1092
1093 * buffer, then the buffer is reused to continue reading from the compressed input 1093 return 0;
1094 * file. */ 1094}
1095static int 1095
1096main_pipe_copier (uint8_t *pipe_buf, 1096/* This runs as a forked process of main_input_decompress_setup() to copy input to the
1097 usize_t pipe_bufsize, 1097 * decompression process. First, the available input is copied out of the existing
1098 usize_t nread, 1098 * buffer, then the buffer is reused to continue reading from the compressed input
1099 main_file *ifile, 1099 * file. */
1100 int outfd) 1100static int
1101{ 1101main_pipe_copier (uint8_t *pipe_buf,
1102 int ret; 1102 usize_t pipe_bufsize,
1103 1103 usize_t nread,
1104 for (;;) 1104 main_file *ifile,
1105 { 1105 int outfd)
1106 if (nread > 0 && (ret = main_pipe_write (outfd, pipe_buf, nread))) 1106{
1107 { 1107 int ret;
1108 return ret; 1108
1109 } 1109 for (;;)
1110 1110 {
1111 if (nread < pipe_bufsize) 1111 if (nread > 0 && (ret = main_pipe_write (outfd, pipe_buf, nread)))
1112 { 1112 {
1113 break; 1113 return ret;
1114 } 1114 }
1115 1115
1116 if ((ret = main_file_read (ifile, pipe_buf, pipe_bufsize, & nread, "pipe read failed")) < 0) 1116 if (nread < pipe_bufsize)
1117 { 1117 {
1118 return ret; 1118 break;
1119 } 1119 }
1120 } 1120
1121 1121 if ((ret = main_file_read (ifile, pipe_buf, pipe_bufsize, & nread, "pipe read failed")) < 0)
1122 return 0; 1122 {
1123} 1123 return ret;
1124 1124 }
1125/* This function is called after we have read some amount of data from the input file and 1125 }
1126 * detected a compressed input. Here we start a decompression subprocess by forking 1126
1127 * twice. The first process runs the decompression command, the second process copies 1127 return 0;
1128 * data to the input of the first. */ 1128}
1129static int 1129
1130main_input_decompress_setup (const main_extcomp *decomp, 1130/* This function is called after we have read some amount of data from the input file and
1131 main_file *ifile, 1131 * detected a compressed input. Here we start a decompression subprocess by forking
1132 uint8_t *input_buf, 1132 * twice. The first process runs the decompression command, the second process copies
1133 usize_t input_bufsize, 1133 * data to the input of the first. */
1134 uint8_t *pipe_buf, 1134static int
1135 usize_t pipe_bufsize, 1135main_input_decompress_setup (const main_extcomp *decomp,
1136 usize_t pipe_avail, 1136 main_file *ifile,
1137 usize_t *nread) 1137 uint8_t *input_buf,
1138{ 1138 usize_t input_bufsize,
1139 int outpipefd[2], inpipefd[2]; /* The two pipes: input and output file descriptors. */ 1139 uint8_t *pipe_buf,
1140 int input_fd = -1; /* The resulting input_fd (output of decompression). */ 1140 usize_t pipe_bufsize,
1141 pid_t decomp_id, copier_id; /* The two subprocs. */ 1141 usize_t pipe_avail,
1142 int ret; 1142 usize_t *nread)
1143 1143{
1144 outpipefd[0] = outpipefd[1] = -1; 1144 int outpipefd[2], inpipefd[2]; /* The two pipes: input and output file descriptors. */
1145 inpipefd[0] = inpipefd[1] = -1; 1145 int input_fd = -1; /* The resulting input_fd (output of decompression). */
1146 1146 pid_t decomp_id, copier_id; /* The two subprocs. */
1147 if (pipe (outpipefd) || pipe (inpipefd)) 1147 int ret;
1148 { 1148
1149 XPR(NT "pipe failed: %s\n", xd3_strerror (ret = get_errno ())); 1149 outpipefd[0] = outpipefd[1] = -1;
1150 goto pipe_cleanup; 1150 inpipefd[0] = inpipefd[1] = -1;
1151 } 1151
1152 1152 if (pipe (outpipefd) || pipe (inpipefd))
1153 if ((decomp_id = fork ()) < 0) 1153 {
1154 { 1154 XPR(NT "pipe failed: %s\n", xd3_strerror (ret = get_errno ()));
1155 XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ())); 1155 goto pipe_cleanup;
1156 goto pipe_cleanup; 1156 }
1157 } 1157
1158 1158 if ((decomp_id = fork ()) < 0)
1159 /* The first child runs the decompression process: */ 1159 {
1160 if (decomp_id == 0) 1160 XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ()));
1161 { 1161 goto pipe_cleanup;
1162 /* Setup pipes: write to the outpipe, read from the inpipe. */ 1162 }
1163 if (dup2 (outpipefd[PIPE_WRITE_FD], STDOUT_FILENO) < 0 || 1163
1164 dup2 (inpipefd[PIPE_READ_FD], STDIN_FILENO) < 0 || 1164 /* The first child runs the decompression process: */
1165 close (outpipefd[PIPE_READ_FD]) || 1165 if (decomp_id == 0)
1166 close (outpipefd[PIPE_WRITE_FD]) || 1166 {
1167 close (inpipefd[PIPE_READ_FD]) || 1167 /* Setup pipes: write to the outpipe, read from the inpipe. */
1168 close (inpipefd[PIPE_WRITE_FD]) || 1168 if (dup2 (outpipefd[PIPE_WRITE_FD], STDOUT_FILENO) < 0 ||
1169 execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, decomp->decomp_options, NULL)) 1169 dup2 (inpipefd[PIPE_READ_FD], STDIN_FILENO) < 0 ||
1170 { 1170 close (outpipefd[PIPE_READ_FD]) ||
1171 XPR(NT "child process %s failed to execute: %s\n", decomp->decomp_cmdname, xd3_strerror (get_errno ())); 1171 close (outpipefd[PIPE_WRITE_FD]) ||
1172 } 1172 close (inpipefd[PIPE_READ_FD]) ||
1173 1173 close (inpipefd[PIPE_WRITE_FD]) ||
1174 _exit (127); 1174 execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, decomp->decomp_options, NULL))
1175 } 1175 {
1176 1176 XPR(NT "child process %s failed to execute: %s\n", decomp->decomp_cmdname, xd3_strerror (get_errno ()));
1177 ext_subprocs[0] = decomp_id; 1177 }
1178 1178
1179 if ((copier_id = fork ()) < 0) 1179 _exit (127);
1180 { 1180 }
1181 XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ())); 1181
1182 goto pipe_cleanup; 1182 ext_subprocs[0] = decomp_id;
1183 } 1183
1184 1184 if ((copier_id = fork ()) < 0)
1185 /* The second child runs the copier process: */ 1185 {
1186 if (copier_id == 0) 1186 XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ()));
1187 { 1187 goto pipe_cleanup;
1188 int exitval = 0; 1188 }
1189 1189
1190 if (close (inpipefd[PIPE_READ_FD]) || 1190 /* The second child runs the copier process: */
1191 main_pipe_copier (pipe_buf, pipe_bufsize, pipe_avail, ifile, inpipefd[PIPE_WRITE_FD]) || 1191 if (copier_id == 0)
1192 close (inpipefd[PIPE_WRITE_FD])) 1192 {
1193 { 1193 int exitval = 0;
1194 XPR(NT "child copier process failed: %s\n", xd3_strerror (get_errno ())); 1194
1195 exitval = 1; 1195 if (close (inpipefd[PIPE_READ_FD]) ||
1196 } 1196 main_pipe_copier (pipe_buf, pipe_bufsize, pipe_avail, ifile, inpipefd[PIPE_WRITE_FD]) ||
1197 1197 close (inpipefd[PIPE_WRITE_FD]))
1198 _exit (exitval); 1198 {
1199 } 1199 XPR(NT "child copier process failed: %s\n", xd3_strerror (get_errno ()));
1200 1200 exitval = 1;
1201 ext_subprocs[1] = copier_id; 1201 }
1202 1202
1203 /* The parent closes both pipes after duplicating the output of compression. */ 1203 _exit (exitval);
1204 input_fd = dup (outpipefd[PIPE_READ_FD]); 1204 }
1205 1205
1206 if (input_fd < 0 || 1206 ext_subprocs[1] = copier_id;
1207 main_file_close (ifile) || 1207
1208 close (outpipefd[PIPE_READ_FD]) || 1208 /* The parent closes both pipes after duplicating the output of compression. */
1209 close (outpipefd[PIPE_WRITE_FD]) || 1209 input_fd = dup (outpipefd[PIPE_READ_FD]);
1210 close (inpipefd[PIPE_READ_FD]) || 1210
1211 close (inpipefd[PIPE_WRITE_FD])) 1211 if (input_fd < 0 ||
1212 { 1212 main_file_close (ifile) ||
1213 XPR(NT "dup/close failed: %s\n", xd3_strerror (ret = get_errno ())); 1213 close (outpipefd[PIPE_READ_FD]) ||
1214 goto pipe_cleanup; 1214 close (outpipefd[PIPE_WRITE_FD]) ||
1215 } 1215 close (inpipefd[PIPE_READ_FD]) ||
1216 1216 close (inpipefd[PIPE_WRITE_FD]))
1217#if XD3_STDIO 1217 {
1218 /* Note: fdopen() acquires the fd, closes it when finished. */ 1218 XPR(NT "dup/close failed: %s\n", xd3_strerror (ret = get_errno ()));
1219 if ((ifile->file = fdopen (input_fd, "r")) == NULL) 1219 goto pipe_cleanup;
1220 { 1220 }
1221 XPR(NT "fdopen failed: %s\n", xd3_strerror (ret = get_errno ())); 1221
1222 goto pipe_cleanup; 1222#if XD3_STDIO
1223 } 1223 /* Note: fdopen() acquires the fd, closes it when finished. */
1224 1224 if ((ifile->file = fdopen (input_fd, "r")) == NULL)
1225#elif XD3_POSIX 1225 {
1226 ifile->file = input_fd; 1226 XPR(NT "fdopen failed: %s\n", xd3_strerror (ret = get_errno ()));
1227#endif 1227 goto pipe_cleanup;
1228 1228 }
1229 ifile->compressor = decomp; 1229
1230 1230#elif XD3_POSIX
1231 /* Now the input file is decompressed. */ 1231 ifile->file = input_fd;
1232 return main_file_read (ifile, input_buf, input_bufsize, nread, "input decompression failed"); 1232#endif
1233 1233
1234 pipe_cleanup: 1234 ifile->compressor = decomp;
1235 close (input_fd); 1235
1236 close (outpipefd[PIPE_READ_FD]); 1236 /* Now the input file is decompressed. */
1237 close (outpipefd[PIPE_WRITE_FD]); 1237 return main_file_read (ifile, input_buf, input_bufsize, nread, "input decompression failed");
1238 close (inpipefd[PIPE_READ_FD]); 1238
1239 close (inpipefd[PIPE_WRITE_FD]); 1239 pipe_cleanup:
1240 return ret; 1240 close (input_fd);
1241} 1241 close (outpipefd[PIPE_READ_FD]);
1242 1242 close (outpipefd[PIPE_WRITE_FD]);
1243 1243 close (inpipefd[PIPE_READ_FD]);
1244/* This routine is called when the first buffer of input data is read by the main program 1244 close (inpipefd[PIPE_WRITE_FD]);
1245 * (unless input decompression is disabled by command-line option). If it recognizes the 1245 return ret;
1246 * magic number of a known input type it invokes decompression. 1246}
1247 * 1247
1248 * Skips decompression if the decompression type or the file type is RD_NONEXTERNAL. 1248
1249 * 1249/* This routine is called when the first buffer of input data is read by the main program
1250 * Behaves exactly like main_file_read, otherwise. 1250 * (unless input decompression is disabled by command-line option). If it recognizes the
1251 * 1251 * magic number of a known input type it invokes decompression.
1252 * This function uses a separate buffer to read the first small block of input. If a 1252 *
1253 * compressed input is detected, the separate buffer is passed to the pipe copier. This 1253 * Skips decompression if the decompression type or the file type is RD_NONEXTERNAL.
1254 * avoids using the same size buffer in both cases. */ 1254 *
1255static int 1255 * Behaves exactly like main_file_read, otherwise.
1256main_decompress_input_check (main_file *ifile, 1256 *
1257 uint8_t *input_buf, 1257 * This function uses a separate buffer to read the first small block of input. If a
1258 usize_t input_size, 1258 * compressed input is detected, the separate buffer is passed to the pipe copier. This
1259 usize_t *nread) 1259 * avoids using the same size buffer in both cases. */
1260{ 1260static int
1261 int i; 1261main_decompress_input_check (main_file *ifile,
1262 int ret; 1262 uint8_t *input_buf,
1263 uint8_t check_buf[XD3_ALLOCSIZE]; 1263 usize_t input_size,
1264 usize_t check_nread; 1264 usize_t *nread)
1265 1265{
1266 if ((ret = main_file_read (ifile, check_buf, min (input_size, XD3_ALLOCSIZE), & check_nread, "input read failed"))) 1266 int i;
1267 { 1267 int ret;
1268 return ret; 1268 uint8_t check_buf[XD3_ALLOCSIZE];
1269 } 1269 usize_t check_nread;
1270 1270
1271 for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) 1271 if ((ret = main_file_read (ifile, check_buf, min (input_size, XD3_ALLOCSIZE), & check_nread, "input read failed")))
1272 { 1272 {
1273 const main_extcomp *decomp = & extcomp_types[i]; 1273 return ret;
1274 1274 }
1275 if ((check_nread > decomp->magic_size) && 1275
1276 /* The following expr skips decompression if we are trying to read a VCDIFF 1276 for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)
1277 * input and that is the magic number. */ 1277 {
1278 !((decomp->flags & RD_NONEXTERNAL) && (ifile->flags & RD_NONEXTERNAL)) && 1278 const main_extcomp *decomp = & extcomp_types[i];
1279 memcmp (check_buf, decomp->magic, decomp->magic_size) == 0) 1279
1280 { 1280 if ((check_nread > decomp->magic_size) &&
1281 if (! option_quiet) 1281 /* The following expr skips decompression if we are trying to read a VCDIFF
1282 { 1282 * input and that is the magic number. */
1283 XPR(NT "%s | %s %s\n", 1283 !((decomp->flags & RD_NONEXTERNAL) && (ifile->flags & RD_NONEXTERNAL)) &&
1284 ifile->filename, 1284 memcmp (check_buf, decomp->magic, decomp->magic_size) == 0)
1285 decomp->decomp_cmdname, 1285 {
1286 decomp->decomp_options); 1286 if (! option_quiet)
1287 } 1287 {
1288 1288 XPR(NT "%s | %s %s\n",
1289 return main_input_decompress_setup (decomp, ifile, 1289 ifile->filename,
1290 input_buf, input_size, 1290 decomp->decomp_cmdname,
1291 check_buf, XD3_ALLOCSIZE, 1291 decomp->decomp_options);
1292 check_nread, nread); 1292 }
1293 } 1293
1294 } 1294 return main_input_decompress_setup (decomp, ifile,
1295 1295 input_buf, input_size,
1296 /* Now read the rest of the input block. */ 1296 check_buf, XD3_ALLOCSIZE,
1297 (*nread) = 0; 1297 check_nread, nread);
1298 1298 }
1299 if (check_nread == XD3_ALLOCSIZE) 1299 }
1300 { 1300
1301 ret = main_file_read (ifile, input_buf + XD3_ALLOCSIZE, 1301 /* Now read the rest of the input block. */
1302 input_size - XD3_ALLOCSIZE, nread, 1302 (*nread) = 0;
1303 "input read failed"); 1303
1304 } 1304 if (check_nread == XD3_ALLOCSIZE)
1305 1305 {
1306 memcpy (input_buf, check_buf, check_nread); 1306 ret = main_file_read (ifile, input_buf + XD3_ALLOCSIZE,
1307 1307 input_size - XD3_ALLOCSIZE, nread,
1308 (*nread) += check_nread; 1308 "input read failed");
1309 1309 }
1310 return 0; 1310
1311} 1311 memcpy (input_buf, check_buf, check_nread);
1312 1312
1313/* This is called when the source file needs to be decompressed. We fork/exec a 1313 (*nread) += check_nread;
1314 * decompression command with the proper input and output to a temporary file. */ 1314
1315static int 1315 return 0;
1316main_decompress_source (main_file *sfile, xd3_source *source) 1316}
1317{ 1317
1318 const main_extcomp *decomp = sfile->compressor; 1318/* This is called when the source file needs to be decompressed. We fork/exec a
1319 pid_t decomp_id; /* One subproc. */ 1319 * decompression command with the proper input and output to a temporary file. */
1320 int input_fd = -1; 1320static int
1321 int output_fd = -1; 1321main_decompress_source (main_file *sfile, xd3_source *source)
1322 int ret; 1322{
1323 char *tmpname = NULL; 1323 const main_extcomp *decomp = sfile->compressor;
1324 char *tmpdir = getenv ("TMPDIR"); 1324 pid_t decomp_id; /* One subproc. */
1325 static const char tmpl[] = "/xd3src.XXXXXX"; 1325 int input_fd = -1;
1326 1326 int output_fd = -1;
1327 /* Make a template for mkstmp() */ 1327 int ret;
1328 if (tmpdir == NULL) { tmpdir = "/tmp"; } 1328 char *tmpname = NULL;
1329 if ((tmpname = main_malloc (strlen (tmpdir) + sizeof (tmpl) + 1)) == NULL) { return ENOMEM; } 1329 char *tmpdir = getenv ("TMPDIR");
1330 sprintf (tmpname, "%s%s", tmpdir, tmpl); 1330 static const char tmpl[] = "/xd3src.XXXXXX";
1331 1331
1332 XD3_ASSERT (ext_tmpfile == NULL); 1332 /* Make a template for mkstmp() */
1333 ext_tmpfile = tmpname; 1333 if (tmpdir == NULL) { tmpdir = "/tmp"; }
1334 1334 if ((tmpname = main_malloc (strlen (tmpdir) + sizeof (tmpl) + 1)) == NULL) { return ENOMEM; }
1335 /* Open the output FD. */ 1335 sprintf (tmpname, "%s%s", tmpdir, tmpl);
1336 if ((output_fd = mkstemp (tmpname)) < 0) 1336
1337 { 1337 XD3_ASSERT (ext_tmpfile == NULL);
1338 XPR(NT "mkstemp failed: %s: %s", tmpname, xd3_strerror (ret = get_errno ())); 1338 ext_tmpfile = tmpname;
1339 goto cleanup; 1339
1340 } 1340 /* Open the output FD. */
1341 1341 if ((output_fd = mkstemp (tmpname)) < 0)
1342 /* Copy the input FD, reset file position. */ 1342 {
1343 XD3_ASSERT (main_file_isopen (sfile)); 1343 XPR(NT "mkstemp failed: %s: %s", tmpname, xd3_strerror (ret = get_errno ()));
1344#if XD3_STDIO 1344 goto cleanup;
1345 if ((input_fd = dup (fileno (sfile->file))) < 0) 1345 }
1346 { 1346
1347 XPR(NT "dup failed: %s", xd3_strerror (ret = get_errno ())); 1347 /* Copy the input FD, reset file position. */
1348 goto cleanup; 1348 XD3_ASSERT (main_file_isopen (sfile));
1349 } 1349#if XD3_STDIO
1350 main_file_close (sfile); 1350 if ((input_fd = dup (fileno (sfile->file))) < 0)
1351 sfile->file = NULL; 1351 {
1352#elif XD3_POSIX 1352 XPR(NT "dup failed: %s", xd3_strerror (ret = get_errno ()));
1353 input_fd = sfile->file; 1353 goto cleanup;
1354 sfile->file = -1; 1354 }
1355#endif 1355 main_file_close (sfile);
1356 1356 sfile->file = NULL;
1357 if ((ret = lseek (input_fd, SEEK_SET, 0)) != 0) 1357#elif XD3_POSIX
1358 { 1358 input_fd = sfile->file;
1359 XPR(NT "lseek failed: : %s", xd3_strerror (ret = get_errno ())); 1359 sfile->file = -1;
1360 goto cleanup; 1360#endif
1361 } 1361
1362 1362 if ((ret = lseek (input_fd, SEEK_SET, 0)) != 0)
1363 if ((decomp_id = fork ()) < 0) 1363 {
1364 { 1364 XPR(NT "lseek failed: : %s", xd3_strerror (ret = get_errno ()));
1365 XPR(NT "fork failed: %s", xd3_strerror (ret = get_errno ())); 1365 goto cleanup;
1366 goto cleanup; 1366 }
1367 } 1367
1368 1368 if ((decomp_id = fork ()) < 0)
1369 /* The child runs the decompression process: */ 1369 {
1370 if (decomp_id == 0) 1370 XPR(NT "fork failed: %s", xd3_strerror (ret = get_errno ()));
1371 { 1371 goto cleanup;
1372 /* Setup pipes: write to the output file, read from the pipe. */ 1372 }
1373 if (dup2 (input_fd, STDIN_FILENO) < 0 || 1373
1374 dup2 (output_fd, STDOUT_FILENO) < 0 || 1374 /* The child runs the decompression process: */
1375 execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, decomp->decomp_options, NULL)) 1375 if (decomp_id == 0)
1376 { 1376 {
1377 XPR(NT "child process %s failed to execute: %s\n", 1377 /* Setup pipes: write to the output file, read from the pipe. */
1378 decomp->decomp_cmdname, xd3_strerror (get_errno ())); 1378 if (dup2 (input_fd, STDIN_FILENO) < 0 ||
1379 } 1379 dup2 (output_fd, STDOUT_FILENO) < 0 ||
1380 1380 execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, decomp->decomp_options, NULL))
1381 _exit (127); 1381 {
1382 } 1382 XPR(NT "child process %s failed to execute: %s\n",
1383 1383 decomp->decomp_cmdname, xd3_strerror (get_errno ()));
1384 close (input_fd); 1384 }
1385 close (output_fd); 1385
1386 input_fd = -1; 1386 _exit (127);
1387 output_fd = -1; 1387 }
1388 1388
1389 /* Then wait for completion. */ 1389 close (input_fd);
1390 if ((ret = main_waitpid_check (decomp_id))) 1390 close (output_fd);
1391 { 1391 input_fd = -1;
1392 goto cleanup; 1392 output_fd = -1;
1393 } 1393
1394 1394 /* Then wait for completion. */
1395 /* Open/stat the decompressed source file. */ 1395 if ((ret = main_waitpid_check (decomp_id)))
1396 if ((ret = main_file_open (sfile, tmpname, XO_READ))) { goto cleanup; } 1396 {
1397 if ((ret = main_file_stat (sfile, & source->size, 1))) { goto cleanup; } 1397 goto cleanup;
1398 return 0; 1398 }
1399 1399
1400 cleanup: 1400 /* Open/stat the decompressed source file. */
1401 close (input_fd); 1401 if ((ret = main_file_open (sfile, tmpname, XO_READ))) { goto cleanup; }
1402 close (output_fd); 1402 if ((ret = main_file_stat (sfile, & source->size, 1))) { goto cleanup; }
1403 if (tmpname) { free (tmpname); } 1403 return 0;
1404 ext_tmpfile = NULL; 1404
1405 return ret; 1405 cleanup:
1406} 1406 close (input_fd);
1407 1407 close (output_fd);
1408/* Initiate re-compression of the output stream. This is easier than input decompression 1408 if (tmpname) { free (tmpname); }
1409 * because we know beforehand that the stream will be compressed, whereas the input has 1409 ext_tmpfile = NULL;
1410 * already been read when we decide it should be decompressed. Thus, it only requires one 1410 return ret;
1411 * subprocess and one pipe. */ 1411}
1412static int 1412
1413main_recompress_output (main_file *ofile) 1413/* Initiate re-compression of the output stream. This is easier than input decompression
1414{ 1414 * because we know beforehand that the stream will be compressed, whereas the input has
1415 pid_t recomp_id; /* One subproc. */ 1415 * already been read when we decide it should be decompressed. Thus, it only requires one
1416 int pipefd[2]; /* One pipe. */ 1416 * subprocess and one pipe. */
1417 int output_fd = -1; 1417static int
1418 int ret; 1418main_recompress_output (main_file *ofile)
1419 const main_extcomp *recomp = ofile->compressor; 1419{
1420 1420 pid_t recomp_id; /* One subproc. */
1421 pipefd[0] = pipefd[1] = -1; 1421 int pipefd[2]; /* One pipe. */
1422 1422 int output_fd = -1;
1423 if (pipe (pipefd)) 1423 int ret;
1424 { 1424 const main_extcomp *recomp = ofile->compressor;
1425 XPR(NT "pipe failed: %s\n", xd3_strerror (ret = get_errno ())); 1425
1426 goto pipe_cleanup; 1426 pipefd[0] = pipefd[1] = -1;
1427 } 1427
1428 1428 if (pipe (pipefd))
1429 if ((recomp_id = fork ()) < 0) 1429 {
1430 { 1430 XPR(NT "pipe failed: %s\n", xd3_strerror (ret = get_errno ()));
1431 XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ())); 1431 goto pipe_cleanup;
1432 goto pipe_cleanup; 1432 }
1433 } 1433
1434 1434 if ((recomp_id = fork ()) < 0)
1435 /* The child runs the recompression process: */ 1435 {
1436 if (recomp_id == 0) 1436 XPR(NT "fork failed: %s\n", xd3_strerror (ret = get_errno ()));
1437 { 1437 goto pipe_cleanup;
1438 /* Setup pipes: write to the output file, read from the pipe. */ 1438 }
1439 if (dup2 (XFNO (ofile), STDOUT_FILENO) < 0 || 1439
1440 dup2 (pipefd[PIPE_READ_FD], STDIN_FILENO) < 0 || 1440 /* The child runs the recompression process: */
1441 close (pipefd[PIPE_READ_FD]) || 1441 if (recomp_id == 0)
1442 close (pipefd[PIPE_WRITE_FD]) || 1442 {
1443 execlp (recomp->recomp_cmdname, recomp->recomp_cmdname, recomp->recomp_options, NULL)) 1443 /* Setup pipes: write to the output file, read from the pipe. */
1444 { 1444 if (dup2 (XFNO (ofile), STDOUT_FILENO) < 0 ||
1445 XPR(NT "child process %s failed to execute: %s\n", recomp->recomp_cmdname, xd3_strerror (get_errno ())); 1445 dup2 (pipefd[PIPE_READ_FD], STDIN_FILENO) < 0 ||
1446 } 1446 close (pipefd[PIPE_READ_FD]) ||
1447 1447 close (pipefd[PIPE_WRITE_FD]) ||
1448 _exit (127); 1448 execlp (recomp->recomp_cmdname, recomp->recomp_cmdname, recomp->recomp_options, NULL))
1449 } 1449 {
1450 1450 XPR(NT "child process %s failed to execute: %s\n", recomp->recomp_cmdname, xd3_strerror (get_errno ()));
1451 ext_subprocs[0] = recomp_id; 1451 }
1452 1452
1453 /* The parent closes both pipes after duplicating the output-fd for writing to the 1453 _exit (127);
1454 * compression pipe. */ 1454 }
1455 output_fd = dup (pipefd[PIPE_WRITE_FD]); 1455
1456 1456 ext_subprocs[0] = recomp_id;
1457 if (output_fd < 0 || 1457
1458 main_file_close (ofile) || 1458 /* The parent closes both pipes after duplicating the output-fd for writing to the
1459 close (pipefd[PIPE_READ_FD]) || 1459 * compression pipe. */
1460 close (pipefd[PIPE_WRITE_FD])) 1460 output_fd = dup (pipefd[PIPE_WRITE_FD]);
1461 { 1461
1462 XPR(NT "close failed: %s\n", xd3_strerror (ret = get_errno ())); 1462 if (output_fd < 0 ||
1463 goto pipe_cleanup; 1463 main_file_close (ofile) ||
1464 } 1464 close (pipefd[PIPE_READ_FD]) ||
1465 1465 close (pipefd[PIPE_WRITE_FD]))
1466#if XD3_STDIO 1466 {
1467 /* Note: fdopen() acquires the fd, closes it when finished. */ 1467 XPR(NT "close failed: %s\n", xd3_strerror (ret = get_errno ()));
1468 if ((ofile->file = fdopen (output_fd, "w")) == NULL) 1468 goto pipe_cleanup;
1469 { 1469 }
1470 XPR(NT "fdopen failed: %s\n", xd3_strerror (ret = get_errno ())); 1470
1471 goto pipe_cleanup; 1471#if XD3_STDIO
1472 } 1472 /* Note: fdopen() acquires the fd, closes it when finished. */
1473 1473 if ((ofile->file = fdopen (output_fd, "w")) == NULL)
1474#elif XD3_POSIX 1474 {
1475 ofile->file = output_fd; 1475 XPR(NT "fdopen failed: %s\n", xd3_strerror (ret = get_errno ()));
1476#endif 1476 goto pipe_cleanup;
1477 1477 }
1478 /* Now the output file will be compressed. */ 1478
1479 return 0; 1479#elif XD3_POSIX
1480 1480 ofile->file = output_fd;
1481 pipe_cleanup: 1481#endif
1482 close (output_fd); 1482
1483 close (pipefd[PIPE_READ_FD]); 1483 /* Now the output file will be compressed. */
1484 close (pipefd[PIPE_WRITE_FD]); 1484 return 0;
1485 return ret; 1485
1486} 1486 pipe_cleanup:
1487#endif /* EXTERNAL_COMPRESSION */ 1487 close (output_fd);
1488 1488 close (pipefd[PIPE_READ_FD]);
1489/* Identify the compressor that was used based on its ident string, which is passed in the 1489 close (pipefd[PIPE_WRITE_FD]);
1490 * application header. */ 1490 return ret;
1491static const main_extcomp* 1491}
1492main_ident_compressor (const char *ident) 1492#endif /* EXTERNAL_COMPRESSION */
1493{ 1493
1494 int i; 1494/* Identify the compressor that was used based on its ident string, which is passed in the
1495 1495 * application header. */
1496 for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) 1496static const main_extcomp*
1497 { 1497main_ident_compressor (const char *ident)
1498 if (strcmp (extcomp_types[i].ident, ident) == 0) 1498{
1499 { 1499 int i;
1500 return & extcomp_types[i]; 1500
1501 } 1501 for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)
1502 } 1502 {
1503 1503 if (strcmp (extcomp_types[i].ident, ident) == 0)
1504 return NULL; 1504 {
1505} 1505 return & extcomp_types[i];
1506 1506 }
1507/* Return the main_extcomp record to use for this identifier, if possible. */ 1507 }
1508static const main_extcomp* 1508
1509main_get_compressor (const char *ident) 1509 return NULL;
1510{ 1510}
1511 const main_extcomp *ext = main_ident_compressor (ident); 1511
1512 1512/* Return the main_extcomp record to use for this identifier, if possible. */
1513 if (ext == NULL) 1513static const main_extcomp*
1514 { 1514main_get_compressor (const char *ident)
1515 if (! option_quiet) 1515{
1516 { 1516 const main_extcomp *ext = main_ident_compressor (ident);
1517 XPR(NT "warning: cannot recompress output: " 1517
1518 "unrecognized external compression ID: %s\n", ident); 1518 if (ext == NULL)
1519 } 1519 {
1520 return NULL; 1520 if (! option_quiet)
1521 } 1521 {
1522 else if (! EXTERNAL_COMPRESSION) 1522 XPR(NT "warning: cannot recompress output: "
1523 { 1523 "unrecognized external compression ID: %s\n", ident);
1524 if (! option_quiet) 1524 }
1525 { 1525 return NULL;
1526 XPR(NT "warning: external support not compiled: " 1526 }
1527 "original input was compressed: %s\n", ext->recomp_cmdname); 1527 else if (! EXTERNAL_COMPRESSION)
1528 } 1528 {
1529 return NULL; 1529 if (! option_quiet)
1530 } 1530 {
1531 else 1531 XPR(NT "warning: external support not compiled: "
1532 { 1532 "original input was compressed: %s\n", ext->recomp_cmdname);
1533 return ext; 1533 }
1534 } 1534 return NULL;
1535} 1535 }
1536 1536 else
1537/****************************************************************************************** 1537 {
1538 APPLICATION HEADER 1538 return ext;
1539 ******************************************************************************************/ 1539 }
1540 1540}
1541#if XD3_ENCODER 1541
1542static const char* 1542/******************************************************************************************
1543main_apphead_string (const char* x) 1543 APPLICATION HEADER
1544{ 1544 ******************************************************************************************/
1545 const char *y; 1545
1546 1546#if XD3_ENCODER
1547 if (x == NULL) { return ""; } 1547static const char*
1548 1548main_apphead_string (const char* x)
1549 if (strcmp (x, "/dev/stdin") == 0 || 1549{
1550 strcmp (x, "/dev/stdout") == 0 || 1550 const char *y;
1551 strcmp (x, "/dev/stderr") == 0) { return "-"; } 1551
1552 1552 if (x == NULL) { return ""; }
1553 // TODO: this is not portable 1553
1554 return (y = strrchr (x, '/')) == NULL ? x : y + 1; 1554 if (strcmp (x, "/dev/stdin") == 0 ||
1555} 1555 strcmp (x, "/dev/stdout") == 0 ||
1556 1556 strcmp (x, "/dev/stderr") == 0) { return "-"; }
1557static int 1557
1558main_set_appheader (xd3_stream *stream, main_file *input, main_file *sfile) 1558 // TODO: this is not portable
1559{ 1559 return (y = strrchr (x, '/')) == NULL ? x : y + 1;
1560 /* The user may disable the application header. Once the appheader is set, this 1560}
1561 * disables setting it again. */ 1561
1562 if (appheader_used || ! option_use_appheader) { return 0; } 1562static int
1563 1563main_set_appheader (xd3_stream *stream, main_file *input, main_file *sfile)
1564 /* The user may specify the application header, otherwise format the default header. */ 1564{
1565 if (option_appheader) 1565 /* The user may disable the application header. Once the appheader is set, this
1566 { 1566 * disables setting it again. */
1567 appheader_used = option_appheader; 1567 if (appheader_used || ! option_use_appheader) { return 0; }
1568 } 1568
1569 else 1569 /* The user may specify the application header, otherwise format the default header. */
1570 { 1570 if (option_appheader)
1571 const char *iname; 1571 {
1572 const char *icomp; 1572 appheader_used = option_appheader;
1573 const char *sname; 1573 }
1574 const char *scomp; 1574 else
1575 int len; 1575 {
1576 1576 const char *iname;
1577 iname = main_apphead_string (input->filename); 1577 const char *icomp;
1578 icomp = (input->compressor == NULL) ? "" : input->compressor->ident; 1578 const char *sname;
1579 len = strlen (iname) + strlen (icomp) + 2; 1579 const char *scomp;
1580 1580 int len;
1581 if (sfile->filename != NULL) 1581
1582 { 1582 iname = main_apphead_string (input->filename);
1583 sname = main_apphead_string (sfile->filename); 1583 icomp = (input->compressor == NULL) ? "" : input->compressor->ident;
1584 scomp = (sfile->compressor == NULL) ? "" : sfile->compressor->ident; 1584 len = strlen (iname) + strlen (icomp) + 2;
1585 len += strlen (sname) + strlen (scomp) + 2; 1585
1586 } 1586 if (sfile->filename != NULL)
1587 else 1587 {
1588 { 1588 sname = main_apphead_string (sfile->filename);
1589 sname = scomp = ""; 1589 scomp = (sfile->compressor == NULL) ? "" : sfile->compressor->ident;
1590 } 1590 len += strlen (sname) + strlen (scomp) + 2;
1591 1591 }
1592 if ((appheader_used = main_malloc (len)) == NULL) 1592 else
1593 { 1593 {
1594 return ENOMEM; 1594 sname = scomp = "";
1595 } 1595 }
1596 1596
1597 if (sfile->filename == NULL) 1597 if ((appheader_used = main_malloc (len)) == NULL)
1598 { 1598 {
1599 sprintf ((char*)appheader_used, "%s/%s", iname, icomp); 1599 return ENOMEM;
1600 } 1600 }
1601 else 1601
1602 { 1602 if (sfile->filename == NULL)
1603 sprintf ((char*)appheader_used, "%s/%s/%s/%s", iname, icomp, sname, scomp); 1603 {
1604 } 1604 sprintf ((char*)appheader_used, "%s/%s", iname, icomp);
1605 } 1605 }
1606 1606 else
1607 xd3_set_appheader (stream, appheader_used, strlen ((char*)appheader_used)); 1607 {
1608 1608 sprintf ((char*)appheader_used, "%s/%s/%s/%s", iname, icomp, sname, scomp);
1609 return 0; 1609 }
1610} 1610 }
1611#endif 1611
1612 1612 xd3_set_appheader (stream, appheader_used, strlen ((char*)appheader_used));
1613static void 1613
1614main_get_appheader_params (main_file *file, char **parsed, int output, const char *type, 1614 return 0;
1615 main_file *other) 1615}
1616{ 1616#endif
1617 /* Set the filename if it was not specified. If output, option_stdout (-c) overrides. */ 1617
1618 if (file->filename == NULL && ! (output && option_stdout) && strcmp (parsed[0], "-") != 0) 1618static void
1619 { 1619main_get_appheader_params (main_file *file, char **parsed, int output, const char *type,
1620 file->filename = parsed[0]; 1620 main_file *other)
1621 1621{
1622 if (other->filename != NULL) { 1622 /* Set the filename if it was not specified. If output, option_stdout (-c) overrides. */
1623 /* Take directory from the other file, if it has one. */ 1623 if (file->filename == NULL && ! (output && option_stdout) && strcmp (parsed[0], "-") != 0)
1624 char *last_slash = strrchr(other->filename, '/'); 1624 {
1625 1625 file->filename = parsed[0];
1626 if (last_slash != NULL) { 1626
1627 int dlen = last_slash - other->filename; 1627 if (other->filename != NULL) {
1628 file->filename_copy = main_malloc(dlen + 2 + strlen(file->filename)); 1628 /* Take directory from the other file, if it has one. */
1629 1629 char *last_slash = strrchr(other->filename, '/');
1630 strncpy(file->filename_copy, other->filename, dlen); 1630
1631 file->filename_copy[dlen] = '/'; 1631 if (last_slash != NULL) {
1632 strcpy(file->filename_copy + dlen + 1, parsed[0]); 1632 int dlen = last_slash - other->filename;
1633 1633 file->filename_copy = main_malloc(dlen + 2 + strlen(file->filename));
1634 file->filename = file->filename_copy; 1634
1635 } 1635 strncpy(file->filename_copy, other->filename, dlen);
1636 } 1636 file->filename_copy[dlen] = '/';
1637 1637 strcpy(file->filename_copy + dlen + 1, parsed[0]);
1638 if (! option_quiet) 1638
1639 { 1639 file->filename = file->filename_copy;
1640 XPR(NT "using default %s filename: %s\n", type, file->filename); 1640 }
1641 } 1641 }
1642 } 1642
1643 1643 if (! option_quiet)
1644 /* Set the compressor, initiate de/recompression later. */ 1644 {
1645 if (file->compressor == NULL && *parsed[1] != 0) 1645 XPR(NT "using default %s filename: %s\n", type, file->filename);
1646 { 1646 }
1647 file->compressor = main_get_compressor (parsed[1]); 1647 }
1648 } 1648
1649} 1649 /* Set the compressor, initiate de/recompression later. */
1650 1650 if (file->compressor == NULL && *parsed[1] != 0)
1651static void 1651 {
1652main_get_appheader (xd3_stream *stream, main_file *ifile, main_file *output, main_file *sfile) 1652 file->compressor = main_get_compressor (parsed[1]);
1653{ 1653 }
1654 uint8_t *apphead; 1654}
1655 usize_t appheadsz; 1655
1656 int ret; 1656static void
1657 1657main_get_appheader (xd3_stream *stream, main_file *ifile, main_file *output, main_file *sfile)
1658 /* The user may disable the application header. Once the appheader is set, this 1658{
1659 * disables setting it again. */ 1659 uint8_t *apphead;
1660 if (! option_use_appheader) { return; } 1660 usize_t appheadsz;
1661 1661 int ret;
1662 ret = xd3_get_appheader (stream, & apphead, & appheadsz); 1662
1663 1663 /* The user may disable the application header. Once the appheader is set, this
1664 /* Ignore failure, it only means we haven't received a header yet. */ 1664 * disables setting it again. */
1665 if (ret != 0) { return; } 1665 if (! option_use_appheader) { return; }
1666 1666
1667 if (appheadsz > 0) 1667 ret = xd3_get_appheader (stream, & apphead, & appheadsz);
1668 { 1668
1669 char *start = (char*)apphead; 1669 /* Ignore failure, it only means we haven't received a header yet. */
1670 char *slash; 1670 if (ret != 0) { return; }
1671 int place = 0; 1671
1672 char *parsed[4]; 1672 if (appheadsz > 0)
1673 1673 {
1674 memset (parsed, 0, sizeof (parsed)); 1674 char *start = (char*)apphead;
1675 1675 char *slash;
1676 while ((slash = strchr (start, '/')) != NULL) 1676 int place = 0;
1677 { 1677 char *parsed[4];
1678 *slash = 0; 1678
1679 parsed[place++] = start; 1679 memset (parsed, 0, sizeof (parsed));
1680 start = slash + 1; 1680
1681 } 1681 while ((slash = strchr (start, '/')) != NULL)
1682 1682 {
1683 parsed[place++] = start; 1683 *slash = 0;
1684 1684 parsed[place++] = start;
1685 /* First take the output parameters. */ 1685 start = slash + 1;
1686 if (place == 2 || place == 4) 1686 }
1687 { 1687
1688 main_get_appheader_params (output, parsed, 1, "output", ifile); 1688 parsed[place++] = start;
1689 } 1689
1690 1690 /* First take the output parameters. */
1691 /* Then take the source parameters. */ 1691 if (place == 2 || place == 4)
1692 if (place == 4) 1692 {
1693 { 1693 main_get_appheader_params (output, parsed, 1, "output", ifile);
1694 main_get_appheader_params (sfile, parsed+2, 0, "source", ifile); 1694 }
1695 } 1695
1696 } 1696 /* Then take the source parameters. */
1697 1697 if (place == 4)
1698 option_use_appheader = 0; 1698 {
1699 return; 1699 main_get_appheader_params (sfile, parsed+2, 0, "source", ifile);
1700} 1700 }
1701 1701 }
1702/****************************************************************************************** 1702
1703 Main I/O routines 1703 option_use_appheader = 0;
1704 ******************************************************************************************/ 1704 return;
1705 1705}
1706/* This function acts like the above except it may also try to recognize a compressed 1706
1707 * input when the first buffer of data is read. The EXTERNAL_COMPRESSION code is called 1707/******************************************************************************************
1708 * to search for magic numbers. */ 1708 Main I/O routines
1709static int 1709 ******************************************************************************************/
1710main_read_primary_input (main_file *ifile, 1710
1711 uint8_t *buf, 1711/* This function acts like the above except it may also try to recognize a compressed
1712 usize_t size, 1712 * input when the first buffer of data is read. The EXTERNAL_COMPRESSION code is called
1713 usize_t *nread) 1713 * to search for magic numbers. */
1714{ 1714static int
1715#if EXTERNAL_COMPRESSION 1715main_read_primary_input (main_file *ifile,
1716 if (option_decompress_inputs && ifile->flags & RD_FIRST) 1716 uint8_t *buf,
1717 { 1717 usize_t size,
1718 ifile->flags &= ~RD_FIRST; 1718 usize_t *nread)
1719 1719{
1720 return main_decompress_input_check (ifile, buf, size, nread); 1720#if EXTERNAL_COMPRESSION
1721 } 1721 if (option_decompress_inputs && ifile->flags & RD_FIRST)
1722#endif 1722 {
1723 1723 ifile->flags &= ~RD_FIRST;
1724 return main_file_read (ifile, buf, size, nread, "input read failed"); 1724
1725} 1725 return main_decompress_input_check (ifile, buf, size, nread);
1726 1726 }
1727/* This function simply writes the stream output buffer, if there is any. This is used 1727#endif
1728 * for both encode and decode commands. (The VCDIFF tools use main_print_func()). */ 1728
1729static int 1729 return main_file_read (ifile, buf, size, nread, "input read failed");
1730main_write_output (xd3_stream* stream, main_file *ofile) 1730}
1731{ 1731
1732 int ret; 1732/* This function simply writes the stream output buffer, if there is any. This is used
1733 1733 * for both encode and decode commands. (The VCDIFF tools use main_print_func()). */
1734 if (stream->avail_out > 0 && (ret = main_file_write (ofile, stream->next_out, stream->avail_out, "write failed"))) 1734static int
1735 { 1735main_write_output (xd3_stream* stream, main_file *ofile)
1736 return ret; 1736{
1737 } 1737 int ret;
1738 1738
1739 return 0; 1739 if (stream->avail_out > 0 && (ret = main_file_write (ofile, stream->next_out, stream->avail_out, "write failed")))
1740} 1740 {
1741 1741 return ret;
1742/* Open the main output file, sets a default file name, initiate recompression. This 1742 }
1743 * function is expected to fprint any error messages. */ 1743
1744static int 1744 return 0;
1745main_open_output (xd3_stream *stream, main_file *ofile) 1745}
1746{ 1746
1747 int ret; 1747/* Open the main output file, sets a default file name, initiate recompression. This
1748 1748 * function is expected to fprint any error messages. */
1749 if (ofile->filename == NULL) 1749static int
1750 { 1750main_open_output (xd3_stream *stream, main_file *ofile)
1751 XSTDOUT_XF (ofile); 1751{
1752 1752 int ret;
1753 if (option_verbose > 1) { XPR(NT "using standard output: %s\n", ofile->filename); } 1753
1754 } 1754 if (ofile->filename == NULL)
1755 else 1755 {
1756 { 1756 XSTDOUT_XF (ofile);
1757 /* Stat the file to check for overwrite. */ 1757
1758 if (option_force == 0 && main_file_exists (ofile)) 1758 if (option_verbose > 1) { XPR(NT "using standard output: %s\n", ofile->filename); }
1759 { 1759 }
1760 XPR(NT "to overwrite output file specify -f: %s\n", ofile->filename); 1760 else
1761 return EEXIST; 1761 {
1762 } 1762 /* Stat the file to check for overwrite. */
1763 1763 if (option_force == 0 && main_file_exists (ofile))
1764 if ((ret = main_file_open (ofile, ofile->filename, XO_WRITE))) 1764 {
1765 { 1765 XPR(NT "to overwrite output file specify -f: %s\n", ofile->filename);
1766 return ret; 1766 return EEXIST;
1767 } 1767 }
1768 1768
1769 if (option_verbose > 1) { XPR(NT "open output: %s\n", ofile->filename); } 1769 if ((ret = main_file_open (ofile, ofile->filename, XO_WRITE)))
1770 } 1770 {
1771 1771 return ret;
1772#if EXTERNAL_COMPRESSION 1772 }
1773 /* Do output recompression. */ 1773
1774 if (ofile->compressor != NULL && option_recompress_outputs == 1) 1774 if (option_verbose > 1) { XPR(NT "open output: %s\n", ofile->filename); }
1775 { 1775 }
1776 if (! option_quiet) 1776
1777 { 1777#if EXTERNAL_COMPRESSION
1778 XPR(NT "%s %s | %s\n", 1778 /* Do output recompression. */
1779 ofile->compressor->recomp_cmdname, 1779 if (ofile->compressor != NULL && option_recompress_outputs == 1)
1780 ofile->compressor->recomp_options, 1780 {
1781 ofile->filename); 1781 if (! option_quiet)
1782 } 1782 {
1783 1783 XPR(NT "%s %s | %s\n",
1784 if ((ret = main_recompress_output (ofile))) 1784 ofile->compressor->recomp_cmdname,
1785 { 1785 ofile->compressor->recomp_options,
1786 return ret; 1786 ofile->filename);
1787 } 1787 }
1788 } 1788
1789#endif 1789 if ((ret = main_recompress_output (ofile)))
1790 1790 {
1791 return 0; 1791 return ret;
1792} 1792 }
1793 1793 }
1794/* This is called at different times for encoding and decoding. The encoder calls it 1794#endif
1795 * immediately, the decoder delays until the application header is received. */ 1795
1796static int 1796 return 0;
1797main_set_source (xd3_stream *stream, int cmd, main_file *sfile, xd3_source *source) 1797}
1798{ 1798
1799 int ret, i; 1799/* This is called at different times for encoding and decoding. The encoder calls it
1800 1800 * immediately, the decoder delays until the application header is received. */
1801 /* Open it, check for seekability, set required xd3_source fields. */ 1801static int
1802 if (allow_fake_source) 1802main_set_source (xd3_stream *stream, int cmd, main_file *sfile, xd3_source *source)
1803 { 1803{
1804 sfile->mode = XO_READ; 1804 int ret, i;
1805 sfile->realname = sfile->filename; 1805
1806 sfile->nread = 0; 1806 /* Open it, check for seekability, set required xd3_source fields. */
1807 source->size = UINT64_MAX; 1807 if (allow_fake_source)
1808 } 1808 {
1809 else if ((ret = main_file_open (sfile, sfile->filename, XO_READ)) || 1809 sfile->mode = XO_READ;
1810 (ret = main_file_stat (sfile, & source->size, 1))) 1810 sfile->realname = sfile->filename;
1811 { 1811 sfile->nread = 0;
1812 return ret; 1812 source->size = UINT64_MAX;
1813 } 1813 }
1814 1814 else if ((ret = main_file_open (sfile, sfile->filename, XO_READ)) ||
1815 source->name = sfile->filename; 1815 (ret = main_file_stat (sfile, & source->size, 1)))
1816 source->ioh = sfile; 1816 {
1817 source->curblkno = (xoff_t) -1; 1817 return ret;
1818 source->curblk = NULL; 1818 }
1819 1819
1820 /* Source block LRU init. */ 1820 source->name = sfile->filename;
1821 main_blklru_list_init (& lru_list); 1821 source->ioh = sfile;
1822 main_blklru_list_init (& lru_free); 1822 source->curblkno = (xoff_t) -1;
1823 1823 source->curblk = NULL;
1824 option_srcwinsz = min(source->size, (xoff_t) option_srcwinsz); 1824
1825 1825 /* Source block LRU init. */
1826 if (option_verbose > 1) { XPR(NT "source window size: %u\n", option_srcwinsz); } 1826 main_blklru_list_init (& lru_list);
1827 if (option_verbose > 1) { XPR(NT "source block size: %u\n", source->blksize); } 1827 main_blklru_list_init (& lru_free);
1828 if (option_verbose > 1) { XPR(NT "memory size: %u\n", option_memsize); } 1828
1829 1829 // Note: to avoid unnecessary allocs for small files: problem
1830 lru_size = (option_srcwinsz / source->blksize); 1830 // was with --decompress_inputs, we don't know the true size
1831 lru_size = max(1, lru_size); 1831 // option_srcwinsz = min(source->size, (xoff_t) option_srcwinsz);
1832 1832
1833 if ((lru = main_malloc (sizeof (main_blklru) * lru_size)) == NULL) 1833 if (option_verbose > 1) { XPR(NT "source window size: %u\n", option_srcwinsz); }
1834 { 1834 if (option_verbose > 1) { XPR(NT "source block size: %u\n", source->blksize); }
1835 return ENOMEM; 1835 if (option_verbose > 1) { XPR(NT "memory size: %u\n", option_memsize); }
1836 } 1836
1837 1837 lru_size = (option_srcwinsz / source->blksize);
1838 for (i = 0; i < lru_size; i += 1) 1838 lru_size = max(1, lru_size);
1839 { 1839
1840 lru[i].blkno = (xoff_t) -1; 1840 if ((lru = main_malloc (sizeof (main_blklru) * lru_size)) == NULL)
1841 1841 {
1842 if ((lru[i].blk = main_malloc (source->blksize)) == NULL) 1842 return ENOMEM;
1843 { 1843 }
1844 return ENOMEM; 1844
1845 } 1845 for (i = 0; i < lru_size; i += 1)
1846 1846 {
1847 main_blklru_list_push_back (& lru_free, & lru[i]); 1847 lru[i].blkno = (xoff_t) -1;
1848 } 1848
1849 1849 if ((lru[i].blk = main_malloc (source->blksize)) == NULL)
1850#if EXTERNAL_COMPRESSION 1850 {
1851 if (option_decompress_inputs) 1851 return ENOMEM;
1852 { 1852 }
1853 if (IS_ENCODE (cmd)) 1853
1854 { 1854 main_blklru_list_push_back (& lru_free, & lru[i]);
1855 usize_t nread; 1855 }
1856 1856
1857 source->curblk = lru[0].blk; 1857#if EXTERNAL_COMPRESSION
1858 1858 if (option_decompress_inputs)
1859 /* If encoding, read the first block now to check for decompression. */ 1859 {
1860 if ((ret = main_file_read (sfile, (uint8_t*) source->curblk, source->blksize, & nread, "source read failed"))) 1860 if (IS_ENCODE (cmd))
1861 { 1861 {
1862 return ret; 1862 usize_t nread;
1863 } 1863
1864 1864 source->curblk = lru[0].blk;
1865 /* Check known magic numbers. */ 1865
1866 for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) 1866 /* If encoding, read the first block now to check for decompression. */
1867 { 1867 if ((ret = main_file_read (sfile, (uint8_t*) source->curblk, source->blksize, & nread, "source read failed")))
1868 const main_extcomp *decomp = & extcomp_types[i]; 1868 {
1869 1869 return ret;
1870 if ((nread > decomp->magic_size) && memcmp (source->curblk, decomp->magic, decomp->magic_size) == 0) 1870 }
1871 { 1871
1872 sfile->compressor = decomp; 1872 /* Check known magic numbers. */
1873 break; 1873 for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)
1874 } 1874 {
1875 } 1875 const main_extcomp *decomp = & extcomp_types[i];
1876 1876
1877 /* If no decompression, the current buffer is now a valid source->curblock. */ 1877 if ((nread > decomp->magic_size) && memcmp (source->curblk, decomp->magic, decomp->magic_size) == 0)
1878 if (sfile->compressor == NULL) 1878 {
1879 { 1879 sfile->compressor = decomp;
1880 main_blklru_list_remove (& lru[0]); 1880 break;
1881 main_blklru_list_push_back (& lru_list, & lru[0]); 1881 }
1882 1882 }
1883 lru[0].blkno = 0; 1883
1884 source->curblkno = 0; 1884 /* If no decompression, the current buffer is now a valid source->curblock. */
1885 source->onblk = nread; 1885 if (sfile->compressor == NULL)
1886 1886 {
1887 if (option_verbose > 2) 1887 main_blklru_list_remove (& lru[0]);
1888 { 1888 main_blklru_list_push_back (& lru_list, & lru[0]);
1889 XPR(NT "source block 0 read (not compressed)\n"); 1889
1890 } 1890 lru[0].blkno = 0;
1891 } 1891 source->curblkno = 0;
1892 } 1892 source->onblk = nread;
1893 1893
1894 /* In either the encoder or decoder, start decompression. */ 1894 if (option_verbose > 2)
1895 if (sfile->compressor) 1895 {
1896 { 1896 XPR(NT "source block 0 read (not compressed)\n");
1897 xoff_t osize = source->size; 1897 }
1898 1898 }
1899 if (osize > XD3_NODECOMPRESSSIZE) 1899 }
1900 { 1900
1901 XPR(NT "source file too large for external decompression: %s: %"Q"u\n", 1901 /* In either the encoder or decoder, start decompression. */
1902 sfile->filename, osize); 1902 if (sfile->compressor)
1903 return XD3_INTERNAL; 1903 {
1904 } 1904 xoff_t osize = source->size;
1905 1905
1906 if ((ret = main_decompress_source (sfile, source))) 1906 if (osize > XD3_NODECOMPRESSSIZE)
1907 { 1907 {
1908 return ret; 1908 XPR(NT "source file too large for external decompression: %s: %"Q"u\n",
1909 } 1909 sfile->filename, osize);
1910 1910 return XD3_INTERNAL;
1911 if (! option_quiet) 1911 }
1912 { 1912
1913 char s1[32], s2[32]; 1913 if ((ret = main_decompress_source (sfile, source)))
1914 XPR(NT "%s | %s %s => %s %.1f%% [ %s , %s ]\n", 1914 {
1915 sfile->filename, 1915 return ret;
1916 sfile->compressor->decomp_cmdname, 1916 }
1917 sfile->compressor->decomp_options, 1917
1918 sfile->realname, 1918 if (! option_quiet)
1919 100.0 * source->size / osize, 1919 {
1920 main_format_bcnt (osize, s1), 1920 char s1[32], s2[32];
1921 main_format_bcnt (source->size, s2)); 1921 XPR(NT "%s | %s %s => %s %.1f%% [ %s , %s ]\n",
1922 } 1922 sfile->filename,
1923 } 1923 sfile->compressor->decomp_cmdname,
1924 } 1924 sfile->compressor->decomp_options,
1925#endif 1925 sfile->realname,
1926 1926 100.0 * source->size / osize,
1927 if (option_verbose > 1) { XPR(NT "source file: %s: %"Q"u bytes\n", sfile->realname, source->size); } 1927 main_format_bcnt (osize, s1),
1928 1928 main_format_bcnt (source->size, s2));
1929 if ((ret = xd3_set_source (stream, source))) 1929 }
1930 { 1930 }
1931 XPR(NT XD3_LIB_ERRMSG (stream, ret)); 1931 }
1932 return EXIT_FAILURE; 1932#endif
1933 } 1933
1934 1934 if (option_verbose > 1) { XPR(NT "source file: %s: %"Q"u bytes\n", sfile->realname, source->size); }
1935 return 0; 1935
1936} 1936 if ((ret = xd3_set_source (stream, source)))
1937 1937 {
1938/****************************************************************************************** 1938 XPR(NT XD3_LIB_ERRMSG (stream, ret));
1939 Source routines 1939 return EXIT_FAILURE;
1940 ******************************************************************************************/ 1940 }
1941 1941
1942/* This is the callback for reading a block of source. This function is blocking and it 1942 return 0;
1943 * implements a small LRU. 1943}
1944 * 1944
1945 * Note that it is possible for main_input() to handle getblk requests in a non-blocking 1945/******************************************************************************************
1946 * manner. If the callback is NULL then the caller of xd3_*_input() must handle the 1946 Source routines
1947 * XD3_GETSRCBLK return value and fill the source in the same way. See xd3_getblk for 1947 ******************************************************************************************/
1948 * details. To see an example of non-blocking getblk, see xdelta-test.h. */ 1948
1949static int 1949/* This is the callback for reading a block of source. This function is blocking and it
1950main_getblk_func (xd3_stream *stream, 1950 * implements a small LRU.
1951 xd3_source *source, 1951 *
1952 xoff_t blkno) 1952 * Note that it is possible for main_input() to handle getblk requests in a non-blocking
1953{ 1953 * manner. If the callback is NULL then the caller of xd3_*_input() must handle the
1954 xoff_t pos = blkno * source->blksize; 1954 * XD3_GETSRCBLK return value and fill the source in the same way. See xd3_getblk for
1955 main_file *sfile = (main_file*) source->ioh; 1955 * details. To see an example of non-blocking getblk, see xdelta-test.h. */
1956 main_blklru *blru = NULL; 1956static int
1957 usize_t onblk = xd3_bytes_on_srcblk (source, blkno); 1957main_getblk_func (xd3_stream *stream,
1958 usize_t nread; 1958 xd3_source *source,
1959 int ret; 1959 xoff_t blkno)
1960 int i; 1960{
1961 1961 xoff_t pos = blkno * source->blksize;
1962 if (allow_fake_source) 1962 main_file *sfile = (main_file*) source->ioh;
1963 { 1963 main_blklru *blru = NULL;
1964 source->curblkno = blkno; 1964 usize_t onblk = xd3_bytes_on_srcblk (source, blkno);
1965 source->onblk = onblk; 1965 usize_t nread;
1966 source->curblk = lru[0].blk; 1966 int ret;
1967 return 0; 1967 int i;
1968 } 1968
1969 1969 if (allow_fake_source)
1970 if (do_not_lru) 1970 {
1971 { 1971 source->curblkno = blkno;
1972 /* Direct lookup assumes sequential scan w/o skipping blocks. */ 1972 source->onblk = onblk;
1973 int idx = blkno % lru_size; 1973 source->curblk = lru[0].blk;
1974 if (lru[idx].blkno == blkno) 1974 return 0;
1975 { 1975 }
1976 source->curblkno = blkno; 1976
1977 source->onblk = onblk; 1977 if (do_not_lru)
1978 source->curblk = lru[idx].blk; 1978 {
1979 lru_hits += 1; 1979 /* Direct lookup assumes sequential scan w/o skipping blocks. */
1980 return 0; 1980 int idx = blkno % lru_size;
1981 } 1981 if (lru[idx].blkno == blkno)
1982 1982 {
1983 if (lru[idx].blkno != -1LL && 1983 source->curblkno = blkno;
1984 lru[idx].blkno != blkno - lru_size) 1984 source->onblk = onblk;
1985 { 1985 source->curblk = lru[idx].blk;
1986 return XD3_TOOFARBACK; 1986 lru_hits += 1;
1987 } 1987 return 0;
1988 } 1988 }
1989 else 1989
1990 { 1990 if (lru[idx].blkno != -1LL &&
1991 /* Sequential search through LRU. */ 1991 lru[idx].blkno != blkno - lru_size)
1992 for (i = 0; i < lru_size; i += 1) 1992 {
1993 { 1993 return XD3_TOOFARBACK;
1994 if (lru[i].blkno == blkno) 1994 }
1995 { 1995 }
1996 main_blklru_list_remove (& lru[i]); 1996 else
1997 main_blklru_list_push_back (& lru_list, & lru[i]); 1997 {
1998 1998 /* Sequential search through LRU. */
1999 source->curblkno = blkno; 1999 for (i = 0; i < lru_size; i += 1)
2000 source->onblk = onblk; 2000 {
2001 source->curblk = lru[i].blk; 2001 if (lru[i].blkno == blkno)
2002 lru_hits += 1; 2002 {
2003 return 0; 2003 main_blklru_list_remove (& lru[i]);
2004 } 2004 main_blklru_list_push_back (& lru_list, & lru[i]);
2005 } 2005
2006 } 2006 source->curblkno = blkno;
2007 2007 source->onblk = onblk;
2008 if (! main_blklru_list_empty (& lru_free)) 2008 source->curblk = lru[i].blk;
2009 { 2009 lru_hits += 1;
2010 blru = main_blklru_list_pop_front (& lru_free); 2010 return 0;
2011 } 2011 }
2012 else if (! main_blklru_list_empty (& lru_list)) 2012 }
2013 { 2013 }
2014 if (do_not_lru) { 2014
2015 blru = & lru[blkno % lru_size]; 2015 if (! main_blklru_list_empty (& lru_free))
2016 main_blklru_list_remove(blru); 2016 {
2017 } else { 2017 blru = main_blklru_list_pop_front (& lru_free);
2018 blru = main_blklru_list_pop_front (& lru_list); 2018 }
2019 } 2019 else if (! main_blklru_list_empty (& lru_list))
2020 lru_misses += 1; 2020 {
2021 } 2021 if (do_not_lru) {
2022 2022 blru = & lru[blkno % lru_size];
2023 lru_filled += 1; 2023 main_blklru_list_remove(blru);
2024 2024 } else {
2025 if ((ret = main_file_seek (sfile, pos))) 2025 blru = main_blklru_list_pop_front (& lru_list);
2026 { 2026 }
2027 return ret; 2027 lru_misses += 1;
2028 } 2028 }
2029 2029
2030 if ((ret = main_file_read (sfile, (uint8_t*) blru->blk, source->blksize, 2030 lru_filled += 1;
2031 & nread, "source read failed"))) 2031
2032 { 2032 if ((ret = main_file_seek (sfile, pos)))
2033 return ret; 2033 {
2034 } 2034 return ret;
2035 2035 }
2036 if (nread != onblk) 2036
2037 { 2037 if ((ret = main_file_read (sfile, (uint8_t*) blru->blk, source->blksize,
2038 XPR(NT "source file size change: %s\n", sfile->filename); 2038 & nread, "source read failed")))
2039 return XD3_INTERNAL; 2039 {
2040 } 2040 return ret;
2041 2041 }
2042 main_blklru_list_push_back (& lru_list, blru); 2042
2043 2043 if (nread != onblk)
2044 if (option_verbose > 2) 2044 {
2045 { 2045 XPR(NT "source file size change: %s\n", sfile->filename);
2046 if (blru->blkno != -1LL) 2046 return XD3_INTERNAL;
2047 { 2047 }
2048 XPR(NT "source block %"Q"u ejects %"Q"u (lru_hits=%u, lru_misses=%u, lru_filled=%u)\n", 2048
2049 blkno, blru->blkno, lru_hits, lru_misses, lru_filled); 2049 main_blklru_list_push_back (& lru_list, blru);
2050 } 2050
2051 else 2051 if (option_verbose > 2)
2052 { 2052 {
2053 XPR(NT "source block %"Q"u read (lru_hits=%u, lru_misses=%u, lru_filled=%u)\n", 2053 if (blru->blkno != -1LL)
2054 blkno, lru_hits, lru_misses, lru_filled); 2054 {
2055 } 2055 XPR(NT "source block %"Q"u ejects %"Q"u (lru_hits=%u, lru_misses=%u, lru_filled=%u)\n",
2056 } 2056 blkno, blru->blkno, lru_hits, lru_misses, lru_filled);
2057 2057 }
2058 blru->blkno = blkno; 2058 else
2059 source->curblk = blru->blk; 2059 {
2060 source->curblkno = blkno; 2060 XPR(NT "source block %"Q"u read (lru_hits=%u, lru_misses=%u, lru_filled=%u)\n",
2061 source->onblk = onblk; 2061 blkno, lru_hits, lru_misses, lru_filled);
2062 2062 }
2063 return 0; 2063 }
2064} 2064
2065 2065 blru->blkno = blkno;
2066/****************************************************************************************** 2066 source->curblk = blru->blk;
2067 Main routines 2067 source->curblkno = blkno;
2068 ******************************************************************************************/ 2068 source->onblk = onblk;
2069 2069
2070/* This is a generic input function. It calls the xd3_encode_input or xd3_decode_input 2070 return 0;
2071 * functions and makes calls to the various input handling routines above, which 2071}
2072 * coordinate external decompression. 2072
2073 */ 2073/******************************************************************************************
2074static int 2074 Main routines
2075main_input (xd3_cmd cmd, 2075 ******************************************************************************************/
2076 main_file *ifile, 2076
2077 main_file *ofile, 2077/* This is a generic input function. It calls the xd3_encode_input or xd3_decode_input
2078 main_file *sfile) 2078 * functions and makes calls to the various input handling routines above, which
2079{ 2079 * coordinate external decompression.
2080 int ret; 2080 */
2081 xd3_stream stream; 2081static int
2082 usize_t nread; 2082main_input (xd3_cmd cmd,
2083 int stream_flags = 0; 2083 main_file *ifile,
2084 xd3_config config; 2084 main_file *ofile,
2085 xd3_source source; 2085 main_file *sfile)
2086 xoff_t last_total_in = 0; 2086{
2087 xoff_t last_total_out = 0; 2087 int ret;
2088 long start_time; 2088 xd3_stream stream;
2089 xoff_t input_size = 0; 2089 usize_t nread;
2090 2090 int stream_flags = 0;
2091 int (*input_func) (xd3_stream*); 2091 xd3_config config;
2092 int (*output_func) (xd3_stream*, main_file *); 2092 xd3_source source;
2093 2093 xoff_t last_total_in = 0;
2094 memset (& source, 0, sizeof (source)); 2094 xoff_t last_total_out = 0;
2095 memset (& config, 0, sizeof (config)); 2095 long start_time;
2096 2096 xoff_t input_size = 0;
2097 config.alloc = main_alloc; 2097
2098 config.freef = main_free1; 2098 int (*input_func) (xd3_stream*);
2099 config.sec_data.ngroups = 1; 2099 int (*output_func) (xd3_stream*, main_file *);
2100 config.sec_addr.ngroups = 1; 2100
2101 config.sec_inst.ngroups = 1; 2101 memset (& source, 0, sizeof (source));
2102 2102 memset (& config, 0, sizeof (config));
2103 /* main_input setup. */ 2103
2104 switch ((int) cmd) 2104 config.alloc = main_alloc;
2105 { 2105 config.freef = main_free1;
2106#if VCDIFF_TOOLS 2106 config.sec_data.ngroups = 1;
2107 if (1) { case CMD_PRINTHDR: stream_flags = XD3_JUST_HDR; } 2107 config.sec_addr.ngroups = 1;
2108 else if (1) { case CMD_PRINTHDRS: stream_flags = XD3_SKIP_WINDOW; } 2108 config.sec_inst.ngroups = 1;
2109 else { case CMD_PRINTDELTA: stream_flags = XD3_SKIP_EMIT; } 2109
2110 ifile->flags |= RD_NONEXTERNAL; 2110 /* main_input setup. */
2111 input_func = xd3_decode_input; 2111 switch ((int) cmd)
2112 output_func = main_print_func; 2112 {
2113 stream_flags |= XD3_ADLER32_NOVER; 2113#if VCDIFF_TOOLS
2114 break; 2114 if (1) { case CMD_PRINTHDR: stream_flags = XD3_JUST_HDR; }
2115#endif 2115 else if (1) { case CMD_PRINTHDRS: stream_flags = XD3_SKIP_WINDOW; }
2116#if XD3_ENCODER 2116 else { case CMD_PRINTDELTA: stream_flags = XD3_SKIP_EMIT; }
2117 case CMD_ENCODE: 2117 ifile->flags |= RD_NONEXTERNAL;
2118 input_func = xd3_encode_input; 2118 input_func = xd3_decode_input;
2119 output_func = main_write_output; 2119 output_func = main_print_func;
2120 2120 stream_flags |= XD3_ADLER32_NOVER;
2121 if (option_use_checksum) { stream_flags |= XD3_ADLER32; } 2121 break;
2122 if (option_use_secondary) 2122#endif
2123 { 2123#if XD3_ENCODER
2124 /* The default secondary compressor is DJW, if it's compiled, being used, etc. */ 2124 case CMD_ENCODE:
2125 if (option_secondary == NULL) 2125 input_func = xd3_encode_input;
2126 { 2126 output_func = main_write_output;
2127 if (SECONDARY_DJW) { stream_flags |= XD3_SEC_DJW; } 2127
2128 } 2128 if (option_use_checksum) { stream_flags |= XD3_ADLER32; }
2129 else 2129 if (option_use_secondary)
2130 { 2130 {
2131 if (strcmp (option_secondary, "fgk") == 0 && SECONDARY_FGK) 2131 /* The default secondary compressor is DJW, if it's compiled, being used, etc. */
2132 { 2132 if (option_secondary == NULL)
2133 stream_flags |= XD3_SEC_FGK; 2133 {
2134 } 2134 if (SECONDARY_DJW) { stream_flags |= XD3_SEC_DJW; }
2135 else if (strcmp (option_secondary, "djw") == 0 && SECONDARY_DJW) 2135 }
2136 { 2136 else
2137 stream_flags |= XD3_SEC_DJW; 2137 {
2138 } 2138 if (strcmp (option_secondary, "fgk") == 0 && SECONDARY_FGK)
2139 else 2139 {
2140 { 2140 stream_flags |= XD3_SEC_FGK;
2141 XPR(NT "unrecognized secondary compressor type: %s\n", option_secondary); 2141 }
2142 return EXIT_FAILURE; 2142 else if (strcmp (option_secondary, "djw") == 0 && SECONDARY_DJW)
2143 } 2143 {
2144 } 2144 stream_flags |= XD3_SEC_DJW;
2145 } 2145 }
2146 if (option_no_compress) { stream_flags |= XD3_NOCOMPRESS; } 2146 else
2147 if (option_use_altcodetable) { stream_flags |= XD3_ALT_CODE_TABLE; } 2147 {
2148 if (option_smatch_config) 2148 XPR(NT "unrecognized secondary compressor type: %s\n", option_secondary);
2149 { 2149 return EXIT_FAILURE;
2150 char *s = option_smatch_config, *e; 2150 }
2151 int values[XD3_SOFTCFG_VARCNT]; 2151 }
2152 int got; 2152 }
2153 2153 if (option_no_compress) { stream_flags |= XD3_NOCOMPRESS; }
2154 config.smatch_cfg = XD3_SMATCH_SOFT; 2154 if (option_use_altcodetable) { stream_flags |= XD3_ALT_CODE_TABLE; }
2155 2155 if (option_smatch_config)
2156 for (got = 0; got < XD3_SOFTCFG_VARCNT; got += 1, s = e + 1) 2156 {
2157 { 2157 char *s = option_smatch_config, *e;
2158 values[got] = strtol (s, &e, 10); 2158 int values[XD3_SOFTCFG_VARCNT];
2159 2159 int got;
2160 if ((values[got] < 0) || 2160
2161 (e == s) || 2161 config.smatch_cfg = XD3_SMATCH_SOFT;
2162 (got < XD3_SOFTCFG_VARCNT-1 && *e == 0) || 2162
2163 (got == XD3_SOFTCFG_VARCNT-1 && *e != 0)) 2163 for (got = 0; got < XD3_SOFTCFG_VARCNT; got += 1, s = e + 1)
2164 { 2164 {
2165 XPR(NT "invalid string match specifier (-C)\n"); 2165 values[got] = strtol (s, &e, 10);
2166 return EXIT_FAILURE; 2166
2167 } 2167 if ((values[got] < 0) ||
2168 } 2168 (e == s) ||
2169 2169 (got < XD3_SOFTCFG_VARCNT-1 && *e == 0) ||
2170 config.smatcher_soft.large_look = values[0]; 2170 (got == XD3_SOFTCFG_VARCNT-1 && *e != 0))
2171 config.smatcher_soft.large_step = values[1]; 2171 {
2172 config.smatcher_soft.small_look = values[2]; 2172 XPR(NT "invalid string match specifier (-C)\n");
2173 config.smatcher_soft.small_chain = values[3]; 2173 return EXIT_FAILURE;
2174 config.smatcher_soft.small_lchain = values[4]; 2174 }
2175 config.smatcher_soft.ssmatch = values[5]; 2175 }
2176 config.smatcher_soft.try_lazy = values[6]; 2176
2177 config.smatcher_soft.max_lazy = values[7]; 2177 config.smatcher_soft.large_look = values[0];
2178 config.smatcher_soft.long_enough = values[8]; 2178 config.smatcher_soft.large_step = values[1];
2179 config.smatcher_soft.promote = values[9]; 2179 config.smatcher_soft.small_look = values[2];
2180 } 2180 config.smatcher_soft.small_chain = values[3];
2181 else if (option_level < 5) { config.smatch_cfg = XD3_SMATCH_FAST; } 2181 config.smatcher_soft.small_lchain = values[4];
2182 else { config.smatch_cfg = XD3_SMATCH_SLOW; } 2182 config.smatcher_soft.ssmatch = values[5];
2183 break; 2183 config.smatcher_soft.try_lazy = values[6];
2184#endif 2184 config.smatcher_soft.max_lazy = values[7];
2185 case CMD_DECODE: 2185 config.smatcher_soft.long_enough = values[8];
2186 if (option_use_checksum == 0) { stream_flags |= XD3_ADLER32_NOVER; } 2186 config.smatcher_soft.promote = values[9];
2187 stream_flags = 0; 2187 }
2188 ifile->flags |= RD_NONEXTERNAL; 2188 else if (option_level < 5) { config.smatch_cfg = XD3_SMATCH_FAST; }
2189 input_func = xd3_decode_input; 2189 else { config.smatch_cfg = XD3_SMATCH_SLOW; }
2190 output_func = main_write_output; 2190 break;
2191 break; 2191#endif
2192 default: 2192 case CMD_DECODE:
2193 XPR(NT "internal error\n"); 2193 if (option_use_checksum == 0) { stream_flags |= XD3_ADLER32_NOVER; }
2194 return EXIT_FAILURE; 2194 stream_flags = 0;
2195 } 2195 ifile->flags |= RD_NONEXTERNAL;
2196 2196 input_func = xd3_decode_input;
2197 start_time = get_millisecs_now (); 2197 output_func = main_write_output;
2198 2198 break;
2199 if (main_file_stat (ifile, & input_size, 0) == 0) 2199 default:
2200 { 2200 XPR(NT "internal error\n");
2201 option_winsize = min (input_size, (xoff_t) option_winsize); 2201 return EXIT_FAILURE;
2202 } 2202 }
2203 2203
2204 option_srcwinsz = max(option_srcwinsz, XD3_ALLOCSIZE); 2204 start_time = get_millisecs_now ();
2205 option_winsize = max(option_winsize, XD3_ALLOCSIZE); 2205
2206 2206 if (main_file_stat (ifile, & input_size, 0) == 0)
2207 source.blksize = (option_srcwinsz / 32) & ~(XD3_ALLOCSIZE-1); 2207 {
2208 source.blksize = max(XD3_DEFAULT_WINSIZE, source.blksize); 2208 // Note: to avoid unnecessary allocs for small files: problem
2209 2209 // was with --decompress_inputs, we don't know the true size
2210 config.srcwin_maxsz = option_srcwinsz; 2210 // option_winsize = min (input_size, (xoff_t) option_winsize);
2211 config.winsize = option_winsize; 2211 }
2212 config.memsize = option_memsize; 2212
2213 config.getblk = main_getblk_func; 2213 option_srcwinsz = max(option_srcwinsz, XD3_ALLOCSIZE);
2214 config.flags = stream_flags; 2214 option_winsize = max(option_winsize, XD3_ALLOCSIZE);
2215 2215
2216 if (option_verbose > 1) 2216 source.blksize = (option_srcwinsz / 32) & ~(XD3_ALLOCSIZE-1);
2217 { 2217 source.blksize = max(XD3_DEFAULT_WINSIZE, source.blksize);
2218 XPR(NT "input buffer size: %u\n", option_winsize); 2218
2219 } 2219 config.srcwin_maxsz = option_srcwinsz;
2220 2220 config.winsize = option_winsize;
2221 if ((main_bdata = main_malloc (option_winsize)) == NULL) 2221 config.memsize = option_memsize;
2222 { 2222 config.getblk = main_getblk_func;
2223 return EXIT_FAILURE; 2223 config.flags = stream_flags;
2224 } 2224
2225 2225 if (option_verbose > 1)
2226 if ((ret = xd3_config_stream (& stream, & config))) 2226 {
2227 { 2227 XPR(NT "input buffer size: %u\n", option_winsize);
2228 XPR(NT XD3_LIB_ERRMSG (& stream, ret)); 2228 }
2229 return EXIT_FAILURE; 2229
2230 } 2230 if ((main_bdata = main_malloc (option_winsize)) == NULL)
2231 2231 {
2232 if (IS_ENCODE (cmd)) 2232 return EXIT_FAILURE;
2233 { 2233 }
2234 /* When encoding, open the source file, possibly decompress it. The decoder delays 2234
2235 * this step until XD3_GOTHEADER. */ 2235 if ((ret = xd3_config_stream (& stream, & config)))
2236 if (sfile->filename != NULL && (ret = main_set_source (& stream, cmd, sfile, & source))) 2236 {
2237 { 2237 XPR(NT XD3_LIB_ERRMSG (& stream, ret));
2238 return EXIT_FAILURE; 2238 return EXIT_FAILURE;
2239 } 2239 }
2240 } 2240
2241 2241 if (IS_ENCODE (cmd))
2242 /* This times each window. */ 2242 {
2243 get_millisecs_since (); 2243 /* When encoding, open the source file, possibly decompress it. The decoder delays
2244 2244 * this step until XD3_GOTHEADER. */
2245 /* Main input loop. */ 2245 if (sfile->filename != NULL && (ret = main_set_source (& stream, cmd, sfile, & source)))
2246 do 2246 {
2247 { 2247 return EXIT_FAILURE;
2248 xoff_t input_offset; 2248 }
2249 xoff_t input_remain; 2249 }
2250 usize_t try_read; 2250
2251 2251 /* This times each window. */
2252 input_offset = ifile->nread; 2252 get_millisecs_since ();
2253 /*XD3_ASSERT (input_offset <= option_last_offset);*/ 2253
2254 2254 /* Main input loop. */
2255 input_remain = /*option_last_offset*/ XOFF_T_MAX - input_offset; 2255 do
2256 2256 {
2257 try_read = (usize_t) min ((xoff_t) config.winsize, input_remain); 2257 xoff_t input_offset;
2258 2258 xoff_t input_remain;
2259 if ((ret = main_read_primary_input (ifile, main_bdata, try_read, & nread))) 2259 usize_t try_read;
2260 { 2260
2261 return EXIT_FAILURE; 2261 input_offset = ifile->nread;
2262 } 2262 /*XD3_ASSERT (input_offset <= option_last_offset);*/
2263 2263
2264 /* If we've reached EOF tell the stream to flush. */ 2264 input_remain = /*option_last_offset*/ XOFF_T_MAX - input_offset;
2265 if (nread < try_read) 2265
2266 { 2266 try_read = (usize_t) min ((xoff_t) config.winsize, input_remain);
2267 stream_flags |= XD3_FLUSH; 2267
2268 xd3_set_flags (& stream, stream_flags); 2268 if ((ret = main_read_primary_input (ifile, main_bdata, try_read, & nread)))
2269 } 2269 {
2270 2270 return EXIT_FAILURE;
2271#if XD3_ENCODER 2271 }
2272 /* After the first main_read_primary_input completes, we know all the information 2272
2273 * needed to encode the application header. */ 2273 /* If we've reached EOF tell the stream to flush. */
2274 if (cmd == CMD_ENCODE && (ret = main_set_appheader (& stream, ifile, sfile))) 2274 if (nread < try_read)
2275 { 2275 {
2276 return EXIT_FAILURE; 2276 stream_flags |= XD3_FLUSH;
2277 } 2277 xd3_set_flags (& stream, stream_flags);
2278#endif 2278 }
2279 xd3_avail_input (& stream, main_bdata, nread); 2279
2280 2280#if XD3_ENCODER
2281 /* If we read zero bytes after encoding at least one window... */ 2281 /* After the first main_read_primary_input completes, we know all the information
2282 if (nread == 0 && stream.current_window > 0) { 2282 * needed to encode the application header. */
2283 break; 2283 if (cmd == CMD_ENCODE && (ret = main_set_appheader (& stream, ifile, sfile)))
2284 } 2284 {
2285 2285 return EXIT_FAILURE;
2286 again: 2286 }
2287 ret = input_func (& stream); 2287#endif
2288 /*if (option_verbose > 1) { XPR(NT XD3_LIB_ERRMSG (& stream, ret)); }*/ 2288 xd3_avail_input (& stream, main_bdata, nread);
2289 2289
2290 switch (ret) 2290 /* If we read zero bytes after encoding at least one window... */
2291 { 2291 if (nread == 0 && stream.current_window > 0) {
2292 case XD3_INPUT: 2292 break;
2293 continue; 2293 }
2294 2294
2295 case XD3_GOTHEADER: 2295 again:
2296 { 2296 ret = input_func (& stream);
2297 XD3_ASSERT (stream.current_window == 0); 2297 /*if (option_verbose > 1) { XPR(NT XD3_LIB_ERRMSG (& stream, ret)); }*/
2298 2298
2299 /* Need to process the appheader as soon as possible. It may contain a 2299 switch (ret)
2300 * suggested default filename/decompression routine for the ofile, and it may 2300 {
2301 * contain default/decompression routine for the sources. */ 2301 case XD3_INPUT:
2302 if (cmd == CMD_DECODE) 2302 continue;
2303 { 2303
2304 int have_src = sfile->filename != NULL; 2304 case XD3_GOTHEADER:
2305 int need_src = xd3_decoder_needs_source (& stream); 2305 {
2306 int recv_src; 2306 XD3_ASSERT (stream.current_window == 0);
2307 2307
2308 /* May need to set the sfile->filename if none was given. */ 2308 /* Need to process the appheader as soon as possible. It may contain a
2309 main_get_appheader (& stream, ifile, ofile, sfile); 2309 * suggested default filename/decompression routine for the ofile, and it may
2310 2310 * contain default/decompression routine for the sources. */
2311 recv_src = sfile->filename != NULL; 2311 if (cmd == CMD_DECODE)
2312 2312 {
2313 /* Check if the user expected a source to be required although it was not. */ 2313 int have_src = sfile->filename != NULL;
2314 if (have_src && ! need_src && ! option_quiet) 2314 int need_src = xd3_decoder_needs_source (& stream);
2315 { 2315 int recv_src;
2316 XPR(NT "warning: output window %"Q"u does not copy source\n", stream.current_window); 2316
2317 } 2317 /* May need to set the sfile->filename if none was given. */
2318 2318 main_get_appheader (& stream, ifile, ofile, sfile);
2319 /* Check if we have no source name and need one. */ 2319
2320 /* TODO: this doesn't fire due to cpyblocks_ calculation check */ 2320 recv_src = sfile->filename != NULL;
2321 if (need_src && ! recv_src) 2321
2322 { 2322 /* Check if the user expected a source to be required although it was not. */
2323 XPR(NT "input requires a source file, use -s\n"); 2323 if (have_src && ! need_src && ! option_quiet)
2324 return EXIT_FAILURE; 2324 {
2325 } 2325 XPR(NT "warning: output window %"Q"u does not copy source\n", stream.current_window);
2326 2326 }
2327 /* Now open the source file. */ 2327
2328 if (need_src && (ret = main_set_source (& stream, cmd, sfile, & source))) 2328 /* Check if we have no source name and need one. */
2329 { 2329 /* TODO: this doesn't fire due to cpyblocks_ calculation check */
2330 return EXIT_FAILURE; 2330 if (need_src && ! recv_src)
2331 } 2331 {
2332 } 2332 XPR(NT "input requires a source file, use -s\n");
2333 else if (cmd == CMD_PRINTHDR || 2333 return EXIT_FAILURE;
2334 cmd == CMD_PRINTHDRS || 2334 }
2335 cmd == CMD_PRINTDELTA) 2335
2336 { 2336 /* Now open the source file. */
2337 if (xd3_decoder_needs_source (& stream) && sfile->filename == NULL) 2337 if (need_src && (ret = main_set_source (& stream, cmd, sfile, & source)))
2338 { 2338 {
2339 allow_fake_source = 1; 2339 return EXIT_FAILURE;
2340 sfile->filename = "<placeholder>"; 2340 }
2341 main_set_source (& stream, cmd, sfile, & source); 2341 }
2342 } 2342 else if (cmd == CMD_PRINTHDR ||
2343 } 2343 cmd == CMD_PRINTHDRS ||
2344 } 2344 cmd == CMD_PRINTDELTA)
2345 /* FALLTHROUGH */ 2345 {
2346 case XD3_WINSTART: 2346 if (xd3_decoder_needs_source (& stream) && sfile->filename == NULL)
2347 { 2347 {
2348 /* Set or unset XD3_SKIP_WINDOW. */ 2348 allow_fake_source = 1;
2349 /*if (stream.current_window < option_first_window || stream.current_window > option_last_window) 2349 sfile->filename = "<placeholder>";
2350 { stream_flags |= XD3_SKIP_WINDOW; } 2350 main_set_source (& stream, cmd, sfile, & source);
2351 else 2351 }
2352 { stream_flags &= ~XD3_SKIP_WINDOW; }*/ 2352 }
2353 2353 }
2354 xd3_set_flags (& stream, stream_flags); 2354 /* FALLTHROUGH */
2355 goto again; 2355 case XD3_WINSTART:
2356 } 2356 {
2357 2357 /* Set or unset XD3_SKIP_WINDOW. */
2358 case XD3_OUTPUT: 2358 /*if (stream.current_window < option_first_window || stream.current_window > option_last_window)
2359 { 2359 { stream_flags |= XD3_SKIP_WINDOW; }
2360 if (option_no_output == 0/* && 2360 else
2361 stream.current_window >= option_first_window && 2361 { stream_flags &= ~XD3_SKIP_WINDOW; }*/
2362 stream.current_window <= option_last_window*/) 2362
2363 { 2363 xd3_set_flags (& stream, stream_flags);
2364 /* Defer opening the output file until the stream produces its first 2364 goto again;
2365 * output for both encoder and decoder, this way we delay long enough for 2365 }
2366 * the decoder to receive the application header. (Or longer if there are 2366
2367 * skipped windows, but I can't think of any reason not to delay open.) */ 2367 case XD3_OUTPUT:
2368 2368 {
2369 if (! main_file_isopen (ofile) && (ret = main_open_output (& stream, ofile)) != 0) 2369 if (option_no_output == 0/* &&
2370 { 2370 stream.current_window >= option_first_window &&
2371 return EXIT_FAILURE; 2371 stream.current_window <= option_last_window*/)
2372 } 2372 {
2373 if ((ret = output_func (& stream, ofile)) && (ret != PRINTHDR_SPECIAL)) 2373 /* Defer opening the output file until the stream produces its first
2374 { 2374 * output for both encoder and decoder, this way we delay long enough for
2375 return EXIT_FAILURE; 2375 * the decoder to receive the application header. (Or longer if there are
2376 } 2376 * skipped windows, but I can't think of any reason not to delay open.) */
2377 if (ret == PRINTHDR_SPECIAL) 2377
2378 { 2378 if (! main_file_isopen (ofile) && (ret = main_open_output (& stream, ofile)) != 0)
2379 xd3_abort_stream (& stream); 2379 {
2380 ret = EXIT_SUCCESS; 2380 return EXIT_FAILURE;
2381 goto done; 2381 }
2382 } 2382 if ((ret = output_func (& stream, ofile)) && (ret != PRINTHDR_SPECIAL))
2383 ret = 0; 2383 {
2384 } 2384 return EXIT_FAILURE;
2385 2385 }
2386 xd3_consume_output (& stream); 2386 if (ret == PRINTHDR_SPECIAL)
2387 goto again; 2387 {
2388 } 2388 xd3_abort_stream (& stream);
2389 2389 ret = EXIT_SUCCESS;
2390 case XD3_WINFINISH: 2390 goto done;
2391 { 2391 }
2392 if (IS_ENCODE (cmd) || cmd == CMD_DECODE) 2392 ret = 0;
2393 { 2393 }
2394 int used_source = xd3_encoder_used_source (& stream); 2394
2395 2395 xd3_consume_output (& stream);
2396 if (! option_quiet && IS_ENCODE (cmd) && main_file_isopen (sfile) && ! used_source) 2396 goto again;
2397 { 2397 }
2398 XPR(NT "warning: input position %"Q"u no source copies\n", 2398
2399 stream.current_window * option_winsize); 2399 case XD3_WINFINISH:
2400 } 2400 {
2401 2401 if (IS_ENCODE (cmd) || cmd == CMD_DECODE)
2402 if (option_verbose) 2402 {
2403 { 2403 int used_source = xd3_encoder_used_source (& stream);
2404 char rrateavg[32], wrateavg[32], tm[32]; 2404
2405 char rdb[32], wdb[32]; 2405 if (! option_quiet && IS_ENCODE (cmd) && main_file_isopen (sfile) && ! used_source)
2406 char trdb[32], twdb[32]; 2406 {
2407 long millis = get_millisecs_since (); 2407 XPR(NT "warning: input position %"Q"u no source copies\n",
2408 usize_t this_read = stream.total_in - last_total_in; 2408 stream.current_window * option_winsize);
2409 usize_t this_write = stream.total_out - last_total_out; 2409 }
2410 last_total_in = stream.total_in; 2410
2411 last_total_out = stream.total_out; 2411 if (option_verbose)
2412 2412 {
2413 if (option_verbose > 1) 2413 char rrateavg[32], wrateavg[32], tm[32];
2414 { 2414 char rdb[32], wdb[32];
2415 XPR(NT "%"Q"u: in %s (%s): out %s (%s): total in %s: out %s: %s\n", 2415 char trdb[32], twdb[32];
2416 stream.current_window, 2416 long millis = get_millisecs_since ();
2417 main_format_bcnt (this_read, rdb), 2417 usize_t this_read = stream.total_in - last_total_in;
2418 main_format_rate (this_read, millis, rrateavg), 2418 usize_t this_write = stream.total_out - last_total_out;
2419 main_format_bcnt (this_write, wdb), 2419 last_total_in = stream.total_in;
2420 main_format_rate (this_write, millis, wrateavg), 2420 last_total_out = stream.total_out;
2421 main_format_bcnt (stream.total_in, trdb), 2421
2422 main_format_bcnt (stream.total_out, twdb), 2422 if (option_verbose > 1)
2423 main_format_millis (millis, tm)); 2423 {
2424 } 2424 XPR(NT "%"Q"u: in %s (%s): out %s (%s): total in %s: out %s: %s\n",
2425 else 2425 stream.current_window,
2426 { 2426 main_format_bcnt (this_read, rdb),
2427 XPR(NT "%"Q"u: in %s: out %s: total in %s: out %s: %s\n", 2427 main_format_rate (this_read, millis, rrateavg),
2428 stream.current_window, 2428 main_format_bcnt (this_write, wdb),
2429 main_format_bcnt (this_read, rdb), 2429 main_format_rate (this_write, millis, wrateavg),
2430 main_format_bcnt (this_write, wdb), 2430 main_format_bcnt (stream.total_in, trdb),
2431 main_format_bcnt (stream.total_in, trdb), 2431 main_format_bcnt (stream.total_out, twdb),
2432 main_format_bcnt (stream.total_out, twdb), 2432 main_format_millis (millis, tm));
2433 main_format_millis (millis, tm)); 2433 }
2434 } 2434 else
2435 } 2435 {
2436 } 2436 XPR(NT "%"Q"u: in %s: out %s: total in %s: out %s: %s\n",
2437 goto again; 2437 stream.current_window,
2438 } 2438 main_format_bcnt (this_read, rdb),
2439 2439 main_format_bcnt (this_write, wdb),
2440 default: 2440 main_format_bcnt (stream.total_in, trdb),
2441 /* input_func() error */ 2441 main_format_bcnt (stream.total_out, twdb),
2442 XPR(NT XD3_LIB_ERRMSG (& stream, ret)); 2442 main_format_millis (millis, tm));
2443 return EXIT_FAILURE; 2443 }
2444 } 2444 }
2445 } 2445 }
2446 while (nread == config.winsize); 2446 goto again;
2447done: 2447 }
2448 /* Close the inputs. (ifile must be open, sfile may be open) */ 2448
2449 main_file_close (ifile); 2449 default:
2450 main_file_close (sfile); 2450 /* input_func() error */
2451 2451 XPR(NT XD3_LIB_ERRMSG (& stream, ret));
2452#if EXTERNAL_COMPRESSION 2452 return EXIT_FAILURE;
2453 if ((ret = main_external_compression_finish ())) 2453 }
2454 { 2454 }
2455 XPR(NT "external compression commands failed\n"); 2455 while (nread == config.winsize);
2456 return EXIT_FAILURE; 2456done:
2457 } 2457 /* Close the inputs. (ifile must be open, sfile may be open) */
2458#endif 2458 main_file_close (ifile);
2459 2459 main_file_close (sfile);
2460 /* If output file is not open yet because of delayed-open, it means we never encountered 2460
2461 * a window in the delta, but it could have had a VCDIFF header? TODO: solve this 2461#if EXTERNAL_COMPRESSION
2462 * elsewhere. For now, it prints "nothing to output" below, but the check doesn't 2462 if ((ret = main_external_compression_finish ()))
2463 * happen in case of option_no_output. */ 2463 {
2464 if (! option_no_output) 2464 XPR(NT "external compression commands failed\n");
2465 { 2465 return EXIT_FAILURE;
2466 if (! main_file_isopen (ofile)) 2466 }
2467 { 2467#endif
2468 XPR(NT "nothing to output: %s\n", ifile->filename); 2468
2469 return EXIT_FAILURE; 2469 /* If output file is not open yet because of delayed-open, it means we never encountered
2470 } 2470 * a window in the delta, but it could have had a VCDIFF header? TODO: solve this
2471 2471 * elsewhere. For now, it prints "nothing to output" below, but the check doesn't
2472 /* Have to close the output before calling main_external_compression_finish, or else it hangs. */ 2472 * happen in case of option_no_output. */
2473 if (main_file_close (ofile) != 0) 2473 if (! option_no_output)
2474 { 2474 {
2475 return EXIT_FAILURE; 2475 if (! main_file_isopen (ofile))
2476 } 2476 {
2477 } 2477 XPR(NT "nothing to output: %s\n", ifile->filename);
2478 2478 return EXIT_FAILURE;
2479 if ((ret = xd3_close_stream (& stream))) 2479 }
2480 { 2480
2481 XPR(NT XD3_LIB_ERRMSG (& stream, ret)); 2481 /* Have to close the output before calling main_external_compression_finish, or else it hangs. */
2482 return EXIT_FAILURE; 2482 if (main_file_close (ofile) != 0)
2483 } 2483 {
2484 2484 return EXIT_FAILURE;
2485 xd3_free_stream (& stream); 2485 }
2486 2486 }
2487 if (option_verbose) 2487
2488 { 2488 if ((ret = xd3_close_stream (& stream)))
2489 char tm[32]; 2489 {
2490 long end_time = get_millisecs_now (); 2490 XPR(NT XD3_LIB_ERRMSG (& stream, ret));
2491 XPR(NT "command finished in %s\n", main_format_millis (end_time - start_time, tm)); 2491 return EXIT_FAILURE;
2492 XPR(NT "input bytes: %"Q"u\n", ifile->nread); 2492 }
2493 XPR(NT "output bytes: %"Q"u\n", ofile->nwrite); 2493
2494 } 2494 xd3_free_stream (& stream);
2495 2495
2496 return EXIT_SUCCESS; 2496 if (option_verbose)
2497} 2497 {
2498 2498 char tm[32];
2499/* free memory before exit, reset single-use variables. */ 2499 long end_time = get_millisecs_now ();
2500static void 2500 XPR(NT "command finished in %s\n", main_format_millis (end_time - start_time, tm));
2501main_cleanup (void) 2501 XPR(NT "input bytes: %"Q"u\n", ifile->nread);
2502{ 2502 XPR(NT "output bytes: %"Q"u\n", ofile->nwrite);
2503 int i; 2503 }
2504 2504
2505 if (option_appheader) { appheader_used = NULL; } 2505 return EXIT_SUCCESS;
2506 2506}
2507 main_free (appheader_used); 2507
2508 main_free (main_bdata); 2508/* free memory before exit, reset single-use variables. */
2509 2509static void
2510#if EXTERNAL_COMPRESSION 2510main_cleanup (void)
2511 main_free (ext_tmpfile); 2511{
2512#endif 2512 int i;
2513 2513
2514 for (i = 0; lru && i < lru_size; i += 1) 2514 if (option_appheader) { appheader_used = NULL; }
2515 { 2515
2516 main_free (lru[i].blk); 2516 main_free (appheader_used);
2517 } 2517 main_free (main_bdata);
2518 2518
2519 main_free (lru); 2519#if EXTERNAL_COMPRESSION
2520 2520 main_free (ext_tmpfile);
2521 lru_hits = 0; 2521#endif
2522 lru_misses = 0; 2522
2523 lru_filled = 0; 2523 for (i = 0; lru && i < lru_size; i += 1)
2524 2524 {
2525 XD3_ASSERT (main_mallocs == 0); 2525 main_free (lru[i].blk);
2526} 2526 }
2527 2527
2528int 2528 main_free (lru);
2529#if PYTHON_MODULE 2529
2530xd3_main_cmdline (int argc, char **argv) 2530 lru_hits = 0;
2531#else 2531 lru_misses = 0;
2532main (int argc, char **argv) 2532 lru_filled = 0;
2533#endif 2533
2534{ 2534 XD3_ASSERT (main_mallocs == 0);
2535 xd3_cmd cmd; 2535}
2536 main_file ifile; 2536
2537 main_file ofile; 2537int
2538 main_file sfile; 2538#if PYTHON_MODULE
2539 static char *flags = "0123456789cdefhnqvDJNORTVs:B:C:E:F:L:O:P:M:W:A::S::"; 2539xd3_main_cmdline (int argc, char **argv)
2540 int my_optind; 2540#else
2541 char *my_optarg; 2541main (int argc, char **argv)
2542 char *my_optstr; 2542#endif
2543 char *sfilename; 2543{
2544 int orig_argc = argc; 2544 xd3_cmd cmd;
2545 char **orig_argv = argv; 2545 main_file ifile;
2546 int ret; 2546 main_file ofile;
2547 2547 main_file sfile;
2548 main_file_init (& ifile); 2548 static char *flags = "0123456789cdefhnqvDJNORTVs:B:C:E:F:L:O:P:M:W:A::S::";
2549 main_file_init (& ofile); 2549 int my_optind;
2550 main_file_init (& sfile); 2550 char *my_optarg;
2551 2551 char *my_optstr;
2552 go: /* Go. */ 2552 char *sfilename;
2553 cmd = CMD_NONE; 2553 int orig_argc = argc;
2554 sfilename = NULL; 2554 char **orig_argv = argv;
2555 my_optind = 1; 2555 int ret;
2556 argv = orig_argv; 2556
2557 argc = orig_argc; 2557 main_file_init (& ifile);
2558 program_name = argv[0]; 2558 main_file_init (& ofile);
2559 extcomp_types[0].recomp_cmdname = program_name; 2559 main_file_init (& sfile);
2560 extcomp_types[0].decomp_cmdname = program_name; 2560
2561 takearg: 2561 go: /* Go. */
2562 my_optarg = NULL; 2562 cmd = CMD_NONE;
2563 my_optstr = argv[my_optind]; 2563 sfilename = NULL;
2564 /* This doesn't use getopt() because it makes trouble for -P & python which reenter 2564 my_optind = 1;
2565 * main() and thus care about freeing all memory. I never had much trust for getopt 2565 argv = orig_argv;
2566 * anyway, it's too opaque. This implements a fairly standard non-long-option getopt 2566 argc = orig_argc;
2567 * with support for named operations (e.g., "xdelta3 [encode|decode|printhdr...] < in > 2567 program_name = argv[0];
2568 * out"). I'll probably add long options at some point. */ 2568 extcomp_types[0].recomp_cmdname = program_name;
2569 if (my_optstr) 2569 extcomp_types[0].decomp_cmdname = program_name;
2570 { 2570 takearg:
2571 if (*my_optstr == '-') { my_optstr += 1; } 2571 my_optarg = NULL;
2572 else if (cmd == CMD_NONE) { goto nonflag; } 2572 my_optstr = argv[my_optind];
2573 else { my_optstr = NULL; } 2573 /* This doesn't use getopt() because it makes trouble for -P & python which reenter
2574 } 2574 * main() and thus care about freeing all memory. I never had much trust for getopt
2575 while (my_optstr) 2575 * anyway, it's too opaque. This implements a fairly standard non-long-option getopt
2576 { 2576 * with support for named operations (e.g., "xdelta3 [encode|decode|printhdr...] < in >
2577 char *s; 2577 * out"). I'll probably add long options at some point. */
2578 my_optarg = NULL; 2578 if (my_optstr)
2579 if ((ret = *my_optstr++) == 0) { my_optind += 1; goto takearg; } 2579 {
2580 2580 if (*my_optstr == '-') { my_optstr += 1; }
2581 /* Option handling: first check for one ':' following the option in flags, then 2581 else if (cmd == CMD_NONE) { goto nonflag; }
2582 * check for two. The syntax allows: 2582 else { my_optstr = NULL; }
2583 * 2583 }
2584 * 1. -Afoo defines optarg="foo" 2584 while (my_optstr)
2585 * 2. -A foo defines optarg="foo" 2585 {
2586 * 3. -A "" defines optarg="" (allows optional empty-string) 2586 char *s;
2587 * 4. -A [EOA or -moreargs] error (mandatory case) 2587 my_optarg = NULL;
2588 * 5. -A [EOA -moreargs] defines optarg=NULL (optional case) 2588 if ((ret = *my_optstr++) == 0) { my_optind += 1; goto takearg; }
2589 * 6. -A=foo defines optarg="foo" 2589
2590 * 7. -A= defines optarg="" (mandatory case) 2590 /* Option handling: first check for one ':' following the option in flags, then
2591 * 8. -A= defines optarg=NULL (optional case) 2591 * check for two. The syntax allows:
2592 * 2592 *
2593 * See tests in test_command_line_arguments(). 2593 * 1. -Afoo defines optarg="foo"
2594 */ 2594 * 2. -A foo defines optarg="foo"
2595 s = strchr (flags, ret); 2595 * 3. -A "" defines optarg="" (allows optional empty-string)
2596 if (s && s[1] && s[1] == ':') 2596 * 4. -A [EOA or -moreargs] error (mandatory case)
2597 { 2597 * 5. -A [EOA -moreargs] defines optarg=NULL (optional case)
2598 int eqcase = 0; 2598 * 6. -A=foo defines optarg="foo"
2599 int option = s[2] && s[2] == ':'; 2599 * 7. -A= defines optarg="" (mandatory case)
2600 2600 * 8. -A= defines optarg=NULL (optional case)
2601 /* Case 1, set optarg to the remaining characters. */ 2601 *
2602 my_optarg = my_optstr; 2602 * See tests in test_command_line_arguments().
2603 my_optstr = ""; 2603 */
2604 2604 s = strchr (flags, ret);
2605 /* Case 2-5 */ 2605 if (s && s[1] && s[1] == ':')
2606 if (*my_optarg == 0) 2606 {
2607 { 2607 int eqcase = 0;
2608 /* Condition 4-5 */ 2608 int option = s[2] && s[2] == ':';
2609 int have_arg = my_optind < (argc - 1) && *argv[my_optind+1] != '-'; 2609
2610 2610 /* Case 1, set optarg to the remaining characters. */
2611 if (! have_arg) 2611 my_optarg = my_optstr;
2612 { 2612 my_optstr = "";
2613 if (! option) 2613
2614 { 2614 /* Case 2-5 */
2615 /* Case 4 */ 2615 if (*my_optarg == 0)
2616 XPR(NT "-%c: requires an argument\n", ret); 2616 {
2617 ret = EXIT_FAILURE; 2617 /* Condition 4-5 */
2618 goto cleanup; 2618 int have_arg = my_optind < (argc - 1) && *argv[my_optind+1] != '-';
2619 } 2619
2620 /* Case 5. */ 2620 if (! have_arg)
2621 my_optarg = NULL; 2621 {
2622 } 2622 if (! option)
2623 else 2623 {
2624 { 2624 /* Case 4 */
2625 /* Case 2-3. */ 2625 XPR(NT "-%c: requires an argument\n", ret);
2626 my_optarg = argv[++my_optind]; 2626 ret = EXIT_FAILURE;
2627 } 2627 goto cleanup;
2628 } 2628 }
2629 /* Case 6-8. */ 2629 /* Case 5. */
2630 else if (*my_optarg == '=') 2630 my_optarg = NULL;
2631 { 2631 }
2632 /* Remove the = in all cases. */ 2632 else
2633 my_optarg += 1; 2633 {
2634 eqcase = 1; 2634 /* Case 2-3. */
2635 2635 my_optarg = argv[++my_optind];
2636 if (option && *my_optarg == 0) 2636 }
2637 { 2637 }
2638 /* Case 8. */ 2638 /* Case 6-8. */
2639 my_optarg = NULL; 2639 else if (*my_optarg == '=')
2640 } 2640 {
2641 } 2641 /* Remove the = in all cases. */
2642 } 2642 my_optarg += 1;
2643 2643 eqcase = 1;
2644 switch (ret) 2644
2645 { 2645 if (option && *my_optarg == 0)
2646 /* case: if no '-' was found, maybe check for a command name. */ 2646 {
2647 nonflag: 2647 /* Case 8. */
2648 if (strcmp (my_optstr, "decode") == 0) { cmd = CMD_DECODE; } 2648 my_optarg = NULL;
2649 else if (strcmp (my_optstr, "encode") == 0) 2649 }
2650 { 2650 }
2651#if XD3_ENCODER 2651 }
2652 cmd = CMD_ENCODE; 2652
2653#else 2653 switch (ret)
2654 XPR(NT "encoder support not compiled\n"); 2654 {
2655 return EXIT_FAILURE; 2655 /* case: if no '-' was found, maybe check for a command name. */
2656#endif 2656 nonflag:
2657 } 2657 if (strcmp (my_optstr, "decode") == 0) { cmd = CMD_DECODE; }
2658 else if (strcmp (my_optstr, "config") == 0) { cmd = CMD_CONFIG; } 2658 else if (strcmp (my_optstr, "encode") == 0)
2659#if REGRESSION_TEST 2659 {
2660 else if (strcmp (my_optstr, "test") == 0) { cmd = CMD_TEST; } 2660#if XD3_ENCODER
2661#endif 2661 cmd = CMD_ENCODE;
2662#if VCDIFF_TOOLS 2662#else
2663 else if (strcmp (my_optstr, "printhdr") == 0) { cmd = CMD_PRINTHDR; } 2663 XPR(NT "encoder support not compiled\n");
2664 else if (strcmp (my_optstr, "printhdrs") == 0) { cmd = CMD_PRINTHDRS; } 2664 return EXIT_FAILURE;
2665 else if (strcmp (my_optstr, "printdelta") == 0) { cmd = CMD_PRINTDELTA; } 2665#endif
2666#endif 2666 }
2667 2667 else if (strcmp (my_optstr, "config") == 0) { cmd = CMD_CONFIG; }
2668 /* If no option was found and still no command, let the default command be 2668#if REGRESSION_TEST
2669 * encode. The remaining args are treated as filenames. */ 2669 else if (strcmp (my_optstr, "test") == 0) { cmd = CMD_TEST; }
2670 if (cmd == CMD_NONE) 2670#endif
2671 { 2671#if VCDIFF_TOOLS
2672 cmd = CMD_DEFAULT; 2672 else if (strcmp (my_optstr, "printhdr") == 0) { cmd = CMD_PRINTHDR; }
2673 my_optstr = NULL; 2673 else if (strcmp (my_optstr, "printhdrs") == 0) { cmd = CMD_PRINTHDRS; }
2674 break; 2674 else if (strcmp (my_optstr, "printdelta") == 0) { cmd = CMD_PRINTDELTA; }
2675 } 2675#endif
2676 else 2676
2677 { 2677 /* If no option was found and still no command, let the default command be
2678 /* But if we find a command name, continue the getopt loop. */ 2678 * encode. The remaining args are treated as filenames. */
2679 my_optind += 1; 2679 if (cmd == CMD_NONE)
2680 goto takearg; 2680 {
2681 } 2681 cmd = CMD_DEFAULT;
2682 2682 my_optstr = NULL;
2683 /* gzip-like options */ 2683 break;
2684 case '0': case '1': case '2': case '3': case '4': 2684 }
2685 case '5': case '6': case '7': case '8': case '9': 2685 else
2686 option_level = ret - '0'; 2686 {
2687 break; 2687 /* But if we find a command name, continue the getopt loop. */
2688 case 'f': option_force = 1; break; 2688 my_optind += 1;
2689 case 'v': option_verbose += 1; option_quiet = 0; break; 2689 goto takearg;
2690 case 'q': option_quiet = 1; option_verbose = 0; break; 2690 }
2691 case 'c': option_stdout = 1; break; 2691
2692 case 'd': 2692 /* gzip-like options */
2693 if (cmd == CMD_NONE) { cmd = CMD_DECODE; } 2693 case '0': case '1': case '2': case '3': case '4':
2694 else { ret = main_help (); goto exit; } 2694 case '5': case '6': case '7': case '8': case '9':
2695 break; 2695 option_level = ret - '0';
2696 case 'e': 2696 break;
2697#if XD3_ENCODER 2697 case 'f': option_force = 1; break;
2698 if (cmd == CMD_NONE) { cmd = CMD_ENCODE; } 2698 case 'v': option_verbose += 1; option_quiet = 0; break;
2699 else { ret = main_help (); goto exit; } 2699 case 'q': option_quiet = 1; option_verbose = 0; break;
2700 break; 2700 case 'c': option_stdout = 1; break;
2701#else 2701 case 'd':
2702 XPR(NT "encoder support not compiled\n"); 2702 if (cmd == CMD_NONE) { cmd = CMD_DECODE; }
2703 return EXIT_FAILURE; 2703 else { ret = main_help (); goto exit; }
2704#endif 2704 break;
2705 case 'P': 2705 case 'e':
2706 /* only set profile count once, since... */ 2706#if XD3_ENCODER
2707 if (option_profile_cnt == 0) 2707 if (cmd == CMD_NONE) { cmd = CMD_ENCODE; }
2708 { 2708 else { ret = main_help (); goto exit; }
2709 if ((ret = main_atou(my_optarg, (usize_t*) & option_profile_cnt, 0, 'P'))) { goto exit; } 2709 break;
2710 2710#else
2711 if (option_profile_cnt <= 0) 2711 XPR(NT "encoder support not compiled\n");
2712 { 2712 return EXIT_FAILURE;
2713 ret = EXIT_SUCCESS; 2713#endif
2714 goto exit; 2714 case 'P':
2715 } 2715 /* only set profile count once, since... */
2716 } 2716 if (option_profile_cnt == 0)
2717 break; 2717 {
2718 2718 if ((ret = main_atou(my_optarg, (usize_t*) & option_profile_cnt, 0, 'P'))) { goto exit; }
2719 case 'n': option_use_checksum = 0; break; 2719
2720 case 'N': option_no_compress = 1; break; 2720 if (option_profile_cnt <= 0)
2721 case 'T': option_use_altcodetable = 1; break; 2721 {
2722 case 'C': option_smatch_config = my_optarg; break; 2722 ret = EXIT_SUCCESS;
2723 case 'J': option_no_output = 1; break; 2723 goto exit;
2724 case 'O': option_xdelta1 = my_optarg; break; 2724 }
2725 case 'S': if (my_optarg == NULL) { option_use_secondary = 0; } 2725 }
2726 else { option_use_secondary = 1; option_secondary = my_optarg; } break; 2726 break;
2727 case 'A': if (my_optarg == NULL) { option_use_appheader = 0; } 2727
2728 else { option_appheader = (uint8_t*) my_optarg; } break; 2728 case 'n': option_use_checksum = 0; break;
2729 case 'B': if ((ret = main_atou (my_optarg, & option_srcwinsz, XD3_ALLOCSIZE, 'B'))) 2729 case 'N': option_no_compress = 1; break;
2730 { 2730 case 'T': option_use_altcodetable = 1; break;
2731 goto exit; 2731 case 'C': option_smatch_config = my_optarg; break;
2732 } 2732 case 'J': option_no_output = 1; break;
2733 break; 2733 case 'O': option_xdelta1 = my_optarg; break;
2734 case 'W': if ((ret = main_atou (my_optarg, & option_winsize, XD3_ALLOCSIZE, 'W'))) 2734 case 'S': if (my_optarg == NULL) { option_use_secondary = 0; }
2735 { 2735 else { option_use_secondary = 1; option_secondary = my_optarg; } break;
2736 goto exit; 2736 case 'A': if (my_optarg == NULL) { option_use_appheader = 0; }
2737 } 2737 else { option_appheader = (uint8_t*) my_optarg; } break;
2738 break; 2738 case 'B': if ((ret = main_atou (my_optarg, & option_srcwinsz, XD3_ALLOCSIZE, 'B')))
2739 case 'M': if ((ret = main_atou (my_optarg, & option_memsize, XD3_ALLOCSIZE, 'M'))) 2739 {
2740 { 2740 goto exit;
2741 goto exit; 2741 }
2742 } 2742 break;
2743 break; 2743 case 'W': if ((ret = main_atou (my_optarg, & option_winsize, XD3_ALLOCSIZE, 'W')))
2744 case 'D': 2744 {
2745#if EXTERNAL_COMPRESSION == 0 2745 goto exit;
2746 if (! option_quiet) 2746 }
2747 { 2747 break;
2748 XPR(NT "warning: -D option ignored, " 2748 case 'M': if ((ret = main_atou (my_optarg, & option_memsize, XD3_ALLOCSIZE, 'M')))
2749 "external compression support was not compiled\n"); 2749 {
2750 } 2750 goto exit;
2751#else 2751 }
2752 option_decompress_inputs = 0; 2752 break;
2753#endif 2753 case 'D':
2754 break; 2754#if EXTERNAL_COMPRESSION == 0
2755 case 'R': 2755 if (! option_quiet)
2756#if EXTERNAL_COMPRESSION == 0 2756 {
2757 if (! option_quiet) 2757 XPR(NT "warning: -D option ignored, "
2758 { 2758 "external compression support was not compiled\n");
2759 XPR(NT "warning: -R option ignored, " 2759 }
2760 "external compression support was not compiled\n"); 2760#else
2761 } 2761 option_decompress_inputs = 0;
2762#else 2762#endif
2763 option_recompress_outputs = 0; 2763 break;
2764#endif 2764 case 'R':
2765 break; 2765#if EXTERNAL_COMPRESSION == 0
2766 case 's': 2766 if (! option_quiet)
2767 if (sfilename != NULL) 2767 {
2768 { 2768 XPR(NT "warning: -R option ignored, "
2769 XPR(NT "specify only one source file\n"); 2769 "external compression support was not compiled\n");
2770 goto cleanup; 2770 }
2771 } 2771#else
2772 2772 option_recompress_outputs = 0;
2773 sfilename = my_optarg; 2773#endif
2774 break; 2774 break;
2775 2775 case 's':
2776 case 'V': 2776 if (sfilename != NULL)
2777 ret = main_version (); goto exit; 2777 {
2778 default: 2778 XPR(NT "specify only one source file\n");
2779 ret = main_help (); goto exit; 2779 goto cleanup;
2780 } 2780 }
2781 } 2781
2782 2782 sfilename = my_optarg;
2783 option_source_filename = sfilename; 2783 break;
2784 2784
2785 /* In case there were no arguments, set the default command. */ 2785 case 'V':
2786 if (cmd == CMD_NONE) { cmd = CMD_DEFAULT; } 2786 ret = main_version (); goto exit;
2787 2787 default:
2788 argc -= my_optind; 2788 ret = main_help (); goto exit;
2789 argv += my_optind; 2789 }
2790 2790 }
2791 /* There may be up to two more arguments. */ 2791
2792 if (argc > 2) 2792 option_source_filename = sfilename;
2793 { 2793
2794 XPR(NT "too many filenames: %s ...\n", argv[2]); 2794 /* In case there were no arguments, set the default command. */
2795 ret = EXIT_FAILURE; 2795 if (cmd == CMD_NONE) { cmd = CMD_DEFAULT; }
2796 goto cleanup; 2796
2797 } 2797 argc -= my_optind;
2798 2798 argv += my_optind;
2799 extcomp_types[1].recomp_cmdname = option_xdelta1; 2799
2800 extcomp_types[1].decomp_cmdname = option_xdelta1; 2800 /* There may be up to two more arguments. */
2801 2801 if (argc > 2)
2802 if (option_verbose > 1) 2802 {
2803 { 2803 XPR(NT "too many filenames: %s ...\n", argv[2]);
2804 int l = 1; 2804 ret = EXIT_FAILURE;
2805 int i; 2805 goto cleanup;
2806 char buf[1024]; 2806 }
2807 for (i = 0; i < orig_argc; i += 1) 2807
2808 { 2808 extcomp_types[1].recomp_cmdname = option_xdelta1;
2809 l += strlen (orig_argv[i]) + 1; 2809 extcomp_types[1].decomp_cmdname = option_xdelta1;
2810 } 2810
2811 buf[0] = 0; 2811 if (option_verbose > 1)
2812 for (i = 0; i < orig_argc; i += 1) 2812 {
2813 { 2813 int l = 1;
2814 strcat (buf, orig_argv[i]); 2814 int i;
2815 strcat (buf, " "); 2815 char buf[1024];
2816 } 2816 for (i = 0; i < orig_argc; i += 1)
2817 XPR(NT "command line: %s\n", buf); 2817 {
2818 } 2818 l += strlen (orig_argv[i]) + 1;
2819 2819 }
2820 ifile.flags = RD_FIRST; 2820 buf[0] = 0;
2821 sfile.flags = RD_FIRST; 2821 for (i = 0; i < orig_argc; i += 1)
2822 sfile.filename = option_source_filename; 2822 {
2823 2823 strcat (buf, orig_argv[i]);
2824 /* The infile takes the next argument, if there is one. But if not, infile is set to 2824 strcat (buf, " ");
2825 * stdin. */ 2825 }
2826 if (argc > 0) 2826 XPR(NT "command line: %s\n", buf);
2827 { 2827 }
2828 ifile.filename = argv[0]; 2828
2829 2829 ifile.flags = RD_FIRST;
2830 if ((ret = main_file_open (& ifile, ifile.filename, XO_READ))) 2830 sfile.flags = RD_FIRST;
2831 { 2831 sfile.filename = option_source_filename;
2832 goto cleanup; 2832
2833 } 2833 /* The infile takes the next argument, if there is one. But if not, infile is set to
2834 } 2834 * stdin. */
2835 else 2835 if (argc > 0)
2836 { 2836 {
2837 XSTDIN_XF (& ifile); 2837 ifile.filename = argv[0];
2838 } 2838
2839 2839 if ((ret = main_file_open (& ifile, ifile.filename, XO_READ)))
2840 /* The ofile takes the following argument, if there is one. But if not, it is left NULL 2840 {
2841 * until the application header is processed. It will be set in main_open_output. */ 2841 goto cleanup;
2842 if (argc > 1) 2842 }
2843 { 2843 }
2844 /* Check for conflicting arguments. */ 2844 else
2845 if (option_stdout && ! option_quiet) 2845 {
2846 { 2846 XSTDIN_XF (& ifile);
2847 XPR(NT "warning: -c option overrides output filename: %s\n", argv[1]); 2847 }
2848 } 2848
2849 2849 /* The ofile takes the following argument, if there is one. But if not, it is left NULL
2850 if (! option_stdout) { ofile.filename = argv[1]; } 2850 * until the application header is processed. It will be set in main_open_output. */
2851 } 2851 if (argc > 1)
2852 2852 {
2853 switch (cmd) 2853 /* Check for conflicting arguments. */
2854 { 2854 if (option_stdout && ! option_quiet)
2855 case CMD_PRINTHDR: 2855 {
2856 case CMD_PRINTHDRS: 2856 XPR(NT "warning: -c option overrides output filename: %s\n", argv[1]);
2857 case CMD_PRINTDELTA: 2857 }
2858#if XD3_ENCODER 2858
2859 case CMD_ENCODE: 2859 if (! option_stdout) { ofile.filename = argv[1]; }
2860 if (cmd == CMD_ENCODE) 2860 }
2861 { 2861
2862 do_not_lru = 1; 2862 switch (cmd)
2863 } 2863 {
2864#endif 2864 case CMD_PRINTHDR:
2865 case CMD_DECODE: 2865 case CMD_PRINTHDRS:
2866 ret = main_input (cmd, & ifile, & ofile, & sfile); 2866 case CMD_PRINTDELTA:
2867 break; 2867#if XD3_ENCODER
2868 2868 case CMD_ENCODE:
2869#if REGRESSION_TEST 2869 if (cmd == CMD_ENCODE)
2870 case CMD_TEST: 2870 {
2871 ret = xd3_selftest (); 2871 do_not_lru = 1;
2872 break; 2872 }
2873#endif 2873#endif
2874 2874 case CMD_DECODE:
2875 case CMD_CONFIG: 2875 ret = main_input (cmd, & ifile, & ofile, & sfile);
2876 ret = main_config (); 2876 break;
2877 break; 2877
2878 2878#if REGRESSION_TEST
2879 default: 2879 case CMD_TEST:
2880 ret = main_help (); 2880 ret = xd3_selftest ();
2881 break; 2881 break;
2882 } 2882#endif
2883 2883
2884#if EXTERNAL_COMPRESSION 2884 case CMD_CONFIG:
2885 if (ext_tmpfile != NULL) { unlink (ext_tmpfile); } 2885 ret = main_config ();
2886#endif 2886 break;
2887 2887
2888 if (0) 2888 default:
2889 { 2889 ret = main_help ();
2890 cleanup: 2890 break;
2891 ret = EXIT_FAILURE; 2891 }
2892 exit: 2892
2893 (void)0; 2893#if EXTERNAL_COMPRESSION
2894 } 2894 if (ext_tmpfile != NULL) { unlink (ext_tmpfile); }
2895 2895#endif
2896 main_file_cleanup (& ifile); 2896
2897 main_file_cleanup (& ofile); 2897 if (0)
2898 main_file_cleanup (& sfile); 2898 {
2899 2899 cleanup:
2900 main_cleanup (); 2900 ret = EXIT_FAILURE;
2901 2901 exit:
2902 if (--option_profile_cnt > 0 && ret == EXIT_SUCCESS) { goto go; } 2902 (void)0;
2903 2903 }
2904 return ret; 2904
2905} 2905 main_file_cleanup (& ifile);
2906 2906 main_file_cleanup (& ofile);
2907static int 2907 main_file_cleanup (& sfile);
2908main_help (void) 2908
2909{ 2909 main_cleanup ();
2910 /* TODO: update www/xdelta3-cmdline.html */ 2910
2911 main_version (); 2911 if (--option_profile_cnt > 0 && ret == EXIT_SUCCESS) { goto go; }
2912 P(RINT "usage: xdelta3 [command/options] [input [output]]\n"); 2912
2913 P(RINT "special command names:\n"); 2913 return ret;
2914 P(RINT " config prints xdelta3 configuration\n"); 2914}
2915 P(RINT " decode decompress the input\n"); 2915
2916 P(RINT " encode compress the input%s\n", XD3_ENCODER ? "" : " [Not compiled]"); 2916static int
2917#if REGRESSION_TEST 2917main_help (void)
2918 P(RINT " test run the builtin tests\n"); 2918{
2919#endif 2919 /* TODO: update www/xdelta3-cmdline.html */
2920#if VCDIFF_TOOLS 2920 main_version ();
2921 P(RINT "special commands for VCDIFF inputs:\n"); 2921 P(RINT "usage: xdelta3 [command/options] [input [output]]\n");
2922 P(RINT " printdelta print information about the entire delta\n"); 2922 P(RINT "special command names:\n");
2923 P(RINT " printhdr print information about the first window\n"); 2923 P(RINT " config prints xdelta3 configuration\n");
2924 P(RINT " printhdrs print information about all windows\n"); 2924 P(RINT " decode decompress the input\n");
2925#endif 2925 P(RINT " encode compress the input%s\n", XD3_ENCODER ? "" : " [Not compiled]");
2926 P(RINT "standard options:\n"); 2926#if REGRESSION_TEST
2927 P(RINT " -0 .. -9 compression level\n"); 2927 P(RINT " test run the builtin tests\n");
2928 P(RINT " -c use stdout\n"); 2928#endif
2929 P(RINT " -d decompress\n"); 2929#if VCDIFF_TOOLS
2930 P(RINT " -e compress%s\n", XD3_ENCODER ? "" : " [Not compiled]"); 2930 P(RINT "special commands for VCDIFF inputs:\n");
2931 P(RINT " -f force overwrite\n"); 2931 P(RINT " printdelta print information about the entire delta\n");
2932 P(RINT " -h show help\n"); 2932 P(RINT " printhdr print information about the first window\n");
2933 P(RINT " -q be quiet\n"); 2933 P(RINT " printhdrs print information about all windows\n");
2934 P(RINT " -v be verbose (max 2)\n"); 2934#endif
2935 P(RINT " -V show version\n"); 2935 P(RINT "standard options:\n");
2936 2936 P(RINT " -0 .. -9 compression level\n");
2937 P(RINT "memory options:\n"); 2937 P(RINT " -c use stdout\n");
2938 P(RINT " -B blksize source file block size\n"); 2938 P(RINT " -d decompress\n");
2939 P(RINT " -M memsize memory budget for hash tables\n"); 2939 P(RINT " -e compress%s\n", XD3_ENCODER ? "" : " [Not compiled]");
2940 P(RINT " -W winsize input window buffer size\n"); 2940 P(RINT " -f force overwrite\n");
2941 2941 P(RINT " -h show help\n");
2942 P(RINT "compression options:\n"); 2942 P(RINT " -q be quiet\n");
2943 P(RINT " -s source source file to copy from (if any)\n"); 2943 P(RINT " -v be verbose (max 2)\n");
2944 P(RINT " -S [djw|fgk] enable/disable secondary compression\n"); 2944 P(RINT " -V show version\n");
2945 P(RINT " -N disable small string-matching compression\n"); 2945
2946 P(RINT " -D disable external decompression (encode/decode)\n"); 2946 P(RINT "memory options:\n");
2947 P(RINT " -R disable external recompression (decode)\n"); 2947 P(RINT " -B blksize source file block size\n");
2948 2948 P(RINT " -M memsize memory budget for hash tables\n");
2949#if XD3_DEBUG > 0 2949 P(RINT " -W winsize input window buffer size\n");
2950 P(RINT "developer options:\n"); 2950
2951 P(RINT " -A [apphead] disable/provide application header\n"); 2951 P(RINT "compression options:\n");
2952 P(RINT " -C soft config (see xdelta3-cfgs.h)\n"); 2952 P(RINT " -s source source file to copy from (if any)\n");
2953 P(RINT " -J disable output (check/compute only)\n"); 2953 P(RINT " -S [djw|fgk] enable/disable secondary compression\n");
2954 P(RINT " -P repeat count (for profiling)\n"); 2954 P(RINT " -N disable small string-matching compression\n");
2955 P(RINT " -T use alternate code table\n"); 2955 P(RINT " -D disable external decompression (encode/decode)\n");
2956 P(RINT " -n disable checksum (encode/decode)\n"); 2956 P(RINT " -R disable external recompression (decode)\n");
2957#endif 2957
2958 return EXIT_FAILURE; 2958#if XD3_DEBUG > 0
2959} 2959 P(RINT "developer options:\n");
2960 P(RINT " -A [apphead] disable/provide application header\n");
2961 P(RINT " -C soft config (see xdelta3-cfgs.h)\n");
2962 P(RINT " -J disable output (check/compute only)\n");
2963 P(RINT " -P repeat count (for profiling)\n");
2964 P(RINT " -T use alternate code table\n");
2965 P(RINT " -n disable checksum (encode/decode)\n");
2966#endif
2967 return EXIT_FAILURE;
2968}
diff --git a/xdelta3/xdelta3-test.h b/xdelta3/xdelta3-test.h
index 30d8629..c3cbd4d 100755
--- a/xdelta3/xdelta3-test.h
+++ b/xdelta3/xdelta3-test.h
@@ -32,7 +32,7 @@ static const usize_t TEST_FILE_MEAN = 16384;
32static const double TEST_ADD_MEAN = 16; 32static const double TEST_ADD_MEAN = 16;
33static const double TEST_ADD_MAX = 256; 33static const double TEST_ADD_MAX = 256;
34static const double TEST_ADD_RATIO = 0.1; 34static const double TEST_ADD_RATIO = 0.1;
35static const double TEST_EPSILON = 0.5; 35static const double TEST_EPSILON = 0.55;
36 36
37#define TESTBUFSIZE (1024 * 16) 37#define TESTBUFSIZE (1024 * 16)
38 38
@@ -1522,8 +1522,8 @@ test_command_line_arguments (xd3_stream *stream, int ignore)
1522 /* Check that it is not too small, not too large. */ 1522 /* Check that it is not too small, not too large. */
1523 if (ratio >= TEST_ADD_RATIO + TEST_EPSILON) 1523 if (ratio >= TEST_ADD_RATIO + TEST_EPSILON)
1524 { 1524 {
1525 P(RINT "xdelta3: test encode with size ratio %.3f, expected < %.3f\n", 1525 P(RINT "xdelta3: test encode with size ratio %.3f, expected < %.3f (%"Q"u, %"Q"u)\n",
1526 ratio, TEST_ADD_RATIO + TEST_EPSILON); 1526 ratio, TEST_ADD_RATIO + TEST_EPSILON, dsize, tsize);
1527 stream->msg = "strange encoding"; 1527 stream->msg = "strange encoding";
1528 return XD3_INTERNAL; 1528 return XD3_INTERNAL;
1529 } 1529 }