summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorjosh.macdonald <jmacd@users.noreply.github.com>2010-10-25 15:23:29 +0000
committerjosh.macdonald <jmacd@users.noreply.github.com>2010-10-25 15:23:29 +0000
commit46593c36a012edda887a0072cf93949086f33b75 (patch)
tree8856f9fcb77a067a10063692ae978b3754c3b782
parent63836024013a4b2859efc792c8ff2aaa65193641 (diff)
Move block-cache code from -main.h to -blkcache.h. Previous merge was
incorrect - contents were branched post-code-removal. To identify diffs in blkcache.h see previous version of -main.h
Diffstat
-rw-r--r--xdelta3/Makefile3
-rw-r--r--xdelta3/linkxd3lib.c4
-rw-r--r--xdelta3/xdelta3-blkcache.h4202
-rw-r--r--xdelta3/xdelta3-main.h646
4 files changed, 422 insertions, 4433 deletions
diff --git a/xdelta3/Makefile b/xdelta3/Makefile
index 08f9a03..88648e2 100644
--- a/xdelta3/Makefile
+++ b/xdelta3/Makefile
@@ -14,7 +14,8 @@ SWIGTGT = xdelta3module.dll
14PYTGT = build/lib.cygwin-1.5.24-i686-$(PYVER)/xdelta3main.dll 14PYTGT = build/lib.cygwin-1.5.24-i686-$(PYVER)/xdelta3main.dll
15endif 15endif
16 16
17SOURCES = xdelta3-cfgs.h \ 17SOURCES = xdelta3-blkcache.h \
18 xdelta3-cfgs.h \
18 xdelta3-decode.h \ 19 xdelta3-decode.h \
19 xdelta3-djw.h \ 20 xdelta3-djw.h \
20 xdelta3-fgk.h \ 21 xdelta3-fgk.h \
diff --git a/xdelta3/linkxd3lib.c b/xdelta3/linkxd3lib.c
index 284cb0d..0f7f739 100644
--- a/xdelta3/linkxd3lib.c
+++ b/xdelta3/linkxd3lib.c
@@ -23,7 +23,6 @@ int main() {
23 xd3_avail_input (& stream, NULL, 0); 23 xd3_avail_input (& stream, NULL, 0);
24 xd3_consume_output (& stream); 24 xd3_consume_output (& stream);
25 25
26 use (xd3_bytes_on_srcblk (& source, 0));
27 use (xd3_set_source (& stream, & source)); 26 use (xd3_set_source (& stream, & source));
28 xd3_set_flags (& stream, 0); 27 xd3_set_flags (& stream, 0);
29 28
@@ -31,9 +30,6 @@ int main() {
31 use (xd3_decode_input (&stream)); 30 use (xd3_decode_input (&stream));
32 use (xd3_get_appheader (& stream, NULL, NULL)); 31 use (xd3_get_appheader (& stream, NULL, NULL));
33 32
34 use ((int) xd3_errstring (& stream));
35 use ((int) xd3_strerror (0));
36
37#if XD3_ENCODER 33#if XD3_ENCODER
38 use (xd3_encode_input (&stream)); 34 use (xd3_encode_input (&stream));
39 use (xd3_encode_stream (& stream, NULL, 0, NULL, NULL, 0)); 35 use (xd3_encode_stream (& stream, NULL, 0, NULL, NULL, 0));
diff --git a/xdelta3/xdelta3-blkcache.h b/xdelta3/xdelta3-blkcache.h
index e47d708..d01322e 100644
--- a/xdelta3/xdelta3-blkcache.h
+++ b/xdelta3/xdelta3-blkcache.h
@@ -18,4034 +18,594 @@
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */ 19 */
20 20
21/* This is all the extra stuff you need for convenience to users in a 21typedef struct _main_blklru main_blklru;
22 * command line application. It contains these major components: 22typedef struct _main_blklru_list main_blklru_list;
23 *
24 * 1. VCDIFF tools 2. external compression support (this is
25 * POSIX-specific). 3. a general read/write loop that handles all of
26 * the Xdelta decode/encode/VCDIFF-print functions 4. command-line
27 * interpreter 5. an Xdelta application header which stores default
28 * filename, external compression settings 6. output/error printing
29 * 7. basic file support and OS interface
30 */
31 23
32/* TODO list: 1. do exact gzip-like filename, stdout handling. make a 24struct _main_blklru_list
33 * .vcdiff extension, refuse to encode to stdout without -cf, etc.
34 * 2. Allow the user to add a comment string to the app header without
35 * disturbing the default behavior. 3. "Source file must be seekable"
36 * is not actually true for encoding, given current behavior. Allow
37 * non-seekable sources? It would in theory let you use a fifo for
38 * the source.
39 */
40
41/* On error handling and printing:
42 *
43 * The xdelta library sets stream->msg to indicate what condition
44 * caused an internal failure, but many failures originate here and
45 * are printed here. The return convention is 0 for success, as
46 * throughout Xdelta code, but special attention is required here for
47 * the operating system calls with different error handling. See the
48 * main_file_* routines. All errors in this file have a message
49 * printed at the time of occurance. Since some of these calls occur
50 * within calls to the library, the error may end up being printed
51 * again with a more general error message.
52 */
53
54/*********************************************************************/
55
56#ifndef XD3_POSIX
57#define XD3_POSIX 0
58#endif
59#ifndef XD3_STDIO
60#define XD3_STDIO 0
61#endif
62#ifndef XD3_WIN32
63#define XD3_WIN32 0
64#endif
65#ifndef NOT_MAIN
66#define NOT_MAIN 0
67#endif
68
69/* Combines xd3_strerror() and strerror() */
70const char* xd3_mainerror(int err_num);
71
72/* XPRINTX (used by main) prefixes an "xdelta3: " to the output. */
73#define XPR fprintf
74#define NT stderr, "xdelta3: "
75
76/* If none are set, default to posix. */
77#if (XD3_POSIX + XD3_STDIO + XD3_WIN32) == 0
78#undef XD3_POSIX
79#define XD3_POSIX 1
80#endif
81
82/* Handle externally-compressed inputs. */
83#ifndef EXTERNAL_COMPRESSION
84#define EXTERNAL_COMPRESSION 1
85#endif
86
87#define PRINTHDR_SPECIAL -4378291
88
89/* The number of soft-config variables. */
90#define XD3_SOFTCFG_VARCNT 7
91
92/* this is used as in XPR(NT XD3_LIB_ERRMSG (stream, ret)) to print an
93 * error message from the library. */
94#define XD3_LIB_ERRMSG(stream, ret) "%s: %s\n", \
95 xd3_errstring (stream), xd3_mainerror (ret)
96
97#include <stdio.h> /* fprintf */
98
99#if XD3_POSIX
100#include <unistd.h> /* close, read, write... */
101#include <sys/types.h>
102#include <fcntl.h>
103#endif
104
105#ifndef _WIN32
106#include <unistd.h> /* lots */
107#include <sys/time.h> /* gettimeofday() */
108#include <sys/stat.h> /* stat() and fstat() */
109#else
110#if defined(_MSC_VER)
111#define strtoll _strtoi64
112#endif
113#include <sys/types.h>
114#include <sys/stat.h>
115#ifndef WIFEXITED
116# define WIFEXITED(stat) (((*((int *) &(stat))) & 0xff) == 0)
117#endif
118#ifndef WEXITSTATUS
119# define WEXITSTATUS(stat) (((*((int *) &(stat))) >> 8) & 0xff)
120#endif
121#ifndef S_ISREG
122//# ifdef S_IFREG
123//# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
124//# else
125# define S_ISREG(m) 1
126//# endif
127#endif /* !S_ISREG */
128
129// For standard input/output handles
130static STARTUPINFO winStartupInfo;
131#endif
132
133/**********************************************************************
134 ENUMS and TYPES
135 *********************************************************************/
136
137/* These flags (mainly pertaining to main_read() operations) are set
138 * in the main_file->flags variable. All are related to with external
139 * decompression support.
140 *
141 * RD_FIRST causes the external decompression check when the input is
142 * first read.
143 *
144 * RD_NONEXTERNAL disables external decompression for reading a
145 * compressed input, in the case of Xdelta inputs. Note: Xdelta is
146 * supported as an external compression type, which makes is the
147 * reason for this flag. An example to justify this is: to create a
148 * delta between two files that are VCDIFF-compressed. Two external
149 * Xdelta decoders are run to supply decompressed source and target
150 * inputs to the Xdelta encoder. */
151typedef enum
152{ 25{
153 RD_FIRST = (1 << 0), 26 main_blklru_list *next;
154 RD_NONEXTERNAL = (1 << 1), 27 main_blklru_list *prev;
155 RD_DECOMPSET = (1 << 2),
156 RD_MAININPUT = (1 << 3),
157} xd3_read_flags;
158
159/* main_file->mode values */
160typedef enum
161{
162 XO_READ = 0,
163 XO_WRITE = 1,
164} main_file_modes;
165
166/* Main commands. For example, CMD_PRINTHDR is the "xdelta printhdr"
167 * command. */
168typedef enum
169{
170 CMD_NONE = 0,
171 CMD_PRINTHDR,
172 CMD_PRINTHDRS,
173 CMD_PRINTDELTA,
174 CMD_RECODE,
175 CMD_MERGE_ARG,
176 CMD_MERGE,
177#if XD3_ENCODER
178 CMD_ENCODE,
179#endif
180 CMD_DECODE,
181 CMD_TEST,
182 CMD_CONFIG,
183} xd3_cmd;
184
185#if XD3_ENCODER
186#define CMD_DEFAULT CMD_ENCODE
187#define IS_ENCODE(cmd) (cmd == CMD_ENCODE)
188#else
189#define CMD_DEFAULT CMD_DECODE
190#define IS_ENCODE(cmd) (0)
191#endif
192
193typedef struct _main_file main_file;
194typedef struct _main_extcomp main_extcomp;
195typedef struct _main_merge main_merge;
196typedef struct _main_merge_list main_merge_list;
197
198/* The main_file object supports abstract system calls like open,
199 * close, read, write, seek, stat. The program uses these to
200 * represent both seekable files and non-seekable files. Source files
201 * must be seekable, but the target input and any output file do not
202 * require seekability.
203 */
204struct _main_file
205{
206#if XD3_STDIO
207 FILE *file;
208#elif XD3_POSIX
209 int file;
210#elif XD3_WIN32
211 HANDLE file;
212#endif
213
214 int mode; /* XO_READ and XO_WRITE */
215 const char *filename; /* File name or /dev/stdin,
216 * /dev/stdout, /dev/stderr. */
217 char *filename_copy; /* File name or /dev/stdin,
218 * /dev/stdout, /dev/stderr. */
219 const char *realname; /* File name or /dev/stdin,
220 * /dev/stdout, /dev/stderr. */
221 const main_extcomp *compressor; /* External compression struct. */
222 int flags; /* RD_FIRST, RD_NONEXTERNAL, ... */
223 xoff_t nread; /* for input position */
224 xoff_t nwrite; /* for output position */
225 uint8_t *snprintf_buf; /* internal snprintf() use */
226 int size_known; /* Set by main_set_souze */
227 xoff_t source_position; /* for avoiding seek in getblk_func */
228 int seek_failed; /* after seek fails once, try FIFO */
229}; 28};
230 29
231/* Various strings and magic values used to detect and call external 30struct _main_blklru
232 * compression. See below for examples. */
233struct _main_extcomp
234{ 31{
235 const char *recomp_cmdname; 32 uint8_t *blk;
236 const char *recomp_options; 33 xoff_t blkno;
237 34 usize_t size;
238 const char *decomp_cmdname; 35 main_blklru_list link;
239 const char *decomp_options;
240
241 const char *ident;
242 const char *magic;
243 usize_t magic_size;
244 int flags;
245}; 36};
246 37
247/* Merge state: */ 38#define MAX_LRU_SIZE 32U
39#define XD3_MINSRCWINSZ XD3_ALLOCSIZE
248 40
249struct _main_merge_list 41XD3_MAKELIST(main_blklru_list,main_blklru,link);
250{
251 main_merge_list *next;
252 main_merge_list *prev;
253};
254
255struct _main_merge
256{
257 const char *filename;
258
259 main_merge_list link;
260};
261
262XD3_MAKELIST(main_merge_list,main_merge,link);
263
264// TODO: really need to put options in a struct so that internal
265// callers can easily reset state.
266
267#define DEFAULT_VERBOSE 0
268
269/* Program options: various command line flags and options. */
270static int option_stdout = 0;
271static int option_force = 0;
272static int option_verbose = DEFAULT_VERBOSE;
273static int option_quiet = 0;
274static int option_use_appheader = 1;
275static uint8_t* option_appheader = NULL;
276static int option_use_secondary = 0;
277static const char* option_secondary = NULL;
278static int option_use_checksum = 1;
279static int option_use_altcodetable = 0;
280static const char* option_smatch_config = NULL;
281static int option_no_compress = 0;
282static int option_no_output = 0; /* do not write output */
283static const char *option_source_filename = NULL;
284
285static int option_level = XD3_DEFAULT_LEVEL;
286static usize_t option_iopt_size = XD3_DEFAULT_IOPT_SIZE;
287static usize_t option_winsize = XD3_DEFAULT_WINSIZE;
288static usize_t option_srcwinsz = XD3_DEFAULT_SRCWINSZ;
289static usize_t option_sprevsz = XD3_DEFAULT_SPREVSZ;
290
291/* These variables are supressed to avoid their use w/o support. main() warns
292 * appropriately when external compression is not enabled. */
293#if EXTERNAL_COMPRESSION
294static int num_subprocs = 0;
295static int option_force2 = 0;
296static int option_decompress_inputs = 1;
297static int option_recompress_outputs = 1;
298#endif
299
300/* This is for comparing "printdelta" output without attention to
301 * copy-instruction modes. */
302#if VCDIFF_TOOLS
303static int option_print_cpymode = 1; /* Note: see reset_defaults(). */
304#endif
305
306/* Static variables */
307IF_DEBUG(static int main_mallocs = 0;)
308
309static char* program_name = NULL;
310static uint8_t* appheader_used = NULL;
311static uint8_t* main_bdata = NULL;
312static usize_t main_bsize = 0;
313
314/* Hacks for VCDIFF tools */
315static int allow_fake_source = 0;
316
317/* recode_stream is used by both recode/merge for reading vcdiff inputs */
318static xd3_stream *recode_stream = NULL;
319
320/* merge_stream is used by merge commands for storing the source encoding */
321static xd3_stream *merge_stream = NULL;
322
323/* This array of compressor types is compiled even if EXTERNAL_COMPRESSION is
324 * false just so the program knows the mapping of IDENT->NAME. */
325static main_extcomp extcomp_types[] =
326{
327 { "bzip2", "-c", "bzip2", "-dc", "B", "BZh", 3, 0 },
328 { "gzip", "-c", "gzip", "-dc", "G", "\037\213", 2, 0 },
329 { "compress", "-c", "uncompress", "-c", "Z", "\037\235", 2, 0 },
330
331 /* TODO: add commandline support for magic-less formats */
332 /*{ "lzma", "-c", "lzma", "-dc", "M", "]\000", 2, 0 },*/
333 42
334 /* Xz is lzma with a magic number http://tukaani.org/xz/ */ 43static usize_t lru_size = 0;
335 { "xz", "-c", "xz", "-dc", "Y", "\xfd\x37\x7a\x58\x5a\x00", 2, 0 }, 44static main_blklru *lru = NULL; /* array of lru_size elts */
336}; 45static main_blklru_list lru_list;
337 46static main_blklru_list lru_free;
338static int main_input (xd3_cmd cmd, main_file *ifile, 47static int do_src_fifo = 0; /* set to avoid lru */
339 main_file *ofile, main_file *sfile);
340static void main_get_appheader (xd3_stream *stream, main_file *ifile,
341 main_file *output, main_file *sfile);
342
343static int main_getblk_func (xd3_stream *stream,
344 xd3_source *source,
345 xoff_t blkno);
346static void main_free (void *ptr);
347static void* main_malloc (usize_t size);
348
349static int main_file_open (main_file *xfile, const char* name, int mode);
350static int main_file_stat (main_file *xfile, xoff_t *size);
351static int main_file_seek (main_file *xfile, xoff_t pos);
352static int main_read_primary_input (main_file *file,
353 uint8_t *buf,
354 usize_t size,
355 usize_t *nread);
356 48
357static const char* main_format_bcnt (xoff_t r, char *buf); 49static int lru_hits = 0;
358static int main_help (void); 50static int lru_misses = 0;
51static int lru_filled = 0;
359 52
360/* The code in xdelta3-blk.h is essentially part of this unit, see 53static void main_lru_reset()
361 * comments there. */
362#include "xdelta3-blkcache.h"
363
364static int
365main_version (void)
366{ 54{
367 /* $Format: " DP(RINT \"Xdelta version $Xdelta3Version$, Copyright (C) 2007, 2008, 2009, 2010, Joshua MacDonald\n\");" $ */ 55 lru_size = 0;
368 DP(RINT "Xdelta version 3.0z, Copyright (C) 2007, 2008, 2009, 2010, Joshua MacDonald\n"); 56 lru = NULL;
369 DP(RINT "Xdelta comes with ABSOLUTELY NO WARRANTY.\n"); 57 do_src_fifo = 0;
370 DP(RINT "This is free software, and you are welcome to redistribute it\n"); 58 lru_hits = 0;
371 DP(RINT "under certain conditions; see \"COPYING\" for details.\n"); 59 lru_misses = 0;
372 return EXIT_SUCCESS; 60 lru_filled = 0;
373}
374
375static int
376main_config (void)
377{
378 main_version ();
379
380 DP(RINT "EXTERNAL_COMPRESSION=%d\n", EXTERNAL_COMPRESSION);
381 DP(RINT "GENERIC_ENCODE_TABLES=%d\n", GENERIC_ENCODE_TABLES);
382 DP(RINT "GENERIC_ENCODE_TABLES_COMPUTE=%d\n", GENERIC_ENCODE_TABLES_COMPUTE);
383 DP(RINT "REGRESSION_TEST=%d\n", REGRESSION_TEST);
384 DP(RINT "SECONDARY_DJW=%d\n", SECONDARY_DJW);
385 DP(RINT "SECONDARY_FGK=%d\n", SECONDARY_FGK);
386 DP(RINT "UNALIGNED_OK=%d\n", UNALIGNED_OK);
387 DP(RINT "VCDIFF_TOOLS=%d\n", VCDIFF_TOOLS);
388 DP(RINT "XD3_ALLOCSIZE=%d\n", XD3_ALLOCSIZE);
389 DP(RINT "XD3_DEBUG=%d\n", XD3_DEBUG);
390 DP(RINT "XD3_ENCODER=%d\n", XD3_ENCODER);
391 DP(RINT "XD3_POSIX=%d\n", XD3_POSIX);
392 DP(RINT "XD3_STDIO=%d\n", XD3_STDIO);
393 DP(RINT "XD3_WIN32=%d\n", XD3_WIN32);
394 DP(RINT "XD3_USE_LARGEFILE64=%d\n", XD3_USE_LARGEFILE64);
395 DP(RINT "XD3_DEFAULT_LEVEL=%d\n", XD3_DEFAULT_LEVEL);
396 DP(RINT "XD3_DEFAULT_IOPT_SIZE=%d\n", XD3_DEFAULT_IOPT_SIZE);
397 DP(RINT "XD3_DEFAULT_SPREVSZ=%d\n", XD3_DEFAULT_SPREVSZ);
398 DP(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ);
399 DP(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE);
400 DP(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE);
401 DP(RINT "sizeof(void*)=%d\n", (int)sizeof(void*));
402 DP(RINT "sizeof(int)=%d\n", (int)sizeof(int));
403 DP(RINT "sizeof(uint32_t)=%d\n", (int)sizeof(uint32_t));
404 DP(RINT "sizeof(uint64_t)=%d\n", (int)sizeof(uint64_t));
405 DP(RINT "sizeof(usize_t)=%d\n", (int)sizeof(usize_t));
406 DP(RINT "sizeof(xoff_t)=%d\n", (int)sizeof(xoff_t));
407
408 return EXIT_SUCCESS;
409}
410
411static void
412reset_defaults(void)
413{
414 option_stdout = 0;
415 option_force = 0;
416 option_verbose = DEFAULT_VERBOSE;
417 option_quiet = 0;
418 option_appheader = NULL;
419 option_use_secondary = 0;
420 option_secondary = NULL;
421 option_use_altcodetable = 0;
422 option_smatch_config = NULL;
423 option_no_compress = 0;
424 option_no_output = 0;
425 option_source_filename = NULL;
426 program_name = NULL;
427 appheader_used = NULL;
428 main_bdata = NULL;
429 main_bsize = 0;
430 allow_fake_source = 0;
431 option_smatch_config = NULL;
432
433 main_lru_reset();
434
435 option_use_appheader = 1;
436 option_use_checksum = 1;
437#if EXTERNAL_COMPRESSION
438 option_force2 = 0;
439 option_decompress_inputs = 1;
440 option_recompress_outputs = 1;
441 num_subprocs = 0;
442#endif
443#if VCDIFF_TOOLS
444 option_print_cpymode = 1;
445#endif
446 option_level = XD3_DEFAULT_LEVEL;
447 option_iopt_size = XD3_DEFAULT_IOPT_SIZE;
448 option_winsize = XD3_DEFAULT_WINSIZE;
449 option_srcwinsz = XD3_DEFAULT_SRCWINSZ;
450 option_sprevsz = XD3_DEFAULT_SPREVSZ;
451}
452
453static void*
454main_malloc1 (usize_t size)
455{
456 void* r = malloc (size);
457 if (r == NULL) { XPR(NT "malloc: %s\n", xd3_mainerror (ENOMEM)); }
458 else if (option_verbose > 4) { XPR(NT "malloc: %u: %p\n", size, r); }
459 return r;
460}
461
462static void*
463main_malloc (usize_t size)
464{
465 void *r = main_malloc1 (size);
466 if (r) { IF_DEBUG (main_mallocs += 1); }
467 return r;
468}
469
470static void*
471main_alloc (void *opaque,
472 usize_t items,
473 usize_t size)
474{
475 return main_malloc1 (items * size);
476}
477
478static void
479main_free1 (void *opaque, void *ptr)
480{
481 if (option_verbose > 4) { XPR(NT "free: %p\n", ptr); }
482 free (ptr);
483}
484
485static void
486main_free (void *ptr)
487{
488 if (ptr)
489 {
490 IF_DEBUG (main_mallocs -= 1);
491 main_free1 (NULL, ptr);
492 IF_DEBUG (XD3_ASSERT(main_mallocs >= 0));
493 }
494}
495
496/* This ensures that (ret = errno) always indicates failure, in case errno was
497 * accidentally not set. If this prints there's a bug somewhere. */
498static int
499get_errno (void)
500{
501#ifndef _WIN32
502 if (errno == 0)
503 {
504 XPR(NT "you found a bug: expected errno != 0\n");
505 errno = XD3_INTERNAL;
506 }
507 return errno;
508#else
509 DWORD err_num = GetLastError();
510 if (err_num == NO_ERROR)
511 {
512 err_num = XD3_INTERNAL;
513 }
514 return err_num;
515#endif
516}
517
518const char*
519xd3_mainerror(int err_num) {
520#ifndef _WIN32
521 const char* x = xd3_strerror (err_num);
522 if (x != NULL)
523 {
524 return x;
525 }
526 return strerror(err_num);
527#else
528 static char err_buf[256];
529 const char* x = xd3_strerror (err_num);
530 if (x != NULL)
531 {
532 return x;
533 }
534 memset (err_buf, 0, 256);
535 FormatMessage (FORMAT_MESSAGE_FROM_SYSTEM |
536 FORMAT_MESSAGE_IGNORE_INSERTS,
537 NULL, err_num,
538 MAKELANGID (LANG_NEUTRAL, SUBLANG_DEFAULT),
539 err_buf, 256, NULL);
540 if (err_buf[0] != 0 && err_buf[strlen(err_buf) - 1] == '\n')
541 {
542 err_buf[strlen(err_buf) - 1] = 0;
543 }
544 return err_buf;
545#endif
546}
547
548static long
549get_millisecs_now (void)
550{
551#ifndef _WIN32
552 struct timeval tv;
553
554 gettimeofday (& tv, NULL);
555
556 return (tv.tv_sec) * 1000L + (tv.tv_usec) / 1000;
557#else
558 SYSTEMTIME st;
559 FILETIME ft;
560 __int64 *pi = (__int64*)&ft;
561 GetLocalTime(&st);
562 SystemTimeToFileTime(&st, &ft);
563 return (long)((*pi) / 10000);
564#endif
565}
566
567/* Always >= 1 millisec, right? */
568static long
569get_millisecs_since (void)
570{
571 static long last = 0;
572 long now = get_millisecs_now();
573 long diff = now - last;
574 last = now;
575 return diff;
576}
577
578static const char*
579main_format_bcnt (xoff_t r, char *buf)
580{
581 static const char* fmts[] = { "B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB" };
582 usize_t i;
583
584 for (i = 0; i < SIZEOF_ARRAY(fmts) - 1; i += 1)
585 {
586 xoff_t new_r;
587
588 if (r == 0)
589 {
590 sprintf (buf, "0 %s", fmts[i]);
591 return buf;
592 }
593
594 if (r >= 1 && r < 10)
595 {
596 sprintf (buf, "%.2f %s", (double) r, fmts[i]);
597 return buf;
598 }
599
600 if (r >= 10 && r < 100)
601 {
602 sprintf (buf, "%.1f %s", (double) r, fmts[i]);
603 return buf;
604 }
605
606 if (r >= 100 && r < 1000)
607 {
608 sprintf (buf, "%"Q"u %s", r, fmts[i]);
609 return buf;
610 }
611
612 new_r = r / 1024;
613
614 if (new_r < 10)
615 {
616 sprintf (buf, "%.2f %s", (double) r / 1024.0, fmts[i + 1]);
617 return buf;
618 }
619
620 if (new_r < 100)
621 {
622 sprintf (buf, "%.1f %s", (double) r / 1024.0, fmts[i + 1]);
623 return buf;
624 }
625
626 r = new_r;
627 }
628 XD3_ASSERT (0);
629 return "";
630}
631
632static char*
633main_format_rate (xoff_t bytes, long millis, char *buf)
634{
635 xoff_t r = (xoff_t)(1.0 * bytes / (1.0 * millis / 1000.0));
636 static char lbuf[32];
637
638 main_format_bcnt (r, lbuf);
639 sprintf (buf, "%s/s", lbuf);
640 return buf;
641}
642
643static char*
644main_format_millis (long millis, char *buf)
645{
646 if (millis < 1000) { sprintf (buf, "%lu ms", millis); }
647 else if (millis < 10000) { sprintf (buf, "%.1f sec", millis / 1000.0); }
648 else { sprintf (buf, "%lu sec", millis / 1000L); }
649 return buf;
650}
651
652/* A safe version of strtol for xoff_t. */
653static int
654main_strtoxoff (const char* s, xoff_t *xo, char which)
655{
656 char *e;
657 xoff_t x;
658
659 XD3_ASSERT(s && *s != 0);
660
661 {
662 /* Should check LONG_MIN, LONG_MAX, LLONG_MIN, LLONG_MAX? */
663#if SIZEOF_XOFF_T == 4
664 long xx = strtol (s, &e, 0);
665#else
666 long long xx = strtoll (s, &e, 0);
667#endif
668
669 if (xx < 0)
670 {
671 XPR(NT "-%c: negative integer: %s\n", which, s);
672 return EXIT_FAILURE;
673 }
674
675 x = xx;
676 }
677
678 if (*e != 0)
679 {
680 XPR(NT "-%c: invalid integer: %s\n", which, s);
681 return EXIT_FAILURE;
682 }
683
684 (*xo) = x;
685 return 0;
686}
687
688static int
689main_atou (const char* arg, usize_t *xo, usize_t low,
690 usize_t high, char which)
691{
692 xoff_t x;
693 int ret;
694
695 if ((ret = main_strtoxoff (arg, & x, which))) { return ret; }
696
697 if (x < low)
698 {
699 XPR(NT "-%c: minimum value: %u\n", which, low);
700 return EXIT_FAILURE;
701 }
702 if (high == 0)
703 {
704 high = USIZE_T_MAX;
705 }
706 if (x > high)
707 {
708 XPR(NT "-%c: maximum value: %u\n", which, high);
709 return EXIT_FAILURE;
710 }
711 (*xo) = (usize_t)x;
712 return 0;
713}
714
715/******************************************************************
716 FILE BASICS
717 ******************************************************************/
718
719/* With all the variation in file system-call semantics, arguments,
720 * return values and error-handling for the POSIX and STDIO file APIs,
721 * the insides of these functions make me sick, which is why these
722 * wrappers exist. */
723
724#define XOPEN_OPNAME (xfile->mode == XO_READ ? "read" : "write")
725#define XOPEN_STDIO (xfile->mode == XO_READ ? "rb" : "wb")
726#define XOPEN_POSIX (xfile->mode == XO_READ ? \
727 O_RDONLY : O_WRONLY | O_CREAT | O_TRUNC)
728#define XOPEN_MODE (xfile->mode == XO_READ ? 0 : 0666)
729
730#define XF_ERROR(op, name, ret) \
731 do { if (!option_quiet) { XPR(NT "file %s failed: %s: %s: %s\n", (op), \
732 XOPEN_OPNAME, (name), xd3_mainerror (ret)); } } while (0)
733
734#if XD3_STDIO
735#define XFNO(f) fileno(f->file)
736#define XSTDOUT_XF(f) { (f)->file = stdout; (f)->filename = "/dev/stdout"; }
737#define XSTDIN_XF(f) { (f)->file = stdin; (f)->filename = "/dev/stdin"; }
738
739#elif XD3_POSIX
740#define XFNO(f) f->file
741#define XSTDOUT_XF(f) \
742 { (f)->file = STDOUT_FILENO; (f)->filename = "/dev/stdout"; }
743#define XSTDIN_XF(f) \
744 { (f)->file = STDIN_FILENO; (f)->filename = "/dev/stdin"; }
745
746#elif XD3_WIN32
747#define XFNO(f) -1
748#define XSTDOUT_XF(f) { \
749 (f)->file = GetStdHandle(STD_OUTPUT_HANDLE); \
750 (f)->filename = "(stdout)"; \
751 }
752#define XSTDIN_XF(f) { \
753 (f)->file = GetStdHandle(STD_INPUT_HANDLE); \
754 (f)->filename = "(stdin)"; \
755 }
756#endif
757
758static void
759main_file_init (main_file *xfile)
760{
761 memset (xfile, 0, sizeof (*xfile));
762
763#if XD3_POSIX
764 xfile->file = -1;
765#endif
766#if XD3_WIN32
767 xfile->file = INVALID_HANDLE_VALUE;
768#endif
769}
770
771static int
772main_file_isopen (main_file *xfile)
773{
774#if XD3_STDIO
775 return xfile->file != NULL;
776
777#elif XD3_POSIX
778 return xfile->file != -1;
779
780#elif XD3_WIN32
781 return xfile->file != INVALID_HANDLE_VALUE;
782#endif
783}
784
785static int
786main_file_close (main_file *xfile)
787{
788 int ret = 0;
789
790 if (! main_file_isopen (xfile))
791 {
792 return 0;
793 }
794
795#if XD3_STDIO
796 ret = fclose (xfile->file);
797 xfile->file = NULL;
798
799#elif XD3_POSIX
800 ret = close (xfile->file);
801 xfile->file = -1;
802
803#elif XD3_WIN32
804 if (!CloseHandle(xfile->file)) {
805 ret = get_errno ();
806 }
807 xfile->file = INVALID_HANDLE_VALUE;
808#endif
809
810 if (ret != 0) { XF_ERROR ("close", xfile->filename, ret = get_errno ()); }
811 return ret;
812}
813
814static void
815main_file_cleanup (main_file *xfile)
816{
817 XD3_ASSERT (xfile != NULL);
818
819 if (main_file_isopen (xfile))
820 {
821 main_file_close (xfile);
822 }
823
824 if (xfile->snprintf_buf != NULL)
825 {
826 main_free(xfile->snprintf_buf);
827 xfile->snprintf_buf = NULL;
828 }
829
830 if (xfile->filename_copy != NULL)
831 {
832 main_free(xfile->filename_copy);
833 xfile->filename_copy = NULL;
834 }
835}
836
837static int
838main_file_open (main_file *xfile, const char* name, int mode)
839{
840 int ret = 0;
841
842 xfile->mode = mode;
843
844 XD3_ASSERT (name != NULL);
845 XD3_ASSERT (! main_file_isopen (xfile));
846 if (name[0] == 0)
847 {
848 XPR(NT "invalid file name: empty string\n");
849 return XD3_INVALID;
850 }
851
852#if XD3_STDIO
853 xfile->file = fopen (name, XOPEN_STDIO);
854
855 ret = (xfile->file == NULL) ? get_errno () : 0;
856
857#elif XD3_POSIX
858 if ((ret = open (name, XOPEN_POSIX, XOPEN_MODE)) < 0)
859 {
860 ret = get_errno ();
861 }
862 else
863 {
864 xfile->file = ret;
865 ret = 0;
866 }
867
868#elif XD3_WIN32
869 xfile->file = CreateFile(name,
870 (mode == XO_READ) ? GENERIC_READ : GENERIC_WRITE,
871 FILE_SHARE_READ,
872 NULL,
873 (mode == XO_READ) ?
874 OPEN_EXISTING :
875 (option_force ? CREATE_ALWAYS : CREATE_NEW),
876 FILE_ATTRIBUTE_NORMAL,
877 NULL);
878 if (xfile->file == INVALID_HANDLE_VALUE)
879 {
880 ret = get_errno ();
881 }
882#endif
883 if (ret) { XF_ERROR ("open", name, ret); }
884 else { xfile->realname = name; xfile->nread = 0; }
885 return ret;
886}
887
888static int
889main_file_stat (main_file *xfile, xoff_t *size)
890{
891 int ret = 0;
892#if XD3_WIN32
893 if (GetFileType(xfile->file) != FILE_TYPE_DISK)
894 {
895 return -1;
896 }
897# if (_WIN32_WINNT >= 0x0500)
898 {
899 LARGE_INTEGER li;
900 if (GetFileSizeEx(xfile->file, &li) == 0)
901 {
902 return get_errno ();
903 }
904 *size = li.QuadPart;
905 }
906# else
907 {
908 DWORD filesize = GetFileSize(xfile->file, NULL);
909 if (filesize == INVALID_FILE_SIZE)
910 {
911 return get_errno ()
912 }
913 *size = filesize;
914 }
915# endif
916#else
917 struct stat sbuf;
918 if (fstat (XFNO (xfile), & sbuf) < 0)
919 {
920 ret = get_errno ();
921 return ret;
922 }
923
924 if (! S_ISREG (sbuf.st_mode))
925 {
926 return ESPIPE;
927 }
928 (*size) = sbuf.st_size;
929#endif
930 return ret;
931}
932
933static int
934main_file_exists (main_file *xfile)
935{
936 struct stat sbuf;
937 return stat (xfile->filename, & sbuf) == 0 && S_ISREG (sbuf.st_mode);
938}
939
940#if (XD3_POSIX || EXTERNAL_COMPRESSION)
941/* POSIX-generic code takes a function pointer to read() or write().
942 * This calls the function repeatedly until the buffer is full or EOF.
943 * The NREAD parameter is not set for write, NULL is passed. Return
944 * is signed, < 0 indicate errors, otherwise byte count. */
945typedef int (xd3_posix_func) (int fd, uint8_t *buf, usize_t size);
946
947static int
948xd3_posix_io (int fd, uint8_t *buf, usize_t size,
949 xd3_posix_func *func, usize_t *nread)
950{
951 int ret;
952 usize_t nproc = 0;
953
954 while (nproc < size)
955 {
956 int result = (*func) (fd, buf + nproc, size - nproc);
957
958 if (result < 0)
959 {
960 ret = get_errno ();
961 if (ret != EAGAIN && ret != EINTR)
962 {
963 return ret;
964 }
965 continue;
966 }
967
968 if (nread != NULL && result == 0) { break; }
969
970 nproc += result;
971 }
972 if (nread != NULL) { (*nread) = nproc; }
973 return 0;
974}
975#endif
976
977#if XD3_WIN32
978static int
979xd3_win32_io (HANDLE file, uint8_t *buf, usize_t size,
980 int is_read, usize_t *nread)
981{
982 int ret = 0;
983 usize_t nproc = 0;
984
985 while (nproc < size)
986 {
987 DWORD nproc2;
988 if ((is_read ?
989 ReadFile (file, buf + nproc, size - nproc, &nproc2, NULL) :
990 WriteFile (file, buf + nproc, size - nproc, &nproc2, NULL)) == 0)
991 {
992 ret = get_errno();
993 if (ret != ERROR_HANDLE_EOF && ret != ERROR_BROKEN_PIPE)
994 {
995 return ret;
996 }
997 /* By falling through here, we'll break this loop in the
998 * read case in case of eof or broken pipe. */
999 }
1000
1001 nproc += (usize_t) nproc2;
1002
1003 if (nread != NULL && nproc2 == 0) { break; }
1004 }
1005 if (nread != NULL) { (*nread) = nproc; }
1006 return 0;
1007} 61}
1008#endif
1009 62
1010/* POSIX is unbuffered, while STDIO is buffered. main_file_read() 63static void main_lru_cleanup()
1011 * should always be called on blocks. */
1012static int
1013main_file_read (main_file *ifile,
1014 uint8_t *buf,
1015 usize_t size,
1016 usize_t *nread,
1017 const char *msg)
1018{ 64{
1019 int ret = 0; 65 int i;
1020 66 /* TODO(jmacd): HERE YOU ARE
1021#if XD3_STDIO 67 * Remove this loop, free only lru[0].blk.
1022 usize_t result; 68 */
1023 69 for (i = 0; lru && i < lru_size; i += 1)
1024 result = fread (buf, 1, size, ifile->file);
1025
1026 if (result < size && ferror (ifile->file))
1027 {
1028 ret = get_errno ();
1029 }
1030 else
1031 { 70 {
1032 *nread = result; 71 main_free (lru[i].blk);
1033 } 72 }
1034 73
1035#elif XD3_POSIX 74 main_free (lru);
1036 ret = xd3_posix_io (ifile->file, buf, size, (xd3_posix_func*) &read, nread); 75 lru = NULL;
1037#elif XD3_WIN32
1038 ret = xd3_win32_io (ifile->file, buf, size, 1 /* is_read */, nread);
1039#endif
1040
1041 if (ret)
1042 {
1043 XPR(NT "%s: %s: %s\n", msg, ifile->filename, xd3_mainerror (ret));
1044 }
1045 else
1046 {
1047 if (option_verbose > 4) { XPR(NT "read %s: %u bytes\n",
1048 ifile->filename, (*nread)); }
1049 ifile->nread += (*nread);
1050 }
1051 76
1052 return ret; 77 lru_hits = 0;
78 lru_misses = 0;
79 lru_filled = 0;
1053} 80}
1054 81
82/* This is called at different times for encoding and decoding. The
83 * encoder calls it immediately, the decoder delays until the
84 * application header is received. */
1055static int 85static int
1056main_file_write (main_file *ofile, uint8_t *buf, usize_t size, const char *msg) 86main_set_source (xd3_stream *stream, xd3_cmd cmd,
87 main_file *sfile, xd3_source *source)
1057{ 88{
1058 int ret = 0; 89 int ret = 0;
90 usize_t i;
91 usize_t blksize;
92 xoff_t source_size = 0;
93 main_blklru block0;
1059 94
1060#if XD3_STDIO 95 XD3_ASSERT (lru == NULL);
1061 usize_t result; 96 XD3_ASSERT (stream->src == NULL);
1062 97 XD3_ASSERT (option_srcwinsz >= XD3_MINSRCWINSZ);
1063 result = fwrite (buf, 1, size, ofile->file);
1064
1065 if (result != size) { ret = get_errno (); }
1066
1067#elif XD3_POSIX
1068 ret = xd3_posix_io (ofile->file, buf, size, (xd3_posix_func*) &write, NULL);
1069
1070#elif XD3_WIN32
1071 ret = xd3_win32_io (ofile->file, buf, size, 0, NULL);
1072 98
1073#endif 99 main_blklru_list_init (& lru_list);
100 main_blklru_list_init (& lru_free);
1074 101
1075 if (ret) 102 if (allow_fake_source)
1076 { 103 {
1077 XPR(NT "%s: %s: %s\n", msg, ofile->filename, xd3_mainerror (ret)); 104 sfile->mode = XO_READ;
105 sfile->realname = sfile->filename;
106 sfile->nread = 0;
1078 } 107 }
1079 else 108 else
1080 { 109 {
1081 if (option_verbose > 4) { XPR(NT "write %s: %u bytes\n", 110 if ((ret = main_file_open (sfile, sfile->filename, XO_READ)))
1082 ofile->filename, size); }
1083 ofile->nwrite += size;
1084 }
1085
1086 return ret;
1087}
1088
1089static int
1090main_file_seek (main_file *xfile, xoff_t pos)
1091{
1092 int ret = 0;
1093
1094#if XD3_STDIO
1095 if (fseek (xfile->file, pos, SEEK_SET) != 0) { ret = get_errno (); }
1096
1097#elif XD3_POSIX
1098 if ((xoff_t) lseek (xfile->file, pos, SEEK_SET) != pos)
1099 { ret = get_errno (); }
1100
1101#elif XD3_WIN32
1102# if (_WIN32_WINNT >= 0x0500)
1103 LARGE_INTEGER move, out;
1104 move.QuadPart = pos;
1105 if (SetFilePointerEx(xfile->file, move, &out, FILE_BEGIN) == 0)
1106 {
1107 ret = get_errno ();
1108 }
1109# else
1110 if (SetFilePointer(xfile->file, (LONG)pos, NULL, FILE_BEGIN) ==
1111 INVALID_SET_FILE_POINTER)
1112 {
1113 ret = get_errno ();
1114 }
1115# endif
1116#endif
1117
1118 return ret;
1119}
1120
1121/* This function simply writes the stream output buffer, if there is
1122 * any, for encode, decode and recode commands. (The VCDIFF tools use
1123 * main_print_func()). */
1124static int
1125main_write_output (xd3_stream* stream, main_file *ofile)
1126{
1127 int ret;
1128
1129 if (option_no_output)
1130 {
1131 return 0;
1132 }
1133
1134 if (stream->avail_out > 0 &&
1135 (ret = main_file_write (ofile, stream->next_out,
1136 stream->avail_out, "write failed")))
1137 {
1138 return ret;
1139 }
1140
1141 return 0;
1142}
1143
1144static int
1145main_set_secondary_flags (xd3_config *config)
1146{
1147 int ret;
1148 if (option_use_secondary)
1149 {
1150 /* The default secondary compressor is DJW, if it's compiled. */
1151 if (option_secondary == NULL)
1152 {
1153 if (SECONDARY_DJW)
1154 {
1155 config->flags |= XD3_SEC_DJW;
1156 }
1157 }
1158 else
1159 { 111 {
1160 if (strcmp (option_secondary, "fgk") == 0 && SECONDARY_FGK)
1161 {
1162 config->flags |= XD3_SEC_FGK;
1163 }
1164 else if (strncmp (option_secondary, "djw", 3) == 0 && SECONDARY_DJW)
1165 {
1166 usize_t level = XD3_DEFAULT_SECONDARY_LEVEL;
1167
1168 config->flags |= XD3_SEC_DJW;
1169
1170 if (strlen (option_secondary) > 3 &&
1171 (ret = main_atou (option_secondary + 3,
1172 &level,
1173 0, 9, 'S')) != 0 &&
1174 !option_quiet)
1175 {
1176 return XD3_INVALID;
1177 }
1178
1179 /* XD3_SEC_NOXXXX flags disable secondary compression on
1180 * a per-section basis. For djw, ngroups=1 indicates
1181 * minimum work, ngroups=0 uses default settings, which
1182 * is > 1 groups by default. */
1183 if (level < 1) { config->flags |= XD3_SEC_NODATA; }
1184 if (level < 7) { config->sec_data.ngroups = 1; }
1185 else { config->sec_data.ngroups = 0; }
1186
1187 if (level < 3) { config->flags |= XD3_SEC_NOINST; }
1188 if (level < 8) { config->sec_inst.ngroups = 1; }
1189 else { config->sec_inst.ngroups = 0; }
1190
1191 if (level < 5) { config->flags |= XD3_SEC_NOADDR; }
1192 if (level < 9) { config->sec_addr.ngroups = 1; }
1193 else { config->sec_addr.ngroups = 0; }
1194 }
1195 else if (strcmp (option_secondary, "none") == 0 && SECONDARY_DJW)
1196 {
1197 /* No secondary */
1198 }
1199 else
1200 {
1201 if (!option_quiet)
1202 {
1203 XPR(NT "unrecognized secondary compressor type: %s\n",
1204 option_secondary);
1205 return XD3_INVALID;
1206 }
1207 }
1208 }
1209 }
1210
1211 return 0;
1212}
1213
1214/******************************************************************
1215 VCDIFF TOOLS
1216 *****************************************************************/
1217
1218#if VCDIFF_TOOLS
1219#include "xdelta3-merge.h"
1220
1221/* According to the internet, Windows vsnprintf() differs from most
1222 * Unix implementations regarding the terminating 0 when the boundary
1223 * condition is met. It doesn't matter here, we don't rely on the
1224 * trailing 0. Besides, both Windows and DJGPP vsnprintf return -1
1225 * upon truncation, which isn't C99 compliant. To overcome this,
1226 * recent MinGW runtimes provided their own vsnprintf (notice the
1227 * absence of the '_' prefix) but they were initially buggy. So,
1228 * always use the native '_'-prefixed version with Win32. */
1229#include <stdarg.h>
1230#ifdef _WIN32
1231#define vsnprintf_func _vsnprintf
1232#else
1233#define vsnprintf_func vsnprintf
1234#endif
1235
1236/* Prior to SVN 303 this function was only defined in DJGPP and WIN32
1237 * environments and other platforms would use the builtin snprintf()
1238 * with an arrangement of macros below. In OS X 10.6, Apply made
1239 * snprintf() a macro, which defeated those macros (since snprintf
1240 * would be evaluated before its argument macros were expanded,
1241 * therefore always define xsnprintf_func. */
1242#undef PRINTF_ATTRIBUTE
1243#ifdef __GNUC__
1244/* Let's just assume no one uses gcc 2.x! */
1245#define PRINTF_ATTRIBUTE(x,y) __attribute__ ((__format__ (__printf__, x, y)))
1246#else
1247#define PRINTF_ATTRIBUTE(x,y)
1248#endif
1249
1250static int
1251xsnprintf_func (char *str, int n, const char *fmt, ...)
1252 PRINTF_ATTRIBUTE(3,4);
1253
1254int
1255xsnprintf_func (char *str, int n, const char *fmt, ...)
1256{
1257 va_list a;
1258 int ret;
1259 va_start (a, fmt);
1260 ret = vsnprintf_func (str, n, fmt, a);
1261 va_end (a);
1262 if (ret < 0)
1263 {
1264 ret = n;
1265 }
1266 return ret;
1267}
1268
1269/* The following macros let VCDIFF print using main_file_write(),
1270 * for example:
1271 *
1272 * VC(UT "trying to be portable: %d\n", x)VE;
1273 */
1274#define SNPRINTF_BUFSIZE 1024
1275#define VC do { if (((ret = xsnprintf_func
1276#define UT (char*)xfile->snprintf_buf, SNPRINTF_BUFSIZE,
1277#define VE ) >= SNPRINTF_BUFSIZE \
1278 && (ret = main_print_overflow(ret)) != 0) \
1279 || (ret = main_file_write(xfile, xfile->snprintf_buf, \
1280 (usize_t)ret, "print")) != 0) \
1281 { return ret; } } while (0)
1282
1283static int
1284main_print_overflow (int x)
1285{
1286 XPR(NT "internal print buffer overflow: %d bytes\n", x);
1287 return XD3_INTERNAL;
1288}
1289
1290/* This function prints a single VCDIFF window. */
1291static int
1292main_print_window (xd3_stream* stream, main_file *xfile)
1293{
1294 int ret;
1295 usize_t size = 0;
1296
1297 VC(UT " Offset Code Type1 Size1 @Addr1 + Type2 Size2 @Addr2\n")VE;
1298
1299 while (stream->inst_sect.buf < stream->inst_sect.buf_max)
1300 {
1301 usize_t code = stream->inst_sect.buf[0];
1302 const uint8_t *addr_before = stream->addr_sect.buf;
1303 const uint8_t *inst_before = stream->inst_sect.buf;
1304 usize_t addr_bytes;
1305 usize_t inst_bytes;
1306 usize_t size_before = size;
1307
1308 if ((ret = xd3_decode_instruction (stream)))
1309 {
1310 XPR(NT "instruction decode error at %"Q"u: %s\n",
1311 stream->dec_winstart + size, stream->msg);
1312 return ret; 112 return ret;
1313 } 113 }
1314 114
1315 addr_bytes = (usize_t)(stream->addr_sect.buf - addr_before); 115 /* Allow non-seekable sources from the start. If the file
1316 inst_bytes = (usize_t)(stream->inst_sect.buf - inst_before); 116 * turns out to be externally compressed, size_known may change. */
1317 117 sfile->size_known = (main_file_stat (sfile, &source_size) == 0);
1318 VC(UT " %06"Q"u %03u %s %6u", stream->dec_winstart + size, 118 }
1319 option_print_cpymode ? code : 0, 119
1320 xd3_rtype_to_string ((xd3_rtype) stream->dec_current1.type, 120 /* The API requires power-of-two blocksize, */
1321 option_print_cpymode), 121 blksize = xd3_pow2_roundup (max (option_srcwinsz / MAX_LRU_SIZE,
1322 (usize_t) stream->dec_current1.size)VE; 122 XD3_MINSRCWINSZ));
1323 123
1324 if (stream->dec_current1.type != XD3_NOOP) 124 /* TODO(jmacd): The organization of this code and the implementation
1325 { 125 * of the LRU cache could be improved. This is too convoluted, the
1326 if (stream->dec_current1.type >= XD3_CPY) 126 * code uses main_getblk_func() to read the first block, which may
1327 { 127 * trigger external-decompression, in large part so that the
1328 if (stream->dec_current1.addr >= stream->dec_cpylen) 128 * verbose-printing and counters maintained by getblk_func are
1329 { 129 * consistent. There used to be additional optimizations going on
1330 VC(UT " T@%-6u", 130 * here: (1) if the source size is known we would like to lower
1331 stream->dec_current1.addr - stream->dec_cpylen)VE; 131 * option_srcwinsz, if possible, (2) avoid allocating more memory
1332 } 132 * than needed, and (3) also want to use a single block when
1333 else 133 * source_size < option_srcwinsz, this because compression is better
1334 { 134 * for larger blocks (especially due to the blockwise-reverse
1335 VC(UT " S@%-6"Q"u", 135 * insertion of checksums).
1336 stream->dec_cpyoff + stream->dec_current1.addr)VE; 136 *
1337 } 137 * (3) is no longer implemented. (2) is only implemented for files
1338 } 138 * that are shorter than the default blocksize, and (1) is not
1339 else 139 * implemented. These optimizations are not taken because the code
1340 { 140 * is already too complicated.
1341 VC(UT " ")VE; 141 *
1342 } 142 * The ideal solution may be to allocate a block of memory equal to
1343 143 * half of the option_srcwinsz. Read as much data as possible into
1344 size += stream->dec_current1.size; 144 * that block. If the entire file fits, pass one block to the
1345 } 145 * library for best compression. If not, copy the 50% block into
1346 146 * smaller (option_srcwinsz / MAX_LRU_SIZE) blocks and proceed. Too
1347 if (stream->dec_current2.type != XD3_NOOP) 147 * much effort for too little payback. */
1348 { 148
1349 VC(UT " %s %6u", 149 memset (&block0, 0, sizeof (block0));
1350 xd3_rtype_to_string ((xd3_rtype) stream->dec_current2.type, 150 block0.blkno = (xoff_t) -1;
1351 option_print_cpymode), 151
1352 (usize_t)stream->dec_current2.size)VE; 152 /* TODO(jmacd): HERE YOU ARE
1353 153 * Allocate only one block of option_srcwinsz.
1354 if (stream->dec_current2.type >= XD3_CPY) 154 */
1355 { 155 /* Allocate the first block. Even if the size is known at this
1356 if (stream->dec_current2.addr >= stream->dec_cpylen) 156 * point, we do not lower it because decompression may happen. */
1357 { 157 if ((block0.blk = (uint8_t*) main_malloc (blksize)) == NULL)
1358 VC(UT " T@%-6u",
1359 stream->dec_current2.addr - stream->dec_cpylen)VE;
1360 }
1361 else
1362 {
1363 VC(UT " S@%-6"Q"u",
1364 stream->dec_cpyoff + stream->dec_current2.addr)VE;
1365 }
1366 }
1367
1368 size += stream->dec_current2.size;
1369 }
1370
1371 VC(UT "\n")VE;
1372
1373 if (option_verbose &&
1374 addr_bytes + inst_bytes >= (size - size_before) &&
1375 (stream->dec_current1.type >= XD3_CPY ||
1376 stream->dec_current2.type >= XD3_CPY))
1377 {
1378 VC(UT " %06"Q"u (inefficiency) %u encoded as %u bytes\n",
1379 stream->dec_winstart + size_before,
1380 size - size_before,
1381 addr_bytes + inst_bytes)VE;
1382 }
1383 }
1384
1385 if (stream->dec_tgtlen != size && (stream->flags & XD3_SKIP_WINDOW) == 0)
1386 {
1387 XPR(NT "target window size inconsistency");
1388 return XD3_INTERNAL;
1389 }
1390
1391 if (stream->dec_position != stream->dec_maxpos)
1392 {
1393 XPR(NT "target window position inconsistency");
1394 return XD3_INTERNAL;
1395 }
1396
1397 if (stream->addr_sect.buf != stream->addr_sect.buf_max)
1398 { 158 {
1399 XPR(NT "address section inconsistency"); 159 ret = ENOMEM;
1400 return XD3_INTERNAL; 160 return ret;
1401 } 161 }
1402 162
1403 return 0; 163 source->blksize = blksize;
1404} 164 source->name = sfile->filename;
165 source->ioh = sfile;
166 source->curblkno = (xoff_t) -1;
167 source->curblk = NULL;
168
169 /* We have to read the first block into the cache now, because
170 * size_known can still change (due to secondary decompression).
171 * Calls main_secondary_decompress_check() via
172 * main_read_primary_input(). */
173 lru_size = 1;
174 lru = &block0;
175 XD3_ASSERT (main_blklru_list_empty (& lru_free));
176 XD3_ASSERT (main_blklru_list_empty (& lru_list));
177 main_blklru_list_push_back (& lru_free, & lru[0]);
178 /* This call is partly so that the diagnostics printed by
179 * this function appear to happen normally for the first read,
180 * which is special. */
181 ret = main_getblk_func (stream, source, 0);
182 main_blklru_list_remove (& lru[0]);
183 XD3_ASSERT (main_blklru_list_empty (& lru_free));
184 XD3_ASSERT (main_blklru_list_empty (& lru_list));
185 lru = NULL;
186
187 if (ret != 0)
188 {
189 main_free (block0.blk);
190
191 XPR(NT "error reading source: %s: %s\n",
192 sfile->filename,
193 xd3_mainerror (ret));
1405 194
1406static int 195 return ret;
1407main_print_vcdiff_file (main_file *xfile, main_file *file, const char *type)
1408{
1409 int ret; /* Used by above macros */
1410 if (file->filename)
1411 {
1412 VC(UT "XDELTA filename (%s): %s\n", type,
1413 file->filename)VE;
1414 }
1415 if (file->compressor)
1416 {
1417 VC(UT "XDELTA ext comp (%s): %s\n", type,
1418 file->compressor->recomp_cmdname)VE;
1419 } 196 }
1420 return 0;
1421}
1422 197
1423/* This function prints a VCDIFF input, mainly for debugging purposes. */ 198 source->onblk = block0.size;
1424static int
1425main_print_func (xd3_stream* stream, main_file *xfile)
1426{
1427 int ret;
1428 199
1429 if (option_no_output) 200 /* If the file is smaller than a block, size is known. */
201 if (!sfile->size_known && source->onblk < blksize)
1430 { 202 {
1431 return 0; 203 source_size = source->onblk;
204 sfile->size_known = 1;
1432 } 205 }
1433 206
1434 if (xfile->snprintf_buf == NULL) 207 /* We update lru_size accordingly, and modify option_srcwinsz, which
208 * will be passed via xd3_config. */
209 if (sfile->size_known && source_size <= option_srcwinsz)
1435 { 210 {
1436 if ((xfile->snprintf_buf = 211 lru_size = (usize_t) ((source_size + blksize - 1) / blksize);
1437 (uint8_t*)main_malloc(SNPRINTF_BUFSIZE)) == NULL)
1438 {
1439 return ENOMEM;
1440 }
1441 }
1442
1443 if (stream->dec_winstart == 0)
1444 {
1445 VC(UT "VCDIFF version: 0\n")VE;
1446 VC(UT "VCDIFF header size: %d\n",
1447 stream->dec_hdrsize)VE;
1448 VC(UT "VCDIFF header indicator: ")VE;
1449 if ((stream->dec_hdr_ind & VCD_SECONDARY) != 0)
1450 VC(UT "VCD_SECONDARY ")VE;
1451 if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0)
1452 VC(UT "VCD_CODETABLE ")VE;
1453 if ((stream->dec_hdr_ind & VCD_APPHEADER) != 0)
1454 VC(UT "VCD_APPHEADER ")VE;
1455 if (stream->dec_hdr_ind == 0)
1456 VC(UT "none")VE;
1457 VC(UT "\n")VE;
1458
1459 IF_SEC(VC(UT "VCDIFF secondary compressor: %s\n",
1460 stream->sec_type ? stream->sec_type->name : "none")VE);
1461 IF_NSEC(VC(UT "VCDIFF secondary compressor: unsupported\n")VE);
1462
1463 if (stream->dec_hdr_ind & VCD_APPHEADER)
1464 {
1465 uint8_t *apphead;
1466 usize_t appheadsz;
1467 ret = xd3_get_appheader (stream, & apphead, & appheadsz);
1468
1469 if (ret == 0 && appheadsz > 0)
1470 {
1471 int sq = option_quiet;
1472 main_file i, o, s;
1473 XD3_ASSERT (apphead != NULL);
1474 VC(UT "VCDIFF application header: ")VE;
1475 if ((ret = main_file_write (xfile, apphead,
1476 appheadsz, "print")) != 0)
1477 { return ret; }
1478 VC(UT "\n")VE;
1479
1480 main_file_init (& i);
1481 main_file_init (& o);
1482 main_file_init (& s);
1483 option_quiet = 1;
1484 main_get_appheader (stream, &i, & o, & s);
1485 option_quiet = sq;
1486 if ((ret = main_print_vcdiff_file (xfile, & o, "output")))
1487 { return ret; }
1488 if ((ret = main_print_vcdiff_file (xfile, & s, "source")))
1489 { return ret; }
1490 main_file_cleanup (& i);
1491 main_file_cleanup (& o);
1492 main_file_cleanup (& s);
1493 }
1494 }
1495 } 212 }
1496 else 213 else
1497 { 214 {
1498 VC(UT "\n")VE; 215 lru_size = (option_srcwinsz + blksize - 1) / blksize;
1499 }
1500
1501 VC(UT "VCDIFF window number: %"Q"u\n", stream->current_window)VE;
1502 VC(UT "VCDIFF window indicator: ")VE;
1503 if ((stream->dec_win_ind & VCD_SOURCE) != 0) VC(UT "VCD_SOURCE ")VE;
1504 if ((stream->dec_win_ind & VCD_TARGET) != 0) VC(UT "VCD_TARGET ")VE;
1505 if ((stream->dec_win_ind & VCD_ADLER32) != 0) VC(UT "VCD_ADLER32 ")VE;
1506 if (stream->dec_win_ind == 0) VC(UT "none")VE;
1507 VC(UT "\n")VE;
1508
1509 if ((stream->dec_win_ind & VCD_ADLER32) != 0)
1510 {
1511 VC(UT "VCDIFF adler32 checksum: %08X\n",
1512 (usize_t)stream->dec_adler32)VE;
1513 }
1514
1515 if (stream->dec_del_ind != 0)
1516 {
1517 VC(UT "VCDIFF delta indicator: ")VE;
1518 if ((stream->dec_del_ind & VCD_DATACOMP) != 0) VC(UT "VCD_DATACOMP ")VE;
1519 if ((stream->dec_del_ind & VCD_INSTCOMP) != 0) VC(UT "VCD_INSTCOMP ")VE;
1520 if ((stream->dec_del_ind & VCD_ADDRCOMP) != 0) VC(UT "VCD_ADDRCOMP ")VE;
1521 if (stream->dec_del_ind == 0) VC(UT "none")VE;
1522 VC(UT "\n")VE;
1523 }
1524
1525 if (stream->dec_winstart != 0)
1526 {
1527 VC(UT "VCDIFF window at offset: %"Q"u\n", stream->dec_winstart)VE;
1528 }
1529
1530 if (SRCORTGT (stream->dec_win_ind))
1531 {
1532 VC(UT "VCDIFF copy window length: %u\n",
1533 (usize_t)stream->dec_cpylen)VE;
1534 VC(UT "VCDIFF copy window offset: %"Q"u\n",
1535 stream->dec_cpyoff)VE;
1536 }
1537
1538 VC(UT "VCDIFF delta encoding length: %u\n",
1539 (usize_t)stream->dec_enclen)VE;
1540 VC(UT "VCDIFF target window length: %u\n",
1541 (usize_t)stream->dec_tgtlen)VE;
1542
1543 VC(UT "VCDIFF data section length: %u\n",
1544 (usize_t)stream->data_sect.size)VE;
1545 VC(UT "VCDIFF inst section length: %u\n",
1546 (usize_t)stream->inst_sect.size)VE;
1547 VC(UT "VCDIFF addr section length: %u\n",
1548 (usize_t)stream->addr_sect.size)VE;
1549
1550 ret = 0;
1551 if ((stream->flags & XD3_JUST_HDR) != 0)
1552 {
1553 /* Print a header -- finished! */
1554 ret = PRINTHDR_SPECIAL;
1555 }
1556 else if ((stream->flags & XD3_SKIP_WINDOW) == 0)
1557 {
1558 ret = main_print_window (stream, xfile);
1559 }
1560
1561 return ret;
1562}
1563
1564static int
1565main_recode_copy (xd3_stream* stream,
1566 xd3_output* output,
1567 xd3_desect* input)
1568{
1569 int ret;
1570
1571 XD3_ASSERT(output != NULL);
1572 XD3_ASSERT(output->next_page == NULL);
1573
1574 if ((ret = xd3_decode_allocate (recode_stream,
1575 input->size,
1576 &output->base,
1577 &output->avail)))
1578 {
1579 XPR(NT XD3_LIB_ERRMSG (stream, ret));
1580 return ret;
1581 } 216 }
1582 217
1583 memcpy (output->base, 218 XD3_ASSERT (lru_size >= 1);
1584 /* Note: decoder advances buf, so get base of buffer with 219 option_srcwinsz = lru_size * blksize;
1585 * buf_max - size */
1586 input->buf_max - input->size,
1587 input->size);
1588 output->next = input->size;
1589 return 0;
1590}
1591 220
1592// Re-encode one window 221 if ((lru = (main_blklru*) main_malloc (lru_size * sizeof (main_blklru)))
1593static int 222 == NULL)
1594main_recode_func (xd3_stream* stream, main_file *ofile)
1595{
1596 int ret;
1597 xd3_source decode_source;
1598
1599 XD3_ASSERT(stream->dec_state == DEC_FINISH);
1600 XD3_ASSERT(recode_stream->enc_state == ENC_INIT ||
1601 recode_stream->enc_state == ENC_INPUT);
1602
1603 // Copy partial decoder output to partial encoder inputs
1604 if ((ret = main_recode_copy (recode_stream,
1605 DATA_HEAD(recode_stream),
1606 &stream->data_sect)) ||
1607 (ret = main_recode_copy (recode_stream,
1608 INST_HEAD(recode_stream),
1609 &stream->inst_sect)) ||
1610 (ret = main_recode_copy (recode_stream,
1611 ADDR_HEAD(recode_stream),
1612 &stream->addr_sect)))
1613 { 223 {
224 ret = ENOMEM;
1614 return ret; 225 return ret;
1615 } 226 }
1616 227
1617 // This jumps to xd3_emit_hdr() 228 lru[0].blk = block0.blk;
1618 recode_stream->enc_state = ENC_FLUSH; 229 lru[0].blkno = 0;
1619 recode_stream->avail_in = stream->dec_tgtlen; 230 lru[0].size = source->onblk;
1620
1621 if (SRCORTGT (stream->dec_win_ind))
1622 {
1623 recode_stream->src = & decode_source;
1624 decode_source.srclen = stream->dec_cpylen;
1625 decode_source.srcbase = stream->dec_cpyoff;
1626 }
1627
1628 if (option_use_checksum &&
1629 (stream->dec_win_ind & VCD_ADLER32) != 0)
1630 {
1631 recode_stream->flags |= XD3_ADLER32_RECODE;
1632 recode_stream->recode_adler32 = stream->dec_adler32;
1633 }
1634 231
1635 if (option_use_appheader != 0 && 232 if (! sfile->size_known)
1636 option_appheader != NULL)
1637 { 233 {
1638 xd3_set_appheader (recode_stream, option_appheader, 234 do_src_fifo = 1;
1639 (usize_t) strlen ((char*) option_appheader));
1640 } 235 }
1641 else if (option_use_appheader != 0 && 236 else if (! do_src_fifo)
1642 option_appheader == NULL)
1643 { 237 {
1644 if (stream->dec_appheader != NULL) 238 main_blklru_list_push_back (& lru_list, & lru[0]);
1645 {
1646 xd3_set_appheader (recode_stream,
1647 stream->dec_appheader, stream->dec_appheadsz);
1648 }
1649 } 239 }
1650 240
1651 // Output loop 241 for (i = 1; i < lru_size; i += 1)
1652 for (;;)
1653 { 242 {
1654 switch((ret = xd3_encode_input (recode_stream))) 243 lru[i].blkno = (xoff_t) -1;
1655 {
1656 case XD3_INPUT: {
1657 /* finished recoding one window */
1658 stream->total_out = recode_stream->total_out;
1659 return 0;
1660 }
1661 case XD3_OUTPUT: {
1662 /* main_file_write below */
1663 break;
1664 }
1665 case XD3_GOTHEADER:
1666 case XD3_WINSTART:
1667 case XD3_WINFINISH: {
1668 /* ignore */
1669 continue;
1670 }
1671 case XD3_GETSRCBLK:
1672 case 0: {
1673 return XD3_INTERNAL;
1674 }
1675 default:
1676 return ret;
1677 }
1678 244
1679 if ((ret = main_write_output (recode_stream, ofile))) 245 if ((lru[i].blk = (uint8_t*) main_malloc (source->blksize)) == NULL)
1680 { 246 {
247 ret = ENOMEM;
1681 return ret; 248 return ret;
1682 } 249 }
1683 250
1684 xd3_consume_output (recode_stream); 251 if (! do_src_fifo)
1685 }
1686}
1687#endif /* VCDIFF_TOOLS */
1688
1689/*******************************************************************
1690 VCDIFF merging
1691 ******************************************************************/
1692
1693#if VCDIFF_TOOLS
1694/* Modifies static state. */
1695static int
1696main_init_recode_stream (void)
1697{
1698 int ret;
1699 int stream_flags = XD3_ADLER32_NOVER | XD3_SKIP_EMIT;
1700 int recode_flags;
1701 xd3_config recode_config;
1702
1703 XD3_ASSERT (recode_stream == NULL);
1704
1705 if ((recode_stream = (xd3_stream*) main_malloc(sizeof(xd3_stream))) == NULL)
1706 {
1707 return ENOMEM;
1708 }
1709
1710 recode_flags = (stream_flags & XD3_SEC_TYPE);
1711
1712 recode_config.alloc = main_alloc;
1713 recode_config.freef = main_free1;
1714
1715 xd3_init_config(&recode_config, recode_flags);
1716
1717 if ((ret = main_set_secondary_flags (&recode_config)) ||
1718 (ret = xd3_config_stream (recode_stream, &recode_config)) ||
1719 (ret = xd3_encode_init_partial (recode_stream)) ||
1720 (ret = xd3_whole_state_init (recode_stream)))
1721 {
1722 XPR(NT XD3_LIB_ERRMSG (recode_stream, ret));
1723 xd3_free_stream (recode_stream);
1724 recode_stream = NULL;
1725 return ret;
1726 }
1727
1728 return 0;
1729}
1730
1731/* This processes the sequence of -m arguments. The final input
1732 * is processed as part of the ordinary main_input() loop. */
1733static int
1734main_merge_arguments (main_merge_list* merges)
1735{
1736 int ret = 0;
1737 int count = 0;
1738 main_merge *merge = NULL;
1739 xd3_stream merge_input;
1740
1741 if (main_merge_list_empty (merges))
1742 {
1743 return 0;
1744 }
1745
1746 if ((ret = xd3_config_stream (& merge_input, NULL)) ||
1747 (ret = xd3_whole_state_init (& merge_input)))
1748 {
1749 XPR(NT XD3_LIB_ERRMSG (& merge_input, ret));
1750 return ret;
1751 }
1752
1753 merge = main_merge_list_front (merges);
1754 while (!main_merge_list_end (merges, merge))
1755 {
1756 main_file mfile;
1757 main_file_init (& mfile);
1758 mfile.filename = merge->filename;
1759 mfile.flags = RD_NONEXTERNAL;
1760
1761 if ((ret = main_file_open (& mfile, merge->filename, XO_READ)))
1762 {
1763 goto error;
1764 }
1765
1766 ret = main_input (CMD_MERGE_ARG, & mfile, NULL, NULL);
1767
1768 if (ret == 0)
1769 { 252 {
1770 if (count++ == 0) 253 main_blklru_list_push_back (& lru_free, & lru[i]);
1771 {
1772 /* The first merge source is the next merge input. */
1773 xd3_swap_whole_state (& recode_stream->whole_target,
1774 & merge_input.whole_target);
1775 }
1776 else
1777 {
1778 /* Merge the recode_stream with merge_input. */
1779 ret = xd3_merge_input_output (recode_stream,
1780 & merge_input.whole_target);
1781
1782 /* Save the next merge source in merge_input. */
1783 xd3_swap_whole_state (& recode_stream->whole_target,
1784 & merge_input.whole_target);
1785 }
1786 } 254 }
1787
1788 main_file_cleanup (& mfile);
1789
1790 if (recode_stream != NULL)
1791 {
1792 xd3_free_stream (recode_stream);
1793 main_free (recode_stream);
1794 recode_stream = NULL;
1795 }
1796
1797 if (main_bdata != NULL)
1798 {
1799 main_free (main_bdata);
1800 main_bdata = NULL;
1801 main_bsize = 0;
1802 }
1803
1804 if (ret != 0)
1805 {
1806 goto error;
1807 }
1808
1809 merge = main_merge_list_next (merge);
1810 }
1811
1812 XD3_ASSERT (merge_stream == NULL);
1813
1814 if ((merge_stream = (xd3_stream*) main_malloc (sizeof(xd3_stream))) == NULL)
1815 {
1816 ret = ENOMEM;
1817 goto error;
1818 } 255 }
1819 256
1820 if ((ret = xd3_config_stream (merge_stream, NULL)) || 257 if (sfile->size_known)
1821 (ret = xd3_whole_state_init (merge_stream)))
1822 { 258 {
1823 XPR(NT XD3_LIB_ERRMSG (& merge_input, ret)); 259 ret = xd3_set_source_and_size (stream, source, source_size);
1824 goto error;
1825 } 260 }
1826 261 else
1827 xd3_swap_whole_state (& merge_stream->whole_target,
1828 & merge_input.whole_target);
1829 ret = 0;
1830 error:
1831 xd3_free_stream (& merge_input);
1832 return ret;
1833}
1834
1835/* This processes each window of the final merge input. This routine
1836 * does not output, it buffers the entire delta into memory. */
1837static int
1838main_merge_func (xd3_stream* stream, main_file *no_write)
1839{
1840 int ret;
1841
1842 if ((ret = xd3_whole_append_window (stream)))
1843 { 262 {
1844 return ret; 263 ret = xd3_set_source (stream, source);
1845 } 264 }
1846 265
1847 return 0; 266 if (ret)
1848}
1849
1850
1851/* This is called after all windows have been read, as a final step in
1852 * main_input(). This is only called for the final merge step. */
1853static int
1854main_merge_output (xd3_stream *stream, main_file *ofile)
1855{
1856 int ret;
1857 usize_t inst_pos = 0;
1858 xoff_t output_pos = 0;
1859 xd3_source recode_source;
1860 usize_t window_num = 0;
1861 int at_least_once = 0;
1862
1863 /* merge_stream is set if there were arguments. this stream's input
1864 * needs to be applied to the merge_stream source. */
1865 if ((merge_stream != NULL) &&
1866 (ret = xd3_merge_input_output (stream,
1867 & merge_stream->whole_target)))
1868 { 267 {
1869 XPR(NT XD3_LIB_ERRMSG (stream, ret)); 268 XPR(NT XD3_LIB_ERRMSG (stream, ret));
1870 return ret; 269 return ret;
1871 } 270 }
1872 271
1873 if (option_use_appheader != 0 && 272 XD3_ASSERT (stream->src == source);
1874 option_appheader != NULL) 273 XD3_ASSERT (source->blksize == blksize);
1875 {
1876 xd3_set_appheader (recode_stream, option_appheader,
1877 (usize_t) strlen ((char*) option_appheader));
1878 }
1879
1880 /* Enter the ENC_INPUT state and bypass the next_in == NULL test
1881 * and (leftover) input buffering logic. */
1882 XD3_ASSERT(recode_stream->enc_state == ENC_INIT);
1883 recode_stream->enc_state = ENC_INPUT;
1884 recode_stream->next_in = main_bdata;
1885 recode_stream->flags |= XD3_FLUSH;
1886 274
1887 /* This encodes the entire target. */ 275 if (option_verbose)
1888 while (inst_pos < stream->whole_target.instlen || !at_least_once)
1889 { 276 {
1890 xoff_t window_start = output_pos; 277 static char srcszbuf[32];
1891 int window_srcset = 0; 278 static char srccntbuf[32];
1892 xoff_t window_srcmin = 0; 279 static char winszbuf[32];
1893 xoff_t window_srcmax = 0; 280 static char blkszbuf[32];
1894 usize_t window_pos = 0; 281 static char nbufs[32];
1895 usize_t window_size;
1896
1897 /* at_least_once ensures that we encode at least one window,
1898 * which handles the 0-byte case. */
1899 at_least_once = 1;
1900
1901 XD3_ASSERT (recode_stream->enc_state == ENC_INPUT);
1902
1903 if ((ret = xd3_encode_input (recode_stream)) != XD3_WINSTART)
1904 {
1905 XPR(NT "invalid merge state: %s\n", xd3_mainerror (ret));
1906 return XD3_INVALID;
1907 }
1908
1909 /* Window sizes must match from the input to the output, so that
1910 * target copies are in-range (and so that checksums carry
1911 * over). */
1912 XD3_ASSERT (window_num < stream->whole_target.wininfolen);
1913 window_size = stream->whole_target.wininfo[window_num].length;
1914 282
1915 /* Output position should also match. */ 283 if (sfile->size_known)
1916 if (output_pos != stream->whole_target.wininfo[window_num].offset)
1917 { 284 {
1918 XPR(NT "internal merge error: offset mismatch\n"); 285 sprintf (srcszbuf, "source size %s [%"Q"u]",
1919 return XD3_INVALID; 286 main_format_bcnt (source_size, srccntbuf),
287 source_size);
1920 } 288 }
1921 289 else
1922 if (option_use_checksum &&
1923 (stream->dec_win_ind & VCD_ADLER32) != 0)
1924 { 290 {
1925 recode_stream->flags |= XD3_ADLER32_RECODE; 291 strcpy(srcszbuf, "source size unknown");
1926 recode_stream->recode_adler32 =
1927 stream->whole_target.wininfo[window_num].adler32;
1928 } 292 }
1929 293
1930 window_num++; 294 nbufs[0] = 0;
1931 295
1932 if (main_bsize < window_size)
1933 {
1934 main_free (main_bdata);
1935 main_bdata = NULL;
1936 main_bsize = 0;
1937 if ((main_bdata = (uint8_t*)
1938 main_malloc (window_size)) == NULL)
1939 {
1940 return ENOMEM;
1941 }
1942 main_bsize = window_size;
1943 }
1944
1945 /* This encodes a single target window. */
1946 while (window_pos < window_size &&
1947 inst_pos < stream->whole_target.instlen)
1948 {
1949 xd3_winst *inst = &stream->whole_target.inst[inst_pos];
1950 usize_t take = min(inst->size, window_size - window_pos);
1951 xoff_t addr;
1952
1953 switch (inst->type)
1954 {
1955 case XD3_RUN:
1956 if ((ret = xd3_emit_run (recode_stream, window_pos, take,
1957 &stream->whole_target.adds[inst->addr])))
1958 {
1959 return ret;
1960 }
1961 break;
1962
1963 case XD3_ADD:
1964 /* Adds are implicit, put them into the input buffer. */
1965 memcpy (main_bdata + window_pos,
1966 stream->whole_target.adds + inst->addr, take);
1967 break;
1968
1969 default: /* XD3_COPY + copy mode */
1970 if (inst->mode != 0)
1971 {
1972 if (window_srcset) {
1973 window_srcmin = min(window_srcmin, inst->addr);
1974 window_srcmax = max(window_srcmax, inst->addr + take);
1975 } else {
1976 window_srcset = 1;
1977 window_srcmin = inst->addr;
1978 window_srcmax = inst->addr + take;
1979 }
1980 addr = inst->addr;
1981 }
1982 else
1983 {
1984 XD3_ASSERT (inst->addr >= window_start);
1985 addr = inst->addr - window_start;
1986 }
1987 IF_DEBUG2 (DP(RINT "[merge copy] winpos %u take %u addr %"Q"u mode %u\n",
1988 window_pos, take, addr, inst->mode));
1989 if ((ret = xd3_found_match (recode_stream, window_pos, take,
1990 addr, inst->mode != 0)))
1991 {
1992 return ret;
1993 }
1994 break;
1995 }
1996
1997 window_pos += take;
1998 output_pos += take;
1999
2000 if (take == inst->size)
2001 {
2002 inst_pos += 1;
2003 }
2004 else
2005 {
2006 /* Modify the instruction for the next pass. */
2007 if (inst->type != XD3_RUN)
2008 {
2009 inst->addr += take;
2010 }
2011 inst->size -= take;
2012 }
2013 }
2014
2015 xd3_avail_input (recode_stream, main_bdata, window_pos);
2016
2017 recode_stream->enc_state = ENC_INSTR;
2018
2019 if (window_srcset) {
2020 recode_stream->srcwin_decided = 1;
2021 recode_stream->src = &recode_source;
2022 recode_source.srclen = (usize_t)(window_srcmax - window_srcmin);
2023 recode_source.srcbase = window_srcmin;
2024 recode_stream->taroff = recode_source.srclen;
2025
2026 XD3_ASSERT (recode_source.srclen != 0);
2027 } else {
2028 recode_stream->srcwin_decided = 0;
2029 recode_stream->src = NULL;
2030 recode_stream->taroff = 0;
2031 }
2032
2033 for (;;)
2034 {
2035 switch ((ret = xd3_encode_input (recode_stream)))
2036 {
2037 case XD3_INPUT: {
2038 goto done_window;
2039 }
2040 case XD3_OUTPUT: {
2041 /* main_file_write below */
2042 break;
2043 }
2044 case XD3_GOTHEADER:
2045 case XD3_WINSTART:
2046 case XD3_WINFINISH: {
2047 /* ignore */
2048 continue;
2049 }
2050 case XD3_GETSRCBLK:
2051 case 0: {
2052 return XD3_INTERNAL;
2053 }
2054 default:
2055 return ret;
2056 }
2057
2058 if ((ret = main_write_output(recode_stream, ofile)))
2059 {
2060 return ret;
2061 }
2062
2063 xd3_consume_output (recode_stream);
2064 }
2065 done_window:
2066 (void) 0;
2067 }
2068
2069 return 0;
2070}
2071#endif
2072
2073/*******************************************************************
2074 Input decompression, output recompression
2075 ******************************************************************/
2076
2077#if EXTERNAL_COMPRESSION
2078/* This is tricky POSIX-specific code with lots of fork(), pipe(),
2079 * dup(), waitpid(), and exec() business. Most of this code
2080 * originated in PRCS1, which did automatic package-file
2081 * decompression. It works with both XD3_POSIX and XD3_STDIO file
2082 * disciplines.
2083 *
2084 * To automatically detect compressed inputs requires a child process
2085 * to reconstruct the input stream, which was advanced in order to
2086 * detect compression, because it may not be seekable. In other
2087 * words, the main program reads part of the input stream, and if it
2088 * detects a compressed input it then forks a pipe copier process,
2089 * which copies the first-read block out of the main-program's memory,
2090 * then streams the remaining compressed input into the
2091 * input-decompression pipe.
2092 */
2093
2094#include <signal.h>
2095#include <unistd.h>
2096#include <sys/stat.h>
2097#include <sys/wait.h>
2098
2099/* Remember which pipe FD is which. */
2100#define PIPE_READ_FD 0
2101#define PIPE_WRITE_FD 1
2102#define MAX_SUBPROCS 4 /* max(source + copier + output,
2103 source + copier + input + copier). */
2104static pid_t ext_subprocs[MAX_SUBPROCS];
2105static char* ext_tmpfile = NULL;
2106
2107/* Like write(), applies to a fd instead of a main_file, for the pipe
2108 * copier subprocess. Does not print an error, to facilitate ignoring
2109 * trailing garbage, see main_pipe_copier(). */
2110static int
2111main_pipe_write (int outfd, uint8_t *exist_buf, usize_t remain)
2112{
2113 int ret;
2114
2115 if ((ret = xd3_posix_io (outfd, exist_buf, remain,
2116 (xd3_posix_func*) &write, NULL)))
2117 {
2118 return ret;
2119 }
2120
2121 return 0;
2122}
2123
2124/* A simple error-reporting waitpid interface. */
2125static int
2126main_waitpid_check(pid_t pid)
2127{
2128 int status;
2129 int ret = 0;
2130
2131 if (waitpid (pid, & status, 0) < 0)
2132 {
2133 ret = get_errno ();
2134 XPR(NT "external compression [pid %d] wait: %s\n",
2135 pid, xd3_mainerror (ret));
2136 }
2137 else if (! WIFEXITED (status))
2138 {
2139 ret = ECHILD;
2140 XPR(NT "external compression [pid %d] signal %d\n",
2141 pid, WIFSIGNALED (status) ? WTERMSIG (status) : WSTOPSIG (status));
2142 }
2143 else if (WEXITSTATUS (status) != 0)
2144 {
2145 ret = ECHILD;
2146 if (option_verbose > 1) 296 if (option_verbose > 1)
2147 { 297 {
2148 /* Presumably, the error was printed by the subprocess. */ 298 sprintf(nbufs, " #bufs %u", lru_size);
2149 XPR(NT "external compression [pid %d] exit %d\n",
2150 pid, WEXITSTATUS (status));
2151 }
2152 }
2153
2154 return ret;
2155}
2156
2157/* Wait for any existing child processes to check for abnormal exit. */
2158static int
2159main_external_compression_finish (void)
2160{
2161 int i;
2162 int ret;
2163
2164 for (i = 0; i < num_subprocs; i += 1)
2165 {
2166 if (! ext_subprocs[i]) { continue; }
2167
2168 if ((ret = main_waitpid_check (ext_subprocs[i])))
2169 {
2170 return ret;
2171 }
2172 }
2173
2174 return 0;
2175}
2176
2177/* This runs as a forked process of main_input_decompress_setup() to
2178 * copy input to the decompression process. First, the available
2179 * input is copied out of the existing buffer, then the buffer is
2180 * reused to continue reading from the compressed input file. */
2181static int
2182main_pipe_copier (uint8_t *pipe_buf,
2183 usize_t pipe_bufsize,
2184 usize_t nread,
2185 main_file *ifile,
2186 int outfd)
2187{
2188 int ret;
2189 xoff_t garbage = 0;
2190
2191 /* Prevent SIGPIPE signals, allow EPIPE return values instead. This
2192 * is safe to comment-out, except that the -F flag will not work
2193 * properly (the parent would need to treat WTERMSIG(status) ==
2194 * SIGPIPE). */
2195 struct sigaction sa;
2196 sa.sa_handler = SIG_IGN;
2197 sigaction (SIGPIPE, &sa, NULL);
2198
2199 for (;;)
2200 {
2201 /* force_drain will be set when option_force and EPIPE cause us
2202 * to skip data. This is reset each time through the loop, so
2203 * the break condition below works. */
2204 int force_drain = 0;
2205 if (nread > 0 && (ret = main_pipe_write (outfd, pipe_buf, nread)))
2206 {
2207 if (option_force && ret == EPIPE)
2208 {
2209 /* This causes the loop to continue reading until nread
2210 * == 0. */
2211 garbage += nread;
2212 force_drain = 1;
2213 }
2214 else if (ret == EPIPE)
2215 {
2216 XPR(NT "external compression closed the pipe\n");
2217 if (option_verbose)
2218 {
2219 if (!option_force2)
2220 {
2221 XPR(NT "use -F to force the subprocess\n");
2222 }
2223 if (!option_force)
2224 {
2225 XPR(NT "use -f to force this process\n");
2226 }
2227 }
2228 return ret;
2229 }
2230 else
2231 {
2232 XPR(NT "pipe write failed: %s\n", xd3_mainerror (ret));
2233 return ret;
2234 }
2235 }
2236
2237 if (nread < pipe_bufsize && !force_drain)
2238 {
2239 break;
2240 } 299 }
2241 300
2242 if ((ret = main_file_read (ifile, pipe_buf, pipe_bufsize, 301 XPR(NT "source %s %s blksize %s window %s%s%s\n",
2243 & nread, "pipe read failed")) < 0) 302 sfile->filename,
2244 { 303 srcszbuf,
2245 return ret; 304 main_format_bcnt (blksize, blkszbuf),
2246 } 305 main_format_bcnt (option_srcwinsz, winszbuf),
306 nbufs,
307 do_src_fifo ? " (FIFO)" : "");
2247 } 308 }
2248 309
2249 if (garbage != 0)
2250 {
2251 XPR(NT "trailing garbage ignored in %s (%"Q"u bytes)\n",
2252 ifile->filename, garbage);
2253 }
2254 return 0; 310 return 0;
2255} 311}
2256 312
2257/* This function is called after we have read some amount of data from
2258 * the input file and detected a compressed input. Here we start a
2259 * decompression subprocess by forking twice. The first process runs
2260 * the decompression command, the second process copies data to the
2261 * input of the first. */
2262static int 313static int
2263main_input_decompress_setup (const main_extcomp *decomp, 314main_getblk_lru (xd3_source *source, xoff_t blkno,
2264 main_file *ifile, 315 main_blklru** blrup, int *is_new)
2265 uint8_t *input_buf,
2266 usize_t input_bufsize,
2267 uint8_t *pipe_buf,
2268 usize_t pipe_bufsize,
2269 usize_t pipe_avail,
2270 usize_t *nread)
2271{ 316{
2272 /* The two pipes: input and output file descriptors. */ 317 main_blklru *blru = NULL;
2273 int outpipefd[2], inpipefd[2]; 318 usize_t i;
2274 int input_fd = -1; /* The resulting input_fd (output of decompression). */
2275 pid_t decomp_id, copier_id; /* The two subprocs. */
2276 int ret;
2277
2278 outpipefd[0] = outpipefd[1] = -1;
2279 inpipefd[0] = inpipefd[1] = -1;
2280
2281 if (pipe (outpipefd) || pipe (inpipefd))
2282 {
2283 XPR(NT "pipe failed: %s\n", xd3_mainerror (ret = get_errno ()));
2284 goto pipe_cleanup;
2285 }
2286 319
2287 if ((decomp_id = fork ()) < 0) 320 (*is_new) = 0;
2288 {
2289 XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ()));
2290 goto pipe_cleanup;
2291 }
2292 321
2293 /* The first child runs the decompression process: */ 322 if (do_src_fifo)
2294 if (decomp_id == 0)
2295 { 323 {
2296 if (option_verbose > 2) 324 /* Direct lookup assumes sequential scan w/o skipping blocks. */
325 int idx = blkno % lru_size;
326 blru = & lru[idx];
327 if (blru->blkno == blkno)
2297 { 328 {
2298 XPR(NT "external decompression pid %d\n", getpid ()); 329 (*blrup) = blru;
330 return 0;
2299 } 331 }
2300 332
2301 /* Setup pipes: write to the outpipe, read from the inpipe. */ 333 if (blru->blkno != (xoff_t)-1 &&
2302 if (dup2 (outpipefd[PIPE_WRITE_FD], STDOUT_FILENO) < 0 || 334 blru->blkno != (xoff_t)(blkno - lru_size))
2303 dup2 (inpipefd[PIPE_READ_FD], STDIN_FILENO) < 0 ||
2304 close (outpipefd[PIPE_READ_FD]) ||
2305 close (outpipefd[PIPE_WRITE_FD]) ||
2306 close (inpipefd[PIPE_READ_FD]) ||
2307 close (inpipefd[PIPE_WRITE_FD]) ||
2308 execlp (decomp->decomp_cmdname, decomp->decomp_cmdname,
2309 decomp->decomp_options,
2310 option_force2 ? "-f" : NULL,
2311 NULL))
2312 { 335 {
2313 XPR(NT "child process %s failed to execute: %s\n", 336 return XD3_TOOFARBACK;
2314 decomp->decomp_cmdname, xd3_mainerror (get_errno ()));
2315 } 337 }
2316
2317 _exit (127);
2318 }
2319
2320 XD3_ASSERT(num_subprocs < MAX_SUBPROCS);
2321 ext_subprocs[num_subprocs++] = decomp_id;
2322
2323 if ((copier_id = fork ()) < 0)
2324 {
2325 XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ()));
2326 goto pipe_cleanup;
2327 }
2328
2329 /* The second child runs the copier process: */
2330 if (copier_id == 0)
2331 {
2332 int exitval = 0;
2333
2334 if (option_verbose > 2)
2335 {
2336 XPR(NT "child pipe-copier pid %d\n", getpid ());
2337 }
2338
2339 if (close (inpipefd[PIPE_READ_FD]) ||
2340 main_pipe_copier (pipe_buf, pipe_bufsize, pipe_avail,
2341 ifile, inpipefd[PIPE_WRITE_FD]) ||
2342 close (inpipefd[PIPE_WRITE_FD]))
2343 {
2344 XPR(NT "child copier process failed: %s\n",
2345 xd3_mainerror (get_errno ()));
2346 exitval = 1;
2347 }
2348
2349 _exit (exitval);
2350 }
2351
2352 XD3_ASSERT(num_subprocs < MAX_SUBPROCS);
2353 ext_subprocs[num_subprocs++] = copier_id;
2354
2355 /* The parent closes both pipes after duplicating the output of
2356 * compression. */
2357 input_fd = dup (outpipefd[PIPE_READ_FD]);
2358
2359 if (input_fd < 0 ||
2360 main_file_close (ifile) ||
2361 close (outpipefd[PIPE_READ_FD]) ||
2362 close (outpipefd[PIPE_WRITE_FD]) ||
2363 close (inpipefd[PIPE_READ_FD]) ||
2364 close (inpipefd[PIPE_WRITE_FD]))
2365 {
2366 XPR(NT "dup/close failed: %s\n", xd3_mainerror (ret = get_errno ()));
2367 goto pipe_cleanup;
2368 }
2369
2370#if XD3_STDIO
2371 /* Note: fdopen() acquires the fd, closes it when finished. */
2372 if ((ifile->file = fdopen (input_fd, "r")) == NULL)
2373 {
2374 XPR(NT "fdopen failed: %s\n", xd3_mainerror (ret = get_errno ()));
2375 goto pipe_cleanup;
2376 }
2377
2378#elif XD3_POSIX
2379 ifile->file = input_fd;
2380#endif
2381
2382 ifile->compressor = decomp;
2383
2384 /* Now the input file is decompressed. */
2385 return main_file_read (ifile, input_buf, input_bufsize,
2386 nread, "input decompression failed");
2387
2388 pipe_cleanup:
2389 close (input_fd);
2390 close (outpipefd[PIPE_READ_FD]);
2391 close (outpipefd[PIPE_WRITE_FD]);
2392 close (inpipefd[PIPE_READ_FD]);
2393 close (inpipefd[PIPE_WRITE_FD]);
2394 return ret;
2395}
2396
2397
2398/* This routine is called when the first buffer of input data is read
2399 * by the main program (unless input decompression is disabled by
2400 * command-line option). If it recognizes the magic number of a known
2401 * input type it invokes decompression.
2402 *
2403 * Skips decompression if the decompression type or the file type is
2404 * RD_NONEXTERNAL.
2405 *
2406 * Behaves exactly like main_file_read, otherwise.
2407 *
2408 * This function uses a separate buffer to read the first small block
2409 * of input. If a compressed input is detected, the separate buffer
2410 * is passed to the pipe copier. This avoids using the same size
2411 * buffer in both cases. */
2412static int
2413main_secondary_decompress_check (main_file *file,
2414 uint8_t *input_buf,
2415 usize_t input_size,
2416 usize_t *nread)
2417{
2418 int ret;
2419 usize_t i;
2420 usize_t try_read = min (input_size, XD3_ALLOCSIZE);
2421 usize_t check_nread;
2422 uint8_t check_buf[XD3_ALLOCSIZE]; /* TODO: stack limit */
2423 const main_extcomp *decompressor = NULL;
2424
2425 if ((ret = main_file_read (file, check_buf,
2426 try_read,
2427 & check_nread, "input read failed")))
2428 {
2429 return ret;
2430 }
2431
2432 if (file->flags & RD_DECOMPSET)
2433 {
2434 /* This allows the application header to override the magic
2435 * number, for whatever reason. */
2436 decompressor = file->compressor;
2437 } 338 }
2438 else 339 else
2439 { 340 {
2440 for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) 341 /* Sequential search through LRU. */
2441 { 342 for (i = 0; i < lru_size; i += 1)
2442 const main_extcomp *decomp = & extcomp_types[i];
2443
2444 if (check_nread > decomp->magic_size)
2445 {
2446 /* The following expr checks if we are trying to read a
2447 * VCDIFF input, in which case do not treat it as
2448 * "secondary" decompression. */
2449 int skip_this_type = (decomp->flags & RD_NONEXTERNAL) &&
2450 (file->flags & RD_NONEXTERNAL);
2451
2452 if (skip_this_type)
2453 {
2454 continue;
2455 }
2456
2457 if (memcmp (check_buf, decomp->magic, decomp->magic_size) == 0)
2458 {
2459 decompressor = decomp;
2460 break;
2461 }
2462 }
2463 }
2464 }
2465
2466 if (decompressor != NULL)
2467 {
2468 if (! option_quiet)
2469 { 343 {
2470 XPR(NT "externally compressed input: %s %s%s < %s\n", 344 blru = & lru[i];
2471 decompressor->decomp_cmdname, 345 if (blru->blkno == blkno)
2472 decompressor->decomp_options,
2473 (option_force2 ? " -f" : ""),
2474 file->filename);
2475 if (file->flags & RD_MAININPUT)
2476 { 346 {
2477 XPR(NT 347 main_blklru_list_remove (blru);
2478 "WARNING: the encoder is automatically decompressing the input file;\n"); 348 main_blklru_list_push_back (& lru_list, blru);
2479 XPR(NT 349 (*blrup) = blru;
2480 "WARNING: the decoder will automatically recompress the output file;\n"); 350 return 0;
2481 XPR(NT
2482 "WARNING: this may result in different compressed data and checksums\n");
2483 XPR(NT
2484 "WARNING: despite being identical data; if this is an issue, use -D\n");
2485 XPR(NT
2486 "WARNING: to avoid decompression and/or use -R to avoid recompression\n");
2487 XPR(NT
2488 "WARNING: and/or manually decompress the input file; if you know the\n");
2489 XPR(NT
2490 "WARNING: compression settings that will produce identical output\n");
2491 XPR(NT
2492 "WARNING: you may set those flags using the environment (e.g., GZIP=-9)\n");
2493 } 351 }
2494 } 352 }
2495
2496 file->size_known = 0;
2497 return main_input_decompress_setup (decompressor, file,
2498 input_buf, input_size,
2499 check_buf, XD3_ALLOCSIZE,
2500 check_nread, nread);
2501 }
2502
2503 /* Now read the rest of the input block. */
2504 (*nread) = 0;
2505
2506 if (check_nread == try_read)
2507 {
2508 ret = main_file_read (file,
2509 input_buf + try_read,
2510 input_size - try_read,
2511 nread,
2512 "input read failed");
2513 }
2514
2515 memcpy (input_buf, check_buf, check_nread);
2516
2517 (*nread) += check_nread;
2518
2519 return 0;
2520}
2521
2522/* Initiate re-compression of the output stream. This is easier than
2523 * input decompression because we know beforehand that the stream will
2524 * be compressed, whereas the input has already been read when we
2525 * decide it should be decompressed. Thus, it only requires one
2526 * subprocess and one pipe. */
2527static int
2528main_recompress_output (main_file *ofile)
2529{
2530 pid_t recomp_id; /* One subproc. */
2531 int pipefd[2]; /* One pipe. */
2532 int output_fd = -1;
2533 int ret;
2534 const main_extcomp *recomp = ofile->compressor;
2535
2536 pipefd[0] = pipefd[1] = -1;
2537
2538 if (pipe (pipefd))
2539 {
2540 XPR(NT "pipe failed: %s\n", xd3_mainerror (ret = get_errno ()));
2541 goto pipe_cleanup;
2542 }
2543
2544 if ((recomp_id = fork ()) < 0)
2545 {
2546 XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ()));
2547 goto pipe_cleanup;
2548 }
2549
2550 /* The child runs the recompression process: */
2551 if (recomp_id == 0)
2552 {
2553 if (option_verbose > 2)
2554 {
2555 XPR(NT "external recompression pid %d\n", getpid ());
2556 }
2557
2558 /* Setup pipes: write to the output file, read from the pipe. */
2559 if (dup2 (XFNO (ofile), STDOUT_FILENO) < 0 ||
2560 dup2 (pipefd[PIPE_READ_FD], STDIN_FILENO) < 0 ||
2561 close (pipefd[PIPE_READ_FD]) ||
2562 close (pipefd[PIPE_WRITE_FD]) ||
2563 execlp (recomp->recomp_cmdname, recomp->recomp_cmdname,
2564 recomp->recomp_options,
2565 option_force2 ? "-f" : NULL,
2566 NULL))
2567 {
2568 XPR(NT "child process %s failed to execute: %s\n",
2569 recomp->recomp_cmdname, xd3_mainerror (get_errno ()));
2570 }
2571
2572 _exit (127);
2573 }
2574
2575 XD3_ASSERT(num_subprocs < MAX_SUBPROCS);
2576 ext_subprocs[num_subprocs++] = recomp_id;
2577
2578 /* The parent closes both pipes after duplicating the output-fd for
2579 * writing to the compression pipe. */
2580 output_fd = dup (pipefd[PIPE_WRITE_FD]);
2581
2582 if (output_fd < 0 ||
2583 main_file_close (ofile) ||
2584 close (pipefd[PIPE_READ_FD]) ||
2585 close (pipefd[PIPE_WRITE_FD]))
2586 {
2587 XPR(NT "close failed: %s\n", xd3_mainerror (ret = get_errno ()));
2588 goto pipe_cleanup;
2589 }
2590
2591#if XD3_STDIO
2592 /* Note: fdopen() acquires the fd, closes it when finished. */
2593 if ((ofile->file = fdopen (output_fd, "w")) == NULL)
2594 {
2595 XPR(NT "fdopen failed: %s\n", xd3_mainerror (ret = get_errno ()));
2596 goto pipe_cleanup;
2597 }
2598
2599#elif XD3_POSIX
2600 ofile->file = output_fd;
2601#endif
2602
2603 /* Now the output file will be compressed. */
2604 return 0;
2605
2606 pipe_cleanup:
2607 close (output_fd);
2608 close (pipefd[PIPE_READ_FD]);
2609 close (pipefd[PIPE_WRITE_FD]);
2610 return ret;
2611}
2612#endif /* EXTERNAL_COMPRESSION */
2613
2614/* Identify the compressor that was used based on its ident string,
2615 * which is passed in the application header. */
2616static const main_extcomp*
2617main_ident_compressor (const char *ident)
2618{
2619 usize_t i;
2620
2621 for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)
2622 {
2623 if (strcmp (extcomp_types[i].ident, ident) == 0)
2624 {
2625 return & extcomp_types[i];
2626 }
2627 }
2628
2629 return NULL;
2630}
2631
2632/* Return the main_extcomp record to use for this identifier, if possible. */
2633static const main_extcomp*
2634main_get_compressor (const char *ident)
2635{
2636 const main_extcomp *ext = main_ident_compressor (ident);
2637
2638 if (ext == NULL)
2639 {
2640 if (! option_quiet)
2641 {
2642 XPR(NT "warning: cannot recompress output: "
2643 "unrecognized external compression ID: %s\n", ident);
2644 }
2645 return NULL;
2646 }
2647 else if (! EXTERNAL_COMPRESSION)
2648 {
2649 if (! option_quiet)
2650 {
2651 XPR(NT "warning: external support not compiled: "
2652 "original input was compressed: %s\n", ext->recomp_cmdname);
2653 }
2654 return NULL;
2655 }
2656 else
2657 {
2658 return ext;
2659 } 353 }
2660}
2661
2662/*********************************************************************
2663 APPLICATION HEADER
2664 *******************************************************************/
2665
2666#if XD3_ENCODER
2667static const char*
2668main_apphead_string (const char* x)
2669{
2670 const char *y;
2671
2672 if (x == NULL) { return ""; }
2673
2674 if (strcmp (x, "/dev/stdin") == 0 ||
2675 strcmp (x, "/dev/stdout") == 0 ||
2676 strcmp (x, "/dev/stderr") == 0) { return "-"; }
2677
2678 // TODO: this is not portable
2679 return (y = strrchr (x, '/')) == NULL ? x : y + 1;
2680}
2681
2682static int
2683main_set_appheader (xd3_stream *stream, main_file *input, main_file *sfile)
2684{
2685 /* The user may disable the application header. Once the appheader
2686 * is set, this disables setting it again. */
2687 if (appheader_used || ! option_use_appheader) { return 0; }
2688 354
2689 /* The user may specify the application header, otherwise format the 355 if (do_src_fifo)
2690 default header. */
2691 if (option_appheader)
2692 { 356 {
2693 appheader_used = option_appheader; 357 int idx = blkno % lru_size;
358 blru = & lru[idx];
2694 } 359 }
2695 else 360 else
2696 { 361 {
2697 const char *iname; 362 if (! main_blklru_list_empty (& lru_free))
2698 const char *icomp;
2699 const char *sname;
2700 const char *scomp;
2701 usize_t len;
2702
2703 iname = main_apphead_string (input->filename);
2704 icomp = (input->compressor == NULL) ? "" : input->compressor->ident;
2705 len = (usize_t) strlen (iname) + (usize_t) strlen (icomp) + 2;
2706
2707 if (sfile->filename != NULL)
2708 {
2709 sname = main_apphead_string (sfile->filename);
2710 scomp = (sfile->compressor == NULL) ? "" : sfile->compressor->ident;
2711 len += (usize_t) strlen (sname) + (usize_t) strlen (scomp) + 2;
2712 }
2713 else
2714 {
2715 sname = scomp = "";
2716 }
2717
2718 if ((appheader_used = (uint8_t*) main_malloc (len)) == NULL)
2719 {
2720 return ENOMEM;
2721 }
2722
2723 if (sfile->filename == NULL)
2724 { 363 {
2725 sprintf ((char*)appheader_used, "%s/%s", iname, icomp); 364 blru = main_blklru_list_pop_front (& lru_free);
365 main_blklru_list_push_back (& lru_list, blru);
2726 } 366 }
2727 else 367 else
2728 { 368 {
2729 sprintf ((char*)appheader_used, "%s/%s/%s/%s", 369 XD3_ASSERT (! main_blklru_list_empty (& lru_list));
2730 iname, icomp, sname, scomp); 370 blru = main_blklru_list_pop_front (& lru_list);
371 main_blklru_list_push_back (& lru_list, blru);
2731 } 372 }
2732 } 373 }
2733 374
2734 xd3_set_appheader (stream, appheader_used, 375 lru_filled += 1;
2735 (usize_t) strlen ((char*)appheader_used)); 376 (*is_new) = 1;
2736 377 (*blrup) = blru;
378 blru->blkno = blkno;
2737 return 0; 379 return 0;
2738} 380}
2739#endif
2740
2741static void
2742main_get_appheader_params (main_file *file, char **parsed,
2743 int output, const char *type,
2744 main_file *other)
2745{
2746 /* Set the filename if it was not specified. If output, option_stdout (-c)
2747 * overrides. */
2748 if (file->filename == NULL &&
2749 ! (output && option_stdout) &&
2750 strcmp (parsed[0], "-") != 0)
2751 {
2752 file->filename = parsed[0];
2753
2754 if (other->filename != NULL) {
2755 /* Take directory from the other file, if it has one. */
2756 /* TODO: This results in nonsense names like /dev/foo.tar.gz
2757 * and probably the filename-default logic interferes with
2758 * multi-file operation and the standard file extension?
2759 * Possibly the name header is bad, should be off by default.
2760 * Possibly we just want to remember external/compression
2761 * settings. */
2762 const char *last_slash = strrchr(other->filename, '/');
2763
2764 if (last_slash != NULL) {
2765 usize_t dlen = (usize_t) (last_slash - other->filename);
2766
2767 XD3_ASSERT(file->filename_copy == NULL);
2768 file->filename_copy =
2769 (char*) main_malloc(dlen + 2 + (usize_t) strlen(file->filename));
2770
2771 strncpy(file->filename_copy, other->filename, dlen);
2772 file->filename_copy[dlen] = '/';
2773 strcpy(file->filename_copy + dlen + 1, parsed[0]);
2774
2775 file->filename = file->filename_copy;
2776 }
2777 }
2778
2779 if (! option_quiet)
2780 {
2781 XPR(NT "using default %s filename: %s\n", type, file->filename);
2782 }
2783 }
2784
2785 /* Set the compressor, initiate de/recompression later. */
2786 if (file->compressor == NULL && *parsed[1] != 0)
2787 {
2788 file->flags |= RD_DECOMPSET;
2789 file->compressor = main_get_compressor (parsed[1]);
2790 }
2791}
2792
2793static void
2794main_get_appheader (xd3_stream *stream, main_file *ifile,
2795 main_file *output, main_file *sfile)
2796{
2797 uint8_t *apphead;
2798 usize_t appheadsz;
2799 int ret;
2800
2801 /* The user may disable the application header. Once the appheader
2802 * is set, this disables setting it again. */
2803 if (! option_use_appheader) { return; }
2804
2805 ret = xd3_get_appheader (stream, & apphead, & appheadsz);
2806
2807 /* Ignore failure, it only means we haven't received a header yet. */
2808 if (ret != 0) { return; }
2809
2810 if (appheadsz > 0)
2811 {
2812 char *start = (char*)apphead;
2813 char *slash;
2814 int place = 0;
2815 char *parsed[4];
2816
2817 memset (parsed, 0, sizeof (parsed));
2818 381
2819 while ((slash = strchr (start, '/')) != NULL)
2820 {
2821 *slash = 0;
2822 parsed[place++] = start;
2823 start = slash + 1;
2824 }
2825
2826 parsed[place++] = start;
2827
2828 /* First take the output parameters. */
2829 if (place == 2 || place == 4)
2830 {
2831 main_get_appheader_params (output, parsed, 1, "output", ifile);
2832 }
2833
2834 /* Then take the source parameters. */
2835 if (place == 4)
2836 {
2837 main_get_appheader_params (sfile, parsed+2, 0, "source", ifile);
2838 }
2839 }
2840
2841 option_use_appheader = 0;
2842 return;
2843}
2844
2845/*********************************************************************
2846 Main I/O routines
2847 **********************************************************************/
2848
2849/* This function acts like the above except it may also try to
2850 * recognize a compressed input (source or target) when the first
2851 * buffer of data is read. The EXTERNAL_COMPRESSION code is called to
2852 * search for magic numbers. */
2853static int 382static int
2854main_read_primary_input (main_file *file, 383main_read_seek_source (xd3_stream *stream,
2855 uint8_t *buf, 384 xd3_source *source,
2856 usize_t size, 385 xoff_t blkno) {
2857 usize_t *nread) 386 xoff_t pos = blkno * source->blksize;
2858{ 387 main_file *sfile = (main_file*) source->ioh;
2859#if EXTERNAL_COMPRESSION 388 main_blklru *blru;
2860 if (option_decompress_inputs && file->flags & RD_FIRST) 389 int is_new;
2861 { 390 usize_t nread = 0;
2862 file->flags &= ~RD_FIRST; 391 int ret = 0;
2863 return main_secondary_decompress_check (file, buf, size, nread);
2864 }
2865#endif
2866
2867 return main_file_read (file, buf, size, nread, "input read failed");
2868}
2869
2870/* Open the main output file, sets a default file name, initiate
2871 * recompression. This function is expected to fprint any error
2872 * messages. */
2873static int
2874main_open_output (xd3_stream *stream, main_file *ofile)
2875{
2876 int ret;
2877
2878 if (option_no_output)
2879 {
2880 return 0;
2881 }
2882 392
2883 if (ofile->filename == NULL) 393 if (!sfile->seek_failed)
2884 { 394 {
2885 XSTDOUT_XF (ofile); 395 ret = main_file_seek (sfile, pos);
2886 396
2887 if (option_verbose > 1) 397 if (ret == 0)
2888 { 398 {
2889 XPR(NT "using standard output: %s\n", ofile->filename); 399 sfile->source_position = pos;
2890 } 400 }
2891 } 401 }
2892 else 402
403 if (sfile->seek_failed || ret != 0)
2893 { 404 {
2894 /* Stat the file to check for overwrite. */ 405 /* For an unseekable file (or other seek error, does it
2895 if (option_force == 0 && main_file_exists (ofile)) 406 * matter?) */
407 if (sfile->source_position > pos)
2896 { 408 {
409 /* Could assert !IS_ENCODE(), this shouldn't happen
410 * because of do_src_fifo during encode. */
2897 if (!option_quiet) 411 if (!option_quiet)
2898 { 412 {
2899 XPR(NT "to overwrite output file specify -f: %s\n", 413 XPR(NT "source can't seek backwards; requested block offset "
2900 ofile->filename); 414 "%"Q"u source position is %"Q"u\n",
415 pos, sfile->source_position);
2901 } 416 }
2902 return EEXIST;
2903 }
2904 417
2905 if ((ret = main_file_open (ofile, ofile->filename, XO_WRITE))) 418 sfile->seek_failed = 1;
2906 { 419 stream->msg = "non-seekable source: "
2907 return ret; 420 "copy is too far back (try raising -B)";
421 return XD3_TOOFARBACK;
2908 } 422 }
2909 423
2910 if (option_verbose > 1) { XPR(NT "output %s\n", ofile->filename); } 424 /* There's a chance here, that an genuine lseek error will cause
2911 } 425 * xdelta3 to shift into non-seekable mode, entering a degraded
2912 426 * condition. */
2913#if EXTERNAL_COMPRESSION 427 if (!sfile->seek_failed && option_verbose)
2914 /* Do output recompression. */
2915 if (ofile->compressor != NULL && option_recompress_outputs == 1)
2916 {
2917 if (! option_quiet)
2918 { 428 {
2919 XPR(NT "externally compressed output: %s %s%s > %s\n", 429 XPR(NT "source can't seek, will use FIFO for %s\n",
2920 ofile->compressor->recomp_cmdname, 430 sfile->filename);
2921 ofile->compressor->recomp_options,
2922 (option_force2 ? " -f" : ""),
2923 ofile->filename);
2924 }
2925 431
2926 if ((ret = main_recompress_output (ofile))) 432 if (option_verbose > 1)
2927 { 433 {
2928 return ret; 434 XPR(NT "seek error at offset %"Q"u: %s\n",
435 pos, xd3_mainerror (ret));
436 }
2929 } 437 }
2930 }
2931#endif
2932
2933 return 0;
2934}
2935 438
2936static usize_t 439 sfile->seek_failed = 1;
2937main_get_winsize (main_file *ifile) {
2938 xoff_t file_size = 0;
2939 usize_t size = option_winsize;
2940 static char iszbuf[32];
2941 440
2942 if (main_file_stat (ifile, &file_size) == 0) 441 while (sfile->source_position < pos)
2943 {
2944 size = (usize_t) min(file_size, (xoff_t) size);
2945 }
2946
2947 size = max(size, XD3_ALLOCSIZE);
2948
2949 if (option_verbose > 1)
2950 {
2951 XPR(NT "input %s window size %s\n",
2952 ifile->filename,
2953 main_format_bcnt (size, iszbuf));
2954 }
2955
2956 return size;
2957}
2958
2959/*********************************************************************
2960 Main routines
2961 ********************************************************************/
2962
2963/* This is a generic input function. It calls the xd3_encode_input or
2964 * xd3_decode_input functions and makes calls to the various input
2965 * handling routines above, which coordinate external decompression.
2966 */
2967static int
2968main_input (xd3_cmd cmd,
2969 main_file *ifile,
2970 main_file *ofile,
2971 main_file *sfile)
2972{
2973 int ret;
2974 xd3_stream stream;
2975 usize_t nread = 0;
2976 usize_t winsize;
2977 int stream_flags = 0;
2978 xd3_config config;
2979 xd3_source source;
2980 xoff_t last_total_in = 0;
2981 xoff_t last_total_out = 0;
2982 long start_time;
2983 int stdout_only = 0;
2984 int (*input_func) (xd3_stream*);
2985 int (*output_func) (xd3_stream*, main_file *);
2986
2987 memset (& stream, 0, sizeof (stream));
2988 memset (& source, 0, sizeof (source));
2989 memset (& config, 0, sizeof (config));
2990
2991 config.alloc = main_alloc;
2992 config.freef = main_free1;
2993
2994 config.iopt_size = option_iopt_size;
2995 config.sprevsz = option_sprevsz;
2996
2997 do_src_fifo = 0;
2998
2999 start_time = get_millisecs_now ();
3000
3001 if (option_use_checksum) { stream_flags |= XD3_ADLER32; }
3002
3003 /* main_input setup. */
3004 switch ((int) cmd)
3005 {
3006#if VCDIFF_TOOLS
3007 if (1) { case CMD_PRINTHDR: stream_flags |= XD3_JUST_HDR; }
3008 else if (1) { case CMD_PRINTHDRS: stream_flags |= XD3_SKIP_WINDOW; }
3009 else { case CMD_PRINTDELTA: stream_flags |= XD3_SKIP_EMIT; }
3010 ifile->flags |= RD_NONEXTERNAL;
3011 input_func = xd3_decode_input;
3012 output_func = main_print_func;
3013 stream_flags |= XD3_ADLER32_NOVER;
3014 stdout_only = 1;
3015 break;
3016
3017 case CMD_RECODE:
3018 case CMD_MERGE:
3019 case CMD_MERGE_ARG:
3020 /* No source will be read */
3021 stream_flags |= XD3_ADLER32_NOVER | XD3_SKIP_EMIT;
3022 ifile->flags |= RD_NONEXTERNAL;
3023 input_func = xd3_decode_input;
3024
3025 if ((ret = main_init_recode_stream ()))
3026 {
3027 return EXIT_FAILURE;
3028 }
3029
3030 if (cmd == CMD_RECODE) { output_func = main_recode_func; }
3031 else { output_func = main_merge_func; }
3032 break;
3033#endif /* VCDIFF_TOOLS */
3034
3035#if XD3_ENCODER
3036 case CMD_ENCODE:
3037 do_src_fifo = 1;
3038 input_func = xd3_encode_input;
3039 output_func = main_write_output;
3040
3041 if (option_no_compress) { stream_flags |= XD3_NOCOMPRESS; }
3042 if (option_use_altcodetable) { stream_flags |= XD3_ALT_CODE_TABLE; }
3043 if (option_smatch_config)
3044 { 442 {
3045 const char *s = option_smatch_config; 443 xoff_t skip_blkno;
3046 char *e; 444 usize_t skip_offset;
3047 int values[XD3_SOFTCFG_VARCNT]; 445
3048 int got; 446 xd3_blksize_div (sfile->source_position, source,
447 &skip_blkno, &skip_offset);
3049 448
3050 config.smatch_cfg = XD3_SMATCH_SOFT; 449 /* Read past unused data */
450 XD3_ASSERT (pos - sfile->source_position >= source->blksize);
451 XD3_ASSERT (skip_offset == 0);
3051 452
3052 for (got = 0; got < XD3_SOFTCFG_VARCNT; got += 1, s = e + 1) 453 if ((ret = main_getblk_lru (source, skip_blkno,
454 & blru, & is_new)))
3053 { 455 {
3054 values[got] = strtol (s, &e, 10); 456 return ret;
3055
3056 if ((values[got] < 0) ||
3057 (e == s) ||
3058 (got < XD3_SOFTCFG_VARCNT-1 && *e == 0) ||
3059 (got == XD3_SOFTCFG_VARCNT-1 && *e != 0))
3060 {
3061 XPR(NT "invalid string match specifier (-C) %d: %s\n",
3062 got, s);
3063 return EXIT_FAILURE;
3064 }
3065 } 457 }
3066 458
3067 config.smatcher_soft.large_look = values[0]; 459 XD3_ASSERT (is_new);
3068 config.smatcher_soft.large_step = values[1]; 460
3069 config.smatcher_soft.small_look = values[2]; 461 if (option_verbose > 1)
3070 config.smatcher_soft.small_chain = values[3];
3071 config.smatcher_soft.small_lchain = values[4];
3072 config.smatcher_soft.max_lazy = values[5];
3073 config.smatcher_soft.long_enough = values[6];
3074 }
3075 else
3076 {
3077 if (option_verbose > 2)
3078 { 462 {
3079 XPR(NT "compression level: %d\n", option_level); 463 XPR(NT "non-seekable source skipping %"Q"u bytes @ %"Q"u\n",
464 pos - sfile->source_position,
465 sfile->source_position);
3080 } 466 }
3081 if (option_level == 0) 467
468 if ((ret = main_read_primary_input (sfile,
469 (uint8_t*) blru->blk,
470 source->blksize,
471 & nread)))
3082 { 472 {
3083 stream_flags |= XD3_NOCOMPRESS; 473 return ret;
3084 config.smatch_cfg = XD3_SMATCH_FASTEST;
3085 } 474 }
3086 else if (option_level == 1)
3087 { config.smatch_cfg = XD3_SMATCH_FASTEST; }
3088 else if (option_level == 2)
3089 { config.smatch_cfg = XD3_SMATCH_FASTER; }
3090 else if (option_level <= 5)
3091 { config.smatch_cfg = XD3_SMATCH_FAST; }
3092 else if (option_level == 6)
3093 { config.smatch_cfg = XD3_SMATCH_DEFAULT; }
3094 else
3095 { config.smatch_cfg = XD3_SMATCH_SLOW; }
3096 }
3097 break;
3098#endif
3099 case CMD_DECODE:
3100 if (option_use_checksum == 0) { stream_flags |= XD3_ADLER32_NOVER; }
3101 ifile->flags |= RD_NONEXTERNAL;
3102 input_func = xd3_decode_input;
3103 output_func = main_write_output;
3104 break;
3105 default:
3106 XPR(NT "internal error\n");
3107 return EXIT_FAILURE;
3108 }
3109
3110 main_bsize = winsize = main_get_winsize (ifile);
3111
3112 if ((main_bdata = (uint8_t*) main_malloc (winsize)) == NULL)
3113 {
3114 return EXIT_FAILURE;
3115 }
3116 475
3117 config.winsize = winsize; 476 if (nread != source->blksize)
3118 config.srcwin_maxsz = option_srcwinsz;
3119 config.getblk = main_getblk_func;
3120 config.flags = stream_flags;
3121
3122 if ((ret = main_set_secondary_flags (&config)) ||
3123 (ret = xd3_config_stream (& stream, & config)))
3124 {
3125 XPR(NT XD3_LIB_ERRMSG (& stream, ret));
3126 return EXIT_FAILURE;
3127 }
3128
3129#if VCDIFF_TOOLS
3130 if ((cmd == CMD_MERGE || cmd == CMD_MERGE_ARG) &&
3131 (ret = xd3_whole_state_init (& stream)))
3132 {
3133 XPR(NT XD3_LIB_ERRMSG (& stream, ret));
3134 return EXIT_FAILURE;
3135 }
3136#endif
3137
3138 if (cmd != CMD_DECODE)
3139 {
3140 /* When not decoding, set source now. The decoder delays this
3141 * step until XD3_GOTHEADER. */
3142 if (sfile && sfile->filename != NULL)
3143 {
3144 if ((ret = main_set_source (& stream, cmd, sfile, & source)))
3145 { 477 {
3146 return EXIT_FAILURE; 478 IF_DEBUG1 (DP(RINT "[getblk] short skip block nread = %u\n",
479 nread));
480 stream->msg = "non-seekable input is short";
481 return XD3_INVALID_INPUT;
3147 } 482 }
3148 483
3149 XD3_ASSERT(stream.src != NULL); 484 sfile->source_position += nread;
3150 } 485 blru->size = nread;
3151 }
3152
3153 if (cmd == CMD_PRINTHDR ||
3154 cmd == CMD_PRINTHDRS ||
3155 cmd == CMD_PRINTDELTA ||
3156 cmd == CMD_RECODE)
3157 {
3158 if (sfile->filename == NULL)
3159 {
3160 allow_fake_source = 1;
3161 sfile->filename = "<placeholder>";
3162 main_set_source (& stream, cmd, sfile, & source);
3163 }
3164 }
3165
3166 /* This times each window. */
3167 get_millisecs_since ();
3168
3169 /* Main input loop. */
3170 do
3171 {
3172 xoff_t input_offset;
3173 xoff_t input_remain;
3174 usize_t try_read;
3175
3176 input_offset = ifile->nread;
3177
3178 input_remain = XOFF_T_MAX - input_offset;
3179
3180 try_read = (usize_t) min ((xoff_t) config.winsize, input_remain);
3181
3182 if ((ret = main_read_primary_input (ifile, main_bdata,
3183 try_read, & nread)))
3184 {
3185 return EXIT_FAILURE;
3186 }
3187 486
3188 /* If we've reached EOF tell the stream to flush. */ 487 IF_DEBUG1 (DP(RINT "[getblk] skip blkno %"Q"u size %u\n",
3189 if (nread < try_read) 488 skip_blkno, blru->size));
3190 {
3191 stream.flags |= XD3_FLUSH;
3192 }
3193 489
3194#if XD3_ENCODER 490 XD3_ASSERT (sfile->source_position <= pos);
3195 /* After the first main_read_primary_input completes, we know
3196 * all the information needed to encode the application
3197 * header. */
3198 if (cmd == CMD_ENCODE &&
3199 (ret = main_set_appheader (& stream, ifile, sfile)))
3200 {
3201 return EXIT_FAILURE;
3202 } 491 }
3203#endif
3204 xd3_avail_input (& stream, main_bdata, nread);
3205
3206 /* If we read zero bytes after encoding at least one window... */
3207 if (nread == 0 && stream.current_window > 0) {
3208 break;
3209 }
3210
3211 again:
3212 ret = input_func (& stream);
3213
3214 switch (ret)
3215 {
3216 case XD3_INPUT:
3217 continue;
3218
3219 case XD3_GOTHEADER:
3220 {
3221 XD3_ASSERT (stream.current_window == 0);
3222
3223 /* Need to process the appheader as soon as possible. It may
3224 * contain a suggested default filename/decompression routine for
3225 * the ofile, and it may contain default/decompression routine for
3226 * the sources. */
3227 if (cmd == CMD_DECODE)
3228 {
3229 /* May need to set the sfile->filename if none was given. */
3230 main_get_appheader (& stream, ifile, ofile, sfile);
3231
3232 /* Now open the source file. */
3233 if ((sfile->filename != NULL) &&
3234 (ret = main_set_source (& stream, cmd, sfile, & source)))
3235 {
3236 return EXIT_FAILURE;
3237 }
3238 }
3239 }
3240 /* FALLTHROUGH */
3241 case XD3_WINSTART:
3242 {
3243 /* e.g., set or unset XD3_SKIP_WINDOW. */
3244 goto again;
3245 }
3246
3247 case XD3_OUTPUT:
3248 {
3249 /* Defer opening the output file until the stream produces its
3250 * first output for both encoder and decoder, this way we
3251 * delay long enough for the decoder to receive the
3252 * application header. (Or longer if there are skipped
3253 * windows, but I can't think of any reason not to delay
3254 * open.) */
3255 if (ofile != NULL &&
3256 ! main_file_isopen (ofile) &&
3257 (ret = main_open_output (& stream, ofile)) != 0)
3258 {
3259 return EXIT_FAILURE;
3260 }
3261
3262 if ((ret = output_func (& stream, ofile)) &&
3263 (ret != PRINTHDR_SPECIAL))
3264 {
3265 return EXIT_FAILURE;
3266 }
3267
3268 if (ret == PRINTHDR_SPECIAL)
3269 {
3270 xd3_abort_stream (& stream);
3271 ret = EXIT_SUCCESS;
3272 goto done;
3273 }
3274
3275 ret = 0;
3276
3277 xd3_consume_output (& stream);
3278 goto again;
3279 }
3280
3281 case XD3_WINFINISH:
3282 {
3283 if (IS_ENCODE (cmd) || cmd == CMD_DECODE || cmd == CMD_RECODE)
3284 {
3285 if (! option_quiet && IS_ENCODE (cmd) &&
3286 main_file_isopen (sfile))
3287 {
3288 /* Warn when no source copies are found */
3289 if (option_verbose && ! xd3_encoder_used_source (& stream))
3290 {
3291 XPR(NT "warning: input window %"Q"u..%"Q"u has "
3292 "no source copies\n",
3293 stream.current_window * winsize,
3294 (stream.current_window+1) * winsize);
3295 XD3_ASSERT (stream.src != NULL);
3296 }
3297
3298 /* Limited i-buffer size affects source copies
3299 * when the sourcewin is decided early. */
3300 if (option_verbose &&
3301 stream.srcwin_decided_early &&
3302 stream.i_slots_used > stream.iopt_size)
3303 {
3304 XPR(NT "warning: input position %"Q"u overflowed "
3305 "instruction buffer, needed %u (vs. %u), "
3306 "consider changing -I\n",
3307 stream.current_window * winsize,
3308 stream.i_slots_used, stream.iopt_size);
3309 }
3310 }
3311
3312 if (option_verbose)
3313 {
3314 char rrateavg[32], wrateavg[32], tm[32];
3315 char rdb[32], wdb[32];
3316 char trdb[32], twdb[32];
3317 char srcpos[32];
3318 long millis = get_millisecs_since ();
3319 usize_t this_read = (usize_t)(stream.total_in -
3320 last_total_in);
3321 usize_t this_write = (usize_t)(stream.total_out -
3322 last_total_out);
3323 last_total_in = stream.total_in;
3324 last_total_out = stream.total_out;
3325
3326 if (option_verbose > 1)
3327 {
3328 XPR(NT "%"Q"u: in %s (%s): out %s (%s): "
3329 "total in %s: out %s: %s: srcpos %s\n",
3330 stream.current_window,
3331 main_format_bcnt (this_read, rdb),
3332 main_format_rate (this_read, millis, rrateavg),
3333 main_format_bcnt (this_write, wdb),
3334 main_format_rate (this_write, millis, wrateavg),
3335 main_format_bcnt (stream.total_in, trdb),
3336 main_format_bcnt (stream.total_out, twdb),
3337 main_format_millis (millis, tm),
3338 main_format_bcnt (sfile->source_position, srcpos));
3339 }
3340 else
3341 {
3342 XPR(NT "%"Q"u: in %s: out %s: total in %s: "
3343 "out %s: %s\n",
3344 stream.current_window,
3345 main_format_bcnt (this_read, rdb),
3346 main_format_bcnt (this_write, wdb),
3347 main_format_bcnt (stream.total_in, trdb),
3348 main_format_bcnt (stream.total_out, twdb),
3349 main_format_millis (millis, tm));
3350 }
3351 }
3352 }
3353 goto again;
3354 }
3355
3356 default:
3357 /* input_func() error */
3358 XPR(NT XD3_LIB_ERRMSG (& stream, ret));
3359 return EXIT_FAILURE;
3360 }
3361 }
3362 while (nread == config.winsize);
3363done:
3364 /* Close the inputs. (ifile must be open, sfile may be open) */
3365 main_file_close (ifile);
3366 if (sfile != NULL)
3367 {
3368 main_file_close (sfile);
3369 }
3370
3371#if VCDIFF_TOOLS
3372 if (cmd == CMD_MERGE &&
3373 (ret = main_merge_output (& stream, ofile)))
3374 {
3375 return EXIT_FAILURE;
3376 } 492 }
3377 493
3378 if (cmd == CMD_MERGE_ARG) 494 return 0;
3379 { 495}
3380 xd3_swap_whole_state (& stream.whole_target,
3381 & recode_stream->whole_target);
3382 }
3383#endif /* VCDIFF_TOOLS */
3384
3385 /* If output file is not open yet because of delayed-open, it means
3386 * we never encountered a window in the delta, but it could have had
3387 * a VCDIFF header? TODO: solve this elsewhere. For now, it prints
3388 * "nothing to output" below, but the check doesn't happen in case
3389 * of option_no_output. */
3390 if (! option_no_output && ofile != NULL)
3391 {
3392 if (!stdout_only && ! main_file_isopen (ofile))
3393 {
3394 XPR(NT "nothing to output: %s\n", ifile->filename);
3395 return EXIT_FAILURE;
3396 }
3397 496
3398 /* Have to close the output before calling 497/* This is the callback for reading a block of source. This function
3399 * main_external_compression_finish, or else it hangs. */ 498 * is blocking and it implements a small LRU.
3400 if (main_file_close (ofile) != 0) 499 *
3401 { 500 * Note that it is possible for main_input() to handle getblk requests
3402 return EXIT_FAILURE; 501 * in a non-blocking manner. If the callback is NULL then the caller
3403 } 502 * of xd3_*_input() must handle the XD3_GETSRCBLK return value and
503 * fill the source in the same way. See xd3_getblk for details. To
504 * see an example of non-blocking getblk, see xdelta-test.h. */
505static int
506main_getblk_func (xd3_stream *stream,
507 xd3_source *source,
508 xoff_t blkno)
509{
510 int ret = 0;
511 xoff_t pos = blkno * source->blksize;
512 main_file *sfile = (main_file*) source->ioh;
513 main_blklru *blru;
514 int is_new;
515 int did_seek = 0;
516 usize_t nread = 0;
517
518 if (allow_fake_source)
519 {
520 source->curblkno = blkno;
521 source->onblk = 0;
522 source->curblk = lru[0].blk;
523 lru[0].size = 0;
524 return 0;
3404 } 525 }
3405 526
3406#if EXTERNAL_COMPRESSION 527 if ((ret = main_getblk_lru (source, blkno, & blru, & is_new)))
3407 if ((ret = main_external_compression_finish ()))
3408 { 528 {
3409 XPR(NT "external compression commands failed\n"); 529 return ret;
3410 return EXIT_FAILURE;
3411 } 530 }
3412#endif
3413 531
3414 if ((ret = xd3_close_stream (& stream))) 532 if (!is_new)
3415 { 533 {
3416 XPR(NT XD3_LIB_ERRMSG (& stream, ret)); 534 source->curblkno = blkno;
3417 return EXIT_FAILURE; 535 source->onblk = blru->size;
536 source->curblk = blru->blk;
537 lru_hits++;
538 return 0;
3418 } 539 }
3419 540
3420#if XD3_ENCODER 541 lru_misses += 1;
3421 if (option_verbose > 1 && cmd == CMD_ENCODE) 542
543 if (pos != sfile->source_position)
3422 { 544 {
3423 XPR(NT "scanner configuration: %s\n", stream.smatcher.name); 545 /* Only try to seek when the position is wrong. This means the
3424 XPR(NT "target hash table size: %u\n", stream.small_hash.size); 546 * decoder will fail when the source buffer is too small, but
3425 if (sfile != NULL && sfile->filename != NULL) 547 * only when the input is non-seekable. */
548 if ((ret = main_read_seek_source (stream, source, blkno)))
3426 { 549 {
3427 XPR(NT "source hash table size: %u\n", stream.large_hash.size); 550 return ret;
3428 } 551 }
3429 }
3430
3431 if (option_verbose > 2 && cmd == CMD_ENCODE)
3432 {
3433 XPR(NT "source copies: %"Q"u (%"Q"u bytes)\n",
3434 stream.n_scpy, stream.l_scpy);
3435 XPR(NT "target copies: %"Q"u (%"Q"u bytes)\n",
3436 stream.n_tcpy, stream.l_tcpy);
3437 XPR(NT "adds: %"Q"u (%"Q"u bytes)\n", stream.n_add, stream.l_add);
3438 XPR(NT "runs: %"Q"u (%"Q"u bytes)\n", stream.n_run, stream.l_run);
3439 }
3440#endif
3441
3442 xd3_free_stream (& stream);
3443
3444 if (option_verbose)
3445 {
3446 char tm[32];
3447 long end_time = get_millisecs_now ();
3448 xoff_t nwrite = ofile != NULL ? ofile->nwrite : 0;
3449 552
3450 XPR(NT "finished in %s; input %"Q"u output %"Q"u bytes (%0.2f%%)\n", 553 /* Indicates that another call to main_getblk_lru() may be
3451 main_format_millis (end_time - start_time, tm), 554 * needed */
3452 ifile->nread, nwrite, 100.0 * nwrite / ifile->nread); 555 did_seek = 1;
3453 } 556 }
3454 557
3455 return EXIT_SUCCESS; 558 XD3_ASSERT (sfile->source_position == pos);
3456}
3457
3458/* free memory before exit, reset single-use variables. */
3459static void
3460main_cleanup (void)
3461{
3462 if (appheader_used != NULL &&
3463 appheader_used != option_appheader)
3464 {
3465 main_free (appheader_used);
3466 appheader_used = NULL;
3467 }
3468
3469 main_free (main_bdata);
3470 main_bdata = NULL;
3471 main_bsize = 0;
3472
3473#if EXTERNAL_COMPRESSION
3474 main_free (ext_tmpfile);
3475 ext_tmpfile = NULL;
3476#endif
3477
3478 main_lru_cleanup();
3479 559
3480 if (recode_stream != NULL) 560 if (did_seek &&
561 (ret = main_getblk_lru (source, blkno, & blru, & is_new)))
3481 { 562 {
3482 xd3_free_stream (recode_stream); 563 return ret;
3483 main_free (recode_stream);
3484 recode_stream = NULL;
3485 } 564 }
3486 565
3487 if (merge_stream != NULL) 566 if ((ret = main_read_primary_input (sfile,
567 (uint8_t*) blru->blk,
568 source->blksize,
569 & nread)))
3488 { 570 {
3489 xd3_free_stream (merge_stream); 571 return ret;
3490 main_free (merge_stream);
3491 merge_stream = NULL;
3492 }
3493
3494 XD3_ASSERT (main_mallocs == 0);
3495}
3496
3497static void
3498setup_environment (int argc,
3499 char **argv,
3500 int *argc_out,
3501 char ***argv_out,
3502 char ***argv_free,
3503 char **env_free)
3504{
3505 int n, i, i0;
3506 char *p, *v = getenv("XDELTA");
3507 if (v == NULL) {
3508 (*argc_out) = argc;
3509 (*argv_out) = argv;
3510 (*argv_free) = NULL;
3511 (*env_free) = NULL;
3512 return;
3513 }
3514
3515 (*env_free) = (char*) main_malloc((usize_t) strlen(v) + 1);
3516 strcpy(*env_free, v);
3517
3518 /* Space needed for extra args, at least # of spaces */
3519 n = argc + 1;
3520 for (p = *env_free; *p != 0; ) {
3521 if (*p++ == ' ') {
3522 n++;
3523 }
3524 }
3525
3526 (*argv_free) = (char**) main_malloc(sizeof(char*) * (n + 1));
3527 (*argv_out) = (*argv_free);
3528 (*argv_out)[0] = argv[0];
3529 (*argv_out)[n] = NULL;
3530
3531 i = 1;
3532 for (p = *env_free; *p != 0; ) {
3533 (*argv_out)[i++] = p;
3534 while (*p != ' ' && *p != 0) {
3535 p++;
3536 }
3537 while (*p == ' ') {
3538 *p++ = 0;
3539 } 572 }
3540 }
3541
3542 for (i0 = 1; i0 < argc; i0++) {
3543 (*argv_out)[i++] = argv[i0];
3544 }
3545 573
3546 /* Counting spaces is an upper bound, argv stays NULL terminated. */ 574 /* Save the last block read, used to handle non-seekable files. */
3547 (*argc_out) = i; 575 sfile->source_position = pos + nread;
3548 while (i <= n) {
3549 (*argv_out)[i++] = NULL;
3550 }
3551}
3552 576
3553int 577 if (option_verbose > 3)
3554#if PYTHON_MODULE || SWIG_MODULE || NOT_MAIN
3555xd3_main_cmdline (int argc, char **argv)
3556#else
3557main (int argc, char **argv)
3558#endif
3559{
3560 static const char *flags =
3561 "0123456789cdefhnqvDFJNORTVs:m:B:C:E:I:L:O:M:P:W:A::S::";
3562 xd3_cmd cmd;
3563 main_file ifile;
3564 main_file ofile;
3565 main_file sfile;
3566 main_merge_list merge_order;
3567 main_merge *merge;
3568 int my_optind;
3569 const char *my_optarg;
3570 const char *my_optstr;
3571 const char *sfilename;
3572 int env_argc;
3573 char **env_argv;
3574 char **free_argv; /* malloc() in setup_environment() */
3575 char *free_value; /* malloc() in setup_environment() */
3576 int ret;
3577
3578#ifdef _WIN32
3579 GetStartupInfo(&winStartupInfo);
3580 setvbuf(stderr, NULL, _IONBF, 0); /* Do not buffer stderr */
3581#endif
3582
3583 main_file_init (& ifile);
3584 main_file_init (& ofile);
3585 main_file_init (& sfile);
3586 main_merge_list_init (& merge_order);
3587
3588 reset_defaults();
3589
3590 free_argv = NULL;
3591 free_value = NULL;
3592 setup_environment(argc, argv, &env_argc, &env_argv,
3593 &free_argv, &free_value);
3594 cmd = CMD_NONE;
3595 sfilename = NULL;
3596 my_optind = 1;
3597 argv = env_argv;
3598 argc = env_argc;
3599 program_name = env_argv[0];
3600
3601 takearg:
3602 my_optarg = NULL;
3603 my_optstr = argv[my_optind];
3604
3605 /* This doesn't use getopt() because it makes trouble for -P & python which
3606 * reenter main() and thus care about freeing all memory. I never had much
3607 * trust for getopt anyway, it's too opaque. This implements a fairly
3608 * standard non-long-option getopt with support for named operations (e.g.,
3609 * "xdelta3 [encode|decode|printhdr...] < in > out"). */
3610 if (my_optstr)
3611 { 578 {
3612 if (*my_optstr == '-') { my_optstr += 1; } 579 if (blru->blkno != (xoff_t)-1)
3613 else if (cmd == CMD_NONE) { goto nonflag; }
3614 else { my_optstr = NULL; }
3615 }
3616 while (my_optstr)
3617 {
3618 const char *s;
3619 my_optarg = NULL;
3620 if ((ret = *my_optstr++) == 0) { my_optind += 1; goto takearg; }
3621
3622 /* Option handling: first check for one ':' following the option in
3623 * flags, then check for two. The syntax allows:
3624 *
3625 * 1. -Afoo defines optarg="foo"
3626 * 2. -A foo defines optarg="foo"
3627 * 3. -A "" defines optarg="" (allows empty-string)
3628 * 4. -A [EOA or -moreargs] error (mandatory case)
3629 * 5. -A [EOA -moreargs] defines optarg=NULL (optional case)
3630 * 6. -A=foo defines optarg="foo"
3631 * 7. -A= defines optarg="" (mandatory case)
3632 * 8. -A= defines optarg=NULL (optional case)
3633 *
3634 * See tests in test_command_line_arguments().
3635 */
3636 s = strchr (flags, ret);
3637 if (s && s[1] && s[1] == ':')
3638 {
3639 int eqcase = 0;
3640 int option = s[2] && s[2] == ':';
3641
3642 /* Case 1, set optarg to the remaining characters. */
3643 my_optarg = my_optstr;
3644 my_optstr = "";
3645
3646 /* Case 2-5 */
3647 if (*my_optarg == 0)
3648 {
3649 /* Condition 4-5 */
3650 int have_arg = (my_optind < (argc - 1) &&
3651 *argv[my_optind+1] != '-');
3652
3653 if (! have_arg)
3654 {
3655 if (! option)
3656 {
3657 /* Case 4 */
3658 XPR(NT "-%c: requires an argument\n", ret);
3659 ret = EXIT_FAILURE;
3660 goto cleanup;
3661 }
3662 /* Case 5. */
3663 my_optarg = NULL;
3664 }
3665 else
3666 {
3667 /* Case 2-3. */
3668 my_optarg = argv[++my_optind];
3669 }
3670 }
3671 /* Case 6-8. */
3672 else if (*my_optarg == '=')
3673 {
3674 /* Remove the = in all cases. */
3675 my_optarg += 1;
3676 eqcase = 1;
3677
3678 if (option && *my_optarg == 0)
3679 {
3680 /* Case 8. */
3681 my_optarg = NULL;
3682 }
3683 }
3684 }
3685
3686 switch (ret)
3687 { 580 {
3688 /* case: if no '-' was found, maybe check for a command name. */ 581 if (blru->blkno != blkno)
3689 nonflag:
3690 if (strcmp (my_optstr, "decode") == 0) { cmd = CMD_DECODE; }
3691 else if (strcmp (my_optstr, "encode") == 0)
3692 {
3693#if XD3_ENCODER
3694 cmd = CMD_ENCODE;
3695#else
3696 XPR(NT "encoder support not compiled\n");
3697 return EXIT_FAILURE;
3698#endif
3699 }
3700 else if (strcmp (my_optstr, "config") == 0) { cmd = CMD_CONFIG; }
3701#if REGRESSION_TEST
3702 else if (strcmp (my_optstr, "test") == 0) { cmd = CMD_TEST; }
3703#endif
3704#if VCDIFF_TOOLS
3705 else if (strcmp (my_optstr, "printhdr") == 0) { cmd = CMD_PRINTHDR; }
3706 else if (strcmp (my_optstr, "printhdrs") == 0)
3707 { cmd = CMD_PRINTHDRS; }
3708 else if (strcmp (my_optstr, "printdelta") == 0)
3709 { cmd = CMD_PRINTDELTA; }
3710 else if (strcmp (my_optstr, "recode") == 0) { cmd = CMD_RECODE; }
3711 else if (strcmp (my_optstr, "merge") == 0) { cmd = CMD_MERGE; }
3712#endif
3713
3714 /* If no option was found and still no command, let the default
3715 * command be encode. The remaining args are treated as
3716 * filenames. */
3717 if (cmd == CMD_NONE)
3718 {
3719 cmd = CMD_DEFAULT;
3720 my_optstr = NULL;
3721 break;
3722 }
3723 else
3724 { 582 {
3725 /* But if we find a command name, continue the getopt loop. */ 583 XPR(NT "source block %"Q"u ejects %"Q"u (lru_hits=%u, "
3726 my_optind += 1; 584 "lru_misses=%u, lru_filled=%u)\n",
3727 goto takearg; 585 blkno, blru->blkno, lru_hits, lru_misses, lru_filled);
3728 }
3729
3730 /* gzip-like options */
3731 case '0': case '1': case '2': case '3': case '4':
3732 case '5': case '6': case '7': case '8': case '9':
3733 option_level = ret - '0';
3734 break;
3735 case 'f': option_force = 1; break;
3736 case 'F':
3737#if EXTERNAL_COMPRESSION
3738 option_force2 = 1;
3739#else
3740 XPR(NT "warning: -F option ignored, "
3741 "external compression support was not compiled\n");
3742 break;
3743#endif
3744 case 'v': option_verbose += 1; option_quiet = 0; break;
3745 case 'q': option_quiet = 1; option_verbose = 0; break;
3746 case 'c': option_stdout = 1; break;
3747 case 'd':
3748 if (cmd == CMD_NONE) { cmd = CMD_DECODE; }
3749 else { ret = main_help (); goto exit; }
3750 break;
3751 case 'e':
3752#if XD3_ENCODER
3753 if (cmd == CMD_NONE) { cmd = CMD_ENCODE; }
3754 else { ret = main_help (); goto exit; }
3755 break;
3756#else
3757 XPR(NT "encoder support not compiled\n");
3758 return EXIT_FAILURE;
3759#endif
3760
3761 case 'n': option_use_checksum = 0; break;
3762 case 'N': option_no_compress = 1; break;
3763 case 'T': option_use_altcodetable = 1; break;
3764 case 'C': option_smatch_config = my_optarg; break;
3765 case 'J': option_no_output = 1; break;
3766 case 'S': if (my_optarg == NULL)
3767 {
3768 option_use_secondary = 1;
3769 option_secondary = "none";
3770 } 586 }
3771 else 587 else
3772 { 588 {
3773 option_use_secondary = 1; 589 XPR(NT "source block %"Q"u read (lru_hits=%u, "
3774 option_secondary = my_optarg; 590 "lru_misses=%u, lru_filled=%u)\n",
3775 } 591 blkno, lru_hits, lru_misses, lru_filled);
3776 break;
3777 case 'A': if (my_optarg == NULL) { option_use_appheader = 0; }
3778 else { option_appheader = (uint8_t*) my_optarg; } break;
3779 case 'B':
3780 if ((ret = main_atou (my_optarg, & option_srcwinsz, XD3_MINSRCWINSZ,
3781 0, 'B')))
3782 {
3783 goto exit;
3784 }
3785 break;
3786 case 'I':
3787 if ((ret = main_atou (my_optarg, & option_iopt_size, 0,
3788 0, 'I')))
3789 {
3790 goto exit;
3791 }
3792 break;
3793 case 'P':
3794 if ((ret = main_atou (my_optarg, & option_sprevsz, 0,
3795 0, 'P')))
3796 {
3797 goto exit;
3798 }
3799 break;
3800 case 'W':
3801 if ((ret = main_atou (my_optarg, & option_winsize, XD3_ALLOCSIZE,
3802 XD3_HARDMAXWINSIZE, 'W')))
3803 {
3804 goto exit;
3805 }
3806 break;
3807 case 'D':
3808#if EXTERNAL_COMPRESSION == 0
3809 if (option_verbose > 0)
3810 {
3811 XPR(NT "warning: -D option ignored, "
3812 "external compression support was not compiled\n");
3813 }
3814#else
3815 option_decompress_inputs = 0;
3816#endif
3817 break;
3818 case 'R':
3819#if EXTERNAL_COMPRESSION == 0
3820 if (option_verbose > 0)
3821 {
3822 XPR(NT "warning: -R option ignored, "
3823 "external compression support was not compiled\n");
3824 }
3825#else
3826 option_recompress_outputs = 0;
3827#endif
3828 break;
3829 case 's':
3830 if (sfilename != NULL)
3831 {
3832 XPR(NT "specify only one source file\n");
3833 goto cleanup;
3834 }
3835
3836 sfilename = my_optarg;
3837 break;
3838 case 'm':
3839 if ((merge = (main_merge*)
3840 main_malloc (sizeof (main_merge))) == NULL)
3841 {
3842 goto cleanup;
3843 } 592 }
3844 main_merge_list_push_back (& merge_order, merge);
3845 merge->filename = my_optarg;
3846 break;
3847 case 'V':
3848 ret = main_version (); goto exit;
3849 default:
3850 ret = main_help (); goto exit;
3851 }
3852 }
3853
3854 option_source_filename = sfilename;
3855
3856 /* In case there were no arguments, set the default command. */
3857 if (cmd == CMD_NONE) { cmd = CMD_DEFAULT; }
3858
3859 argc -= my_optind;
3860 argv += my_optind;
3861
3862 /* There may be up to two more arguments. */
3863 if (argc > 2)
3864 {
3865 XPR(NT "too many filenames: %s ...\n", argv[2]);
3866 goto cleanup;
3867 }
3868
3869 ifile.flags = RD_FIRST | RD_MAININPUT;
3870 sfile.flags = RD_FIRST;
3871 sfile.filename = option_source_filename;
3872
3873 /* The infile takes the next argument, if there is one. But if not, infile
3874 * is set to stdin. */
3875 if (argc > 0)
3876 {
3877 ifile.filename = argv[0];
3878
3879 if ((ret = main_file_open (& ifile, ifile.filename, XO_READ)))
3880 {
3881 goto cleanup;
3882 } 593 }
3883 } 594 else
3884 else
3885 {
3886 XSTDIN_XF (& ifile);
3887 }
3888
3889 /* The ofile takes the following argument, if there is one. But if not, it
3890 * is left NULL until the application header is processed. It will be set
3891 * in main_open_output. */
3892 if (argc > 1)
3893 {
3894 /* Check for conflicting arguments. */
3895 if (option_stdout && ! option_quiet)
3896 { 595 {
3897 XPR(NT "warning: -c option overrides output filename: %s\n", 596 XPR(NT "source block %"Q"u read (lru_hits=%u, lru_misses=%u, "
3898 argv[1]); 597 "lru_filled=%u)\n", blkno, lru_hits, lru_misses, lru_filled);
3899 } 598 }
3900
3901 if (! option_stdout) { ofile.filename = argv[1]; }
3902 }
3903
3904#if VCDIFF_TOOLS
3905 if (cmd == CMD_MERGE &&
3906 (ret = main_merge_arguments (&merge_order)))
3907 {
3908 goto cleanup;
3909 }
3910#endif /* VCDIFF_TOOLS */
3911
3912 switch (cmd)
3913 {
3914 case CMD_PRINTHDR:
3915 case CMD_PRINTHDRS:
3916 case CMD_PRINTDELTA:
3917#if XD3_ENCODER
3918 case CMD_ENCODE:
3919 case CMD_RECODE:
3920 case CMD_MERGE:
3921#endif
3922 case CMD_DECODE:
3923 ret = main_input (cmd, & ifile, & ofile, & sfile);
3924 break;
3925
3926#if REGRESSION_TEST
3927 case CMD_TEST:
3928 main_config ();
3929 ret = xd3_selftest ();
3930 break;
3931#endif
3932
3933 case CMD_CONFIG:
3934 ret = main_config ();
3935 break;
3936
3937 default:
3938 ret = main_help ();
3939 break;
3940 }
3941
3942 if (0)
3943 {
3944 cleanup:
3945 ret = EXIT_FAILURE;
3946 exit:
3947 (void)0;
3948 }
3949
3950#if EXTERNAL_COMPRESSION
3951 if (ext_tmpfile != NULL)
3952 {
3953 unlink (ext_tmpfile);
3954 }
3955#endif
3956
3957 main_file_cleanup (& ifile);
3958 main_file_cleanup (& ofile);
3959 main_file_cleanup (& sfile);
3960
3961 while (! main_merge_list_empty (& merge_order))
3962 {
3963 merge = main_merge_list_pop_front (& merge_order);
3964 main_free (merge);
3965 } 599 }
3966 600
3967 main_free (free_argv); 601 source->curblk = blru->blk;
3968 main_free (free_value); 602 source->curblkno = blkno;
3969 603 source->onblk = nread;
3970 main_cleanup (); 604 blru->size = nread;
3971 605
3972 fflush (stdout); 606 IF_DEBUG1 (DP(RINT "[main_getblk] blkno %"Q"u onblk %u pos %"Q"u "
3973 fflush (stderr); 607 "srcpos %"Q"u\n",
3974 return ret; 608 blkno, nread, pos, sfile->source_position));
3975}
3976 609
3977static int 610 return 0;
3978main_help (void)
3979{
3980 main_version();
3981
3982 /* Note: update wiki when command-line features change */
3983 DP(RINT "usage: xdelta3 [command/options] [input [output]]\n");
3984 DP(RINT "make patch:\n");
3985 DP(RINT "\n");
3986 DP(RINT " xdelta3.exe -e -s old_file new_file delta_file\n");
3987 DP(RINT "\n");
3988 DP(RINT "apply patch:\n");
3989 DP(RINT "\n");
3990 DP(RINT " xdelta3.exe -d -s old_file delta_file decoded_new_file\n");
3991 DP(RINT "\n");
3992 DP(RINT "special command names:\n");
3993 DP(RINT " config prints xdelta3 configuration\n");
3994 DP(RINT " decode decompress the input\n");
3995 DP(RINT " encode compress the input%s\n",
3996 XD3_ENCODER ? "" : " [Not compiled]");
3997#if REGRESSION_TEST
3998 DP(RINT " test run the builtin tests\n");
3999#endif
4000#if VCDIFF_TOOLS
4001 DP(RINT "special commands for VCDIFF inputs:\n");
4002 DP(RINT " printdelta print information about the entire delta\n");
4003 DP(RINT " printhdr print information about the first window\n");
4004 DP(RINT " printhdrs print information about all windows\n");
4005 DP(RINT " recode encode with new application/secondary settings\n");
4006 DP(RINT " merge merge VCDIFF inputs (see below)\n");
4007#endif
4008 DP(RINT "merge patches:\n");
4009 DP(RINT "\n");
4010 DP(RINT " xdelta3 merge -m 1.vcdiff -m 2.vcdiff 3.vcdiff merged.vcdiff\n");
4011 DP(RINT "\n");
4012 DP(RINT "standard options:\n");
4013 DP(RINT " -0 .. -9 compression level\n");
4014 DP(RINT " -c use stdout\n");
4015 DP(RINT " -d decompress\n");
4016 DP(RINT " -e compress%s\n",
4017 XD3_ENCODER ? "" : " [Not compiled]");
4018 DP(RINT " -f force (overwrite, ignore trailing garbage)\n");
4019#if EXTERNAL_COMPRESSION
4020 DP(RINT " -F force the external-compression subprocess\n");
4021#endif
4022 DP(RINT " -h show help\n");
4023 DP(RINT " -q be quiet\n");
4024 DP(RINT " -v be verbose (max 2)\n");
4025 DP(RINT " -V show version\n");
4026
4027 DP(RINT "memory options:\n");
4028 DP(RINT " -B bytes source window size\n");
4029 DP(RINT " -W bytes input window size\n");
4030 DP(RINT " -P size compression duplicates window\n");
4031 DP(RINT " -I size instruction buffer size (0 = unlimited)\n");
4032
4033 DP(RINT "compression options:\n");
4034 DP(RINT " -s source source file to copy from (if any)\n");
4035 DP(RINT " -S [djw|fgk] enable/disable secondary compression\n");
4036 DP(RINT " -N disable small string-matching compression\n");
4037 DP(RINT " -D disable external decompression (encode/decode)\n");
4038 DP(RINT " -R disable external recompression (decode)\n");
4039 DP(RINT " -n disable checksum (encode/decode)\n");
4040 DP(RINT " -C soft config (encode, undocumented)\n");
4041 DP(RINT " -A [apphead] disable/provide application header (encode)\n");
4042 DP(RINT " -J disable output (check/compute only)\n");
4043 DP(RINT " -T use alternate code table (test)\n");
4044 DP(RINT " -m arguments for \"merge\"\n");
4045
4046 DP(RINT "the XDELTA environment variable may contain extra args:\n");
4047 DP(RINT " XDELTA=\"-s source-x.y.tar.gz\" \\\n");
4048 DP(RINT " tar --use-compress-program=xdelta3 \\\n");
4049 DP(RINT " -cf target-x.z.tar.gz.vcdiff target-x.y\n");
4050 return EXIT_FAILURE;
4051} 611}
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h
index ff6ade6..e47d708 100644
--- a/xdelta3/xdelta3-main.h
+++ b/xdelta3/xdelta3-main.h
@@ -192,8 +192,6 @@ typedef enum
192 192
193typedef struct _main_file main_file; 193typedef struct _main_file main_file;
194typedef struct _main_extcomp main_extcomp; 194typedef struct _main_extcomp main_extcomp;
195typedef struct _main_blklru main_blklru;
196typedef struct _main_blklru_list main_blklru_list;
197typedef struct _main_merge main_merge; 195typedef struct _main_merge main_merge;
198typedef struct _main_merge_list main_merge_list; 196typedef struct _main_merge_list main_merge_list;
199 197
@@ -246,29 +244,6 @@ struct _main_extcomp
246 int flags; 244 int flags;
247}; 245};
248 246
249/* This file implements a small LRU of source blocks. For encoding purposes,
250 * we prevent paging in blocks we've already scanned in the source (return
251 * XD3_NOTAVAIL). */
252struct _main_blklru_list
253{
254 main_blklru_list *next;
255 main_blklru_list *prev;
256};
257
258struct _main_blklru
259{
260 uint8_t *blk;
261 xoff_t blkno;
262 usize_t size;
263 main_blklru_list link;
264};
265
266#define MAX_LRU_SIZE 32U
267#define XD3_MINSRCWINSZ XD3_ALLOCSIZE
268
269/* ... represented as a list (no cache index). */
270XD3_MAKELIST(main_blklru_list,main_blklru,link);
271
272/* Merge state: */ 247/* Merge state: */
273 248
274struct _main_merge_list 249struct _main_merge_list
@@ -336,17 +311,6 @@ static uint8_t* appheader_used = NULL;
336static uint8_t* main_bdata = NULL; 311static uint8_t* main_bdata = NULL;
337static usize_t main_bsize = 0; 312static usize_t main_bsize = 0;
338 313
339/* The LRU: obviously this is shared by all callers. */
340static usize_t lru_size = 0;
341static main_blklru *lru = NULL; /* array of lru_size elts */
342static main_blklru_list lru_list;
343static main_blklru_list lru_free;
344static int do_src_fifo = 0; /* set to avoid lru */
345
346static int lru_hits = 0;
347static int lru_misses = 0;
348static int lru_filled = 0;
349
350/* Hacks for VCDIFF tools */ 314/* Hacks for VCDIFF tools */
351static int allow_fake_source = 0; 315static int allow_fake_source = 0;
352 316
@@ -379,8 +343,24 @@ static void main_get_appheader (xd3_stream *stream, main_file *ifile,
379static int main_getblk_func (xd3_stream *stream, 343static int main_getblk_func (xd3_stream *stream,
380 xd3_source *source, 344 xd3_source *source,
381 xoff_t blkno); 345 xoff_t blkno);
346static void main_free (void *ptr);
347static void* main_malloc (usize_t size);
348
349static int main_file_open (main_file *xfile, const char* name, int mode);
350static int main_file_stat (main_file *xfile, xoff_t *size);
351static int main_file_seek (main_file *xfile, xoff_t pos);
352static int main_read_primary_input (main_file *file,
353 uint8_t *buf,
354 usize_t size,
355 usize_t *nread);
356
357static const char* main_format_bcnt (xoff_t r, char *buf);
382static int main_help (void); 358static int main_help (void);
383 359
360/* The code in xdelta3-blk.h is essentially part of this unit, see
361 * comments there. */
362#include "xdelta3-blkcache.h"
363
384static int 364static int
385main_version (void) 365main_version (void)
386{ 366{
@@ -418,12 +398,12 @@ main_config (void)
418 DP(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ); 398 DP(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ);
419 DP(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE); 399 DP(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE);
420 DP(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE); 400 DP(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE);
421 DP(RINT "sizeof(void*)=%d\n", sizeof(void*)); 401 DP(RINT "sizeof(void*)=%d\n", (int)sizeof(void*));
422 DP(RINT "sizeof(int)=%d\n", sizeof(int)); 402 DP(RINT "sizeof(int)=%d\n", (int)sizeof(int));
423 DP(RINT "sizeof(uint32_t)=%d\n", sizeof(uint32_t)); 403 DP(RINT "sizeof(uint32_t)=%d\n", (int)sizeof(uint32_t));
424 DP(RINT "sizeof(uint64_t)=%d\n", sizeof(uint64_t)); 404 DP(RINT "sizeof(uint64_t)=%d\n", (int)sizeof(uint64_t));
425 DP(RINT "sizeof(usize_t)=%d\n", sizeof(usize_t)); 405 DP(RINT "sizeof(usize_t)=%d\n", (int)sizeof(usize_t));
426 DP(RINT "sizeof(xoff_t)=%d\n", sizeof(xoff_t)); 406 DP(RINT "sizeof(xoff_t)=%d\n", (int)sizeof(xoff_t));
427 407
428 return EXIT_SUCCESS; 408 return EXIT_SUCCESS;
429} 409}
@@ -447,15 +427,11 @@ reset_defaults(void)
447 appheader_used = NULL; 427 appheader_used = NULL;
448 main_bdata = NULL; 428 main_bdata = NULL;
449 main_bsize = 0; 429 main_bsize = 0;
450 lru_size = 0;
451 lru = NULL;
452 do_src_fifo = 0;
453 lru_hits = 0;
454 lru_misses = 0;
455 lru_filled = 0;
456 allow_fake_source = 0; 430 allow_fake_source = 0;
457 option_smatch_config = NULL; 431 option_smatch_config = NULL;
458 432
433 main_lru_reset();
434
459 option_use_appheader = 1; 435 option_use_appheader = 1;
460 option_use_checksum = 1; 436 option_use_checksum = 1;
461#if EXTERNAL_COMPRESSION 437#if EXTERNAL_COMPRESSION
@@ -531,7 +507,7 @@ get_errno (void)
531 return errno; 507 return errno;
532#else 508#else
533 DWORD err_num = GetLastError(); 509 DWORD err_num = GetLastError();
534 if (err_num == NO_ERROR) 510 if (err_num == NO_ERROR)
535 { 511 {
536 err_num = XD3_INTERNAL; 512 err_num = XD3_INTERNAL;
537 } 513 }
@@ -543,7 +519,7 @@ const char*
543xd3_mainerror(int err_num) { 519xd3_mainerror(int err_num) {
544#ifndef _WIN32 520#ifndef _WIN32
545 const char* x = xd3_strerror (err_num); 521 const char* x = xd3_strerror (err_num);
546 if (x != NULL) 522 if (x != NULL)
547 { 523 {
548 return x; 524 return x;
549 } 525 }
@@ -894,12 +870,12 @@ main_file_open (main_file *xfile, const char* name, int mode)
894 (mode == XO_READ) ? GENERIC_READ : GENERIC_WRITE, 870 (mode == XO_READ) ? GENERIC_READ : GENERIC_WRITE,
895 FILE_SHARE_READ, 871 FILE_SHARE_READ,
896 NULL, 872 NULL,
897 (mode == XO_READ) ? 873 (mode == XO_READ) ?
898 OPEN_EXISTING : 874 OPEN_EXISTING :
899 (option_force ? CREATE_ALWAYS : CREATE_NEW), 875 (option_force ? CREATE_ALWAYS : CREATE_NEW),
900 FILE_ATTRIBUTE_NORMAL, 876 FILE_ATTRIBUTE_NORMAL,
901 NULL); 877 NULL);
902 if (xfile->file == INVALID_HANDLE_VALUE) 878 if (xfile->file == INVALID_HANDLE_VALUE)
903 { 879 {
904 ret = get_errno (); 880 ret = get_errno ();
905 } 881 }
@@ -1126,7 +1102,7 @@ main_file_seek (main_file *xfile, xoff_t pos)
1126# if (_WIN32_WINNT >= 0x0500) 1102# if (_WIN32_WINNT >= 0x0500)
1127 LARGE_INTEGER move, out; 1103 LARGE_INTEGER move, out;
1128 move.QuadPart = pos; 1104 move.QuadPart = pos;
1129 if (SetFilePointerEx(xfile->file, move, &out, FILE_BEGIN) == 0) 1105 if (SetFilePointerEx(xfile->file, move, &out, FILE_BEGIN) == 0)
1130 { 1106 {
1131 ret = get_errno (); 1107 ret = get_errno ();
1132 } 1108 }
@@ -2500,17 +2476,17 @@ main_secondary_decompress_check (main_file *file,
2500 { 2476 {
2501 XPR(NT 2477 XPR(NT
2502 "WARNING: the encoder is automatically decompressing the input file;\n"); 2478 "WARNING: the encoder is automatically decompressing the input file;\n");
2503 XPR(NT 2479 XPR(NT
2504 "WARNING: the decoder will automatically recompress the output file;\n"); 2480 "WARNING: the decoder will automatically recompress the output file;\n");
2505 XPR(NT 2481 XPR(NT
2506 "WARNING: this may result in different compressed data and checksums\n"); 2482 "WARNING: this may result in different compressed data and checksums\n");
2507 XPR(NT 2483 XPR(NT
2508 "WARNING: despite being identical data; if this is an issue, use -D\n"); 2484 "WARNING: despite being identical data; if this is an issue, use -D\n");
2509 XPR(NT 2485 XPR(NT
2510 "WARNING: to avoid decompression and/or use -R to avoid recompression\n"); 2486 "WARNING: to avoid decompression and/or use -R to avoid recompression\n");
2511 XPR(NT 2487 XPR(NT
2512 "WARNING: and/or manually decompress the input file; if you know the\n"); 2488 "WARNING: and/or manually decompress the input file; if you know the\n");
2513 XPR(NT 2489 XPR(NT
2514 "WARNING: compression settings that will produce identical output\n"); 2490 "WARNING: compression settings that will produce identical output\n");
2515 XPR(NT 2491 XPR(NT
2516 "WARNING: you may set those flags using the environment (e.g., GZIP=-9)\n"); 2492 "WARNING: you may set those flags using the environment (e.g., GZIP=-9)\n");
@@ -2755,7 +2731,7 @@ main_set_appheader (xd3_stream *stream, main_file *input, main_file *sfile)
2755 } 2731 }
2756 } 2732 }
2757 2733
2758 xd3_set_appheader (stream, appheader_used, 2734 xd3_set_appheader (stream, appheader_used,
2759 (usize_t) strlen ((char*)appheader_used)); 2735 (usize_t) strlen ((char*)appheader_used));
2760 2736
2761 return 0; 2737 return 0;
@@ -2957,234 +2933,6 @@ main_open_output (xd3_stream *stream, main_file *ofile)
2957 return 0; 2933 return 0;
2958} 2934}
2959 2935
2960/* This is called at different times for encoding and decoding. The
2961 * encoder calls it immediately, the decoder delays until the
2962 * application header is received. */
2963static int
2964main_set_source (xd3_stream *stream, xd3_cmd cmd,
2965 main_file *sfile, xd3_source *source)
2966{
2967 int ret = 0;
2968 usize_t i;
2969 usize_t blksize;
2970 xoff_t source_size = 0;
2971 main_blklru block0;
2972
2973 XD3_ASSERT (lru == NULL);
2974 XD3_ASSERT (stream->src == NULL);
2975 XD3_ASSERT (option_srcwinsz >= XD3_MINSRCWINSZ);
2976
2977 main_blklru_list_init (& lru_list);
2978 main_blklru_list_init (& lru_free);
2979
2980 if (allow_fake_source)
2981 {
2982 sfile->mode = XO_READ;
2983 sfile->realname = sfile->filename;
2984 sfile->nread = 0;
2985 }
2986 else
2987 {
2988 if ((ret = main_file_open (sfile, sfile->filename, XO_READ)))
2989 {
2990 return ret;
2991 }
2992
2993 /* Allow non-seekable sources from the start. If the file
2994 * turns out to be externally compressed, size_known may change. */
2995 sfile->size_known = (main_file_stat (sfile, &source_size) == 0);
2996 }
2997
2998 /* The API requires power-of-two blocksize, */
2999 blksize = xd3_pow2_roundup (max (option_srcwinsz / MAX_LRU_SIZE,
3000 XD3_MINSRCWINSZ));
3001
3002 /* TODO(jmacd): The organization of this code and the implementation
3003 * of the LRU cache could be improved. This is too convoluted, the
3004 * code uses main_getblk_func() to read the first block, which may
3005 * trigger external-decompression, in large part so that the
3006 * verbose-printing and counters maintained by getblk_func are
3007 * consistent. There used to be additional optimizations going on
3008 * here: (1) if the source size is known we would like to lower
3009 * option_srcwinsz, if possible, (2) avoid allocating more memory
3010 * than needed, and (3) also want to use a single block when
3011 * source_size < option_srcwinsz, this because compression is better
3012 * for larger blocks (especially due to the blockwise-reverse
3013 * insertion of checksums).
3014 *
3015 * (3) is no longer implemented. (2) is only implemented for files
3016 * that are shorter than the default blocksize, and (1) is not
3017 * implemented. These optimizations are not taken because the code
3018 * is already too complicated.
3019 *
3020 * The ideal solution may be to allocate a block of memory equal to
3021 * half of the option_srcwinsz. Read as much data as possible into
3022 * that block. If the entire file fits, pass one block to the
3023 * library for best compression. If not, copy the 50% block into
3024 * smaller (option_srcwinsz / MAX_LRU_SIZE) blocks and proceed. Too
3025 * much effort for too little payback. */
3026
3027 memset (&block0, 0, sizeof (block0));
3028 block0.blkno = (xoff_t) -1;
3029
3030 /* Allocate the first block. Even if the size is known at this
3031 * point, we do not lower it because decompression may happen. */
3032 if ((block0.blk = (uint8_t*) main_malloc (blksize)) == NULL)
3033 {
3034 ret = ENOMEM;
3035 return ret;
3036 }
3037
3038 source->blksize = blksize;
3039 source->name = sfile->filename;
3040 source->ioh = sfile;
3041 source->curblkno = (xoff_t) -1;
3042 source->curblk = NULL;
3043
3044 /* We have to read the first block into the cache now, because
3045 * size_known can still change (due to secondary decompression).
3046 * Calls main_secondary_decompress_check() via
3047 * main_read_primary_input(). */
3048 lru_size = 1;
3049 lru = &block0;
3050 XD3_ASSERT (main_blklru_list_empty (& lru_free));
3051 XD3_ASSERT (main_blklru_list_empty (& lru_list));
3052 main_blklru_list_push_back (& lru_free, & lru[0]);
3053 /* This call is partly so that the diagnostics printed by
3054 * this function appear to happen normally for the first read,
3055 * which is special. */
3056 ret = main_getblk_func (stream, source, 0);
3057 main_blklru_list_remove (& lru[0]);
3058 XD3_ASSERT (main_blklru_list_empty (& lru_free));
3059 XD3_ASSERT (main_blklru_list_empty (& lru_list));
3060 lru = NULL;
3061
3062 if (ret != 0)
3063 {
3064 main_free (block0.blk);
3065
3066 XPR(NT "error reading source: %s: %s\n",
3067 sfile->filename,
3068 xd3_mainerror (ret));
3069
3070 return ret;
3071 }
3072
3073 source->onblk = block0.size;
3074
3075 /* If the file is smaller than a block, size is known. */
3076 if (!sfile->size_known && source->onblk < blksize)
3077 {
3078 source_size = source->onblk;
3079 sfile->size_known = 1;
3080 }
3081
3082 /* We update lru_size accordingly, and modify option_srcwinsz, which
3083 * will be passed via xd3_config. */
3084 if (sfile->size_known && source_size <= option_srcwinsz)
3085 {
3086 lru_size = (usize_t) ((source_size + blksize - 1) / blksize);
3087 }
3088 else
3089 {
3090 lru_size = (option_srcwinsz + blksize - 1) / blksize;
3091 }
3092
3093 XD3_ASSERT (lru_size >= 1);
3094 option_srcwinsz = lru_size * blksize;
3095
3096 if ((lru = (main_blklru*) main_malloc (lru_size * sizeof (main_blklru)))
3097 == NULL)
3098 {
3099 ret = ENOMEM;
3100 return ret;
3101 }
3102
3103 lru[0].blk = block0.blk;
3104 lru[0].blkno = 0;
3105 lru[0].size = source->onblk;
3106
3107 if (! sfile->size_known)
3108 {
3109 do_src_fifo = 1;
3110 }
3111 else if (! do_src_fifo)
3112 {
3113 main_blklru_list_push_back (& lru_list, & lru[0]);
3114 }
3115
3116 for (i = 1; i < lru_size; i += 1)
3117 {
3118 lru[i].blkno = (xoff_t) -1;
3119
3120 if ((lru[i].blk = (uint8_t*) main_malloc (source->blksize)) == NULL)
3121 {
3122 ret = ENOMEM;
3123 return ret;
3124 }
3125
3126 if (! do_src_fifo)
3127 {
3128 main_blklru_list_push_back (& lru_free, & lru[i]);
3129 }
3130 }
3131
3132 if (sfile->size_known)
3133 {
3134 ret = xd3_set_source_and_size (stream, source, source_size);
3135 }
3136 else
3137 {
3138 ret = xd3_set_source (stream, source);
3139 }
3140
3141 if (ret)
3142 {
3143 XPR(NT XD3_LIB_ERRMSG (stream, ret));
3144 return ret;
3145 }
3146
3147 XD3_ASSERT (stream->src == source);
3148 XD3_ASSERT (source->blksize == blksize);
3149
3150 if (option_verbose)
3151 {
3152 static char srcszbuf[32];
3153 static char srccntbuf[32];
3154 static char winszbuf[32];
3155 static char blkszbuf[32];
3156 static char nbufs[32];
3157
3158 if (sfile->size_known)
3159 {
3160 sprintf (srcszbuf, "source size %s [%"Q"u]",
3161 main_format_bcnt (source_size, srccntbuf),
3162 source_size);
3163 }
3164 else
3165 {
3166 strcpy(srcszbuf, "source size unknown");
3167 }
3168
3169 nbufs[0] = 0;
3170
3171 if (option_verbose > 1)
3172 {
3173 sprintf(nbufs, " #bufs %u", lru_size);
3174 }
3175
3176 XPR(NT "source %s %s blksize %s window %s%s%s\n",
3177 sfile->filename,
3178 srcszbuf,
3179 main_format_bcnt (blksize, blkszbuf),
3180 main_format_bcnt (option_srcwinsz, winszbuf),
3181 nbufs,
3182 do_src_fifo ? " (FIFO)" : "");
3183 }
3184
3185 return 0;
3186}
3187
3188static usize_t 2936static usize_t
3189main_get_winsize (main_file *ifile) { 2937main_get_winsize (main_file *ifile) {
3190 xoff_t file_size = 0; 2938 xoff_t file_size = 0;
@@ -3208,310 +2956,6 @@ main_get_winsize (main_file *ifile) {
3208 return size; 2956 return size;
3209} 2957}
3210 2958
3211/*******************************************************************
3212 Source routines
3213 *******************************************************************/
3214
3215static int
3216main_getblk_lru (xd3_source *source, xoff_t blkno,
3217 main_blklru** blrup, int *is_new)
3218{
3219 main_blklru *blru = NULL;
3220 usize_t i;
3221
3222 (*is_new) = 0;
3223
3224 if (do_src_fifo)
3225 {
3226 /* Direct lookup assumes sequential scan w/o skipping blocks. */
3227 int idx = blkno % lru_size;
3228 blru = & lru[idx];
3229 if (blru->blkno == blkno)
3230 {
3231 (*blrup) = blru;
3232 return 0;
3233 }
3234
3235 if (blru->blkno != (xoff_t)-1 &&
3236 blru->blkno != (xoff_t)(blkno - lru_size))
3237 {
3238 return XD3_TOOFARBACK;
3239 }
3240 }
3241 else
3242 {
3243 /* Sequential search through LRU. */
3244 for (i = 0; i < lru_size; i += 1)
3245 {
3246 blru = & lru[i];
3247 if (blru->blkno == blkno)
3248 {
3249 main_blklru_list_remove (blru);
3250 main_blklru_list_push_back (& lru_list, blru);
3251 (*blrup) = blru;
3252 return 0;
3253 }
3254 }
3255 }
3256
3257 if (do_src_fifo)
3258 {
3259 int idx = blkno % lru_size;
3260 blru = & lru[idx];
3261 }
3262 else
3263 {
3264 if (! main_blklru_list_empty (& lru_free))
3265 {
3266 blru = main_blklru_list_pop_front (& lru_free);
3267 main_blklru_list_push_back (& lru_list, blru);
3268 }
3269 else
3270 {
3271 XD3_ASSERT (! main_blklru_list_empty (& lru_list));
3272 blru = main_blklru_list_pop_front (& lru_list);
3273 main_blklru_list_push_back (& lru_list, blru);
3274 }
3275 }
3276
3277 lru_filled += 1;
3278 (*is_new) = 1;
3279 (*blrup) = blru;
3280 blru->blkno = blkno;
3281 return 0;
3282}
3283
3284static int
3285main_read_seek_source (xd3_stream *stream,
3286 xd3_source *source,
3287 xoff_t blkno) {
3288 xoff_t pos = blkno * source->blksize;
3289 main_file *sfile = (main_file*) source->ioh;
3290 main_blklru *blru;
3291 int is_new;
3292 usize_t nread = 0;
3293 int ret = 0;
3294
3295 if (!sfile->seek_failed)
3296 {
3297 ret = main_file_seek (sfile, pos);
3298
3299 if (ret == 0)
3300 {
3301 sfile->source_position = pos;
3302 }
3303 }
3304
3305 if (sfile->seek_failed || ret != 0)
3306 {
3307 /* For an unseekable file (or other seek error, does it
3308 * matter?) */
3309 if (sfile->source_position > pos)
3310 {
3311 /* Could assert !IS_ENCODE(), this shouldn't happen
3312 * because of do_src_fifo during encode. */
3313 if (!option_quiet)
3314 {
3315 XPR(NT "source can't seek backwards; requested block offset "
3316 "%"Q"u source position is %"Q"u\n",
3317 pos, sfile->source_position);
3318 }
3319
3320 sfile->seek_failed = 1;
3321 stream->msg = "non-seekable source: "
3322 "copy is too far back (try raising -B)";
3323 return XD3_TOOFARBACK;
3324 }
3325
3326 /* There's a chance here, that an genuine lseek error will cause
3327 * xdelta3 to shift into non-seekable mode, entering a degraded
3328 * condition. */
3329 if (!sfile->seek_failed && option_verbose)
3330 {
3331 XPR(NT "source can't seek, will use FIFO for %s\n",
3332 sfile->filename);
3333
3334 if (option_verbose > 1)
3335 {
3336 XPR(NT "seek error at offset %"Q"u: %s\n",
3337 pos, xd3_mainerror (ret));
3338 }
3339 }
3340
3341 sfile->seek_failed = 1;
3342
3343 while (sfile->source_position < pos)
3344 {
3345 xoff_t skip_blkno;
3346 usize_t skip_offset;
3347
3348 xd3_blksize_div (sfile->source_position, source,
3349 &skip_blkno, &skip_offset);
3350
3351 /* Read past unused data */
3352 XD3_ASSERT (pos - sfile->source_position >= source->blksize);
3353 XD3_ASSERT (skip_offset == 0);
3354
3355 if ((ret = main_getblk_lru (source, skip_blkno,
3356 & blru, & is_new)))
3357 {
3358 return ret;
3359 }
3360
3361 XD3_ASSERT (is_new);
3362
3363 if (option_verbose > 1)
3364 {
3365 XPR(NT "non-seekable source skipping %"Q"u bytes @ %"Q"u\n",
3366 pos - sfile->source_position,
3367 sfile->source_position);
3368 }
3369
3370 if ((ret = main_read_primary_input (sfile,
3371 (uint8_t*) blru->blk,
3372 source->blksize,
3373 & nread)))
3374 {
3375 return ret;
3376 }
3377
3378 if (nread != source->blksize)
3379 {
3380 IF_DEBUG1 (DP(RINT "[getblk] short skip block nread = %u\n",
3381 nread));
3382 stream->msg = "non-seekable input is short";
3383 return XD3_INVALID_INPUT;
3384 }
3385
3386 sfile->source_position += nread;
3387 blru->size = nread;
3388
3389 IF_DEBUG1 (DP(RINT "[getblk] skip blkno %"Q"u size %u\n",
3390 skip_blkno, blru->size));
3391
3392 XD3_ASSERT (sfile->source_position <= pos);
3393 }
3394 }
3395
3396 return 0;
3397}
3398
3399/* This is the callback for reading a block of source. This function
3400 * is blocking and it implements a small LRU.
3401 *
3402 * Note that it is possible for main_input() to handle getblk requests
3403 * in a non-blocking manner. If the callback is NULL then the caller
3404 * of xd3_*_input() must handle the XD3_GETSRCBLK return value and
3405 * fill the source in the same way. See xd3_getblk for details. To
3406 * see an example of non-blocking getblk, see xdelta-test.h. */
3407static int
3408main_getblk_func (xd3_stream *stream,
3409 xd3_source *source,
3410 xoff_t blkno)
3411{
3412 int ret = 0;
3413 xoff_t pos = blkno * source->blksize;
3414 main_file *sfile = (main_file*) source->ioh;
3415 main_blklru *blru;
3416 int is_new;
3417 int did_seek = 0;
3418 usize_t nread = 0;
3419
3420 if (allow_fake_source)
3421 {
3422 source->curblkno = blkno;
3423 source->onblk = 0;
3424 source->curblk = lru[0].blk;
3425 lru[0].size = 0;
3426 return 0;
3427 }
3428
3429 if ((ret = main_getblk_lru (source, blkno, & blru, & is_new)))
3430 {
3431 return ret;
3432 }
3433
3434 if (!is_new)
3435 {
3436 source->curblkno = blkno;
3437 source->onblk = blru->size;
3438 source->curblk = blru->blk;
3439 lru_hits++;
3440 return 0;
3441 }
3442
3443 lru_misses += 1;
3444
3445 if (pos != sfile->source_position)
3446 {
3447 /* Only try to seek when the position is wrong. This means the
3448 * decoder will fail when the source buffer is too small, but
3449 * only when the input is non-seekable. */
3450 if ((ret = main_read_seek_source (stream, source, blkno)))
3451 {
3452 return ret;
3453 }
3454
3455 /* Indicates that another call to main_getblk_lru() may be
3456 * needed */
3457 did_seek = 1;
3458 }
3459
3460 XD3_ASSERT (sfile->source_position == pos);
3461
3462 if (did_seek &&
3463 (ret = main_getblk_lru (source, blkno, & blru, & is_new)))
3464 {
3465 return ret;
3466 }
3467
3468 if ((ret = main_read_primary_input (sfile,
3469 (uint8_t*) blru->blk,
3470 source->blksize,
3471 & nread)))
3472 {
3473 return ret;
3474 }
3475
3476 /* Save the last block read, used to handle non-seekable files. */
3477 sfile->source_position = pos + nread;
3478
3479 if (option_verbose > 3)
3480 {
3481 if (blru->blkno != (xoff_t)-1)
3482 {
3483 if (blru->blkno != blkno)
3484 {
3485 XPR(NT "source block %"Q"u ejects %"Q"u (lru_hits=%u, "
3486 "lru_misses=%u, lru_filled=%u)\n",
3487 blkno, blru->blkno, lru_hits, lru_misses, lru_filled);
3488 }
3489 else
3490 {
3491 XPR(NT "source block %"Q"u read (lru_hits=%u, "
3492 "lru_misses=%u, lru_filled=%u)\n",
3493 blkno, lru_hits, lru_misses, lru_filled);
3494 }
3495 }
3496 else
3497 {
3498 XPR(NT "source block %"Q"u read (lru_hits=%u, lru_misses=%u, "
3499 "lru_filled=%u)\n", blkno, lru_hits, lru_misses, lru_filled);
3500 }
3501 }
3502
3503 source->curblk = blru->blk;
3504 source->curblkno = blkno;
3505 source->onblk = nread;
3506 blru->size = nread;
3507
3508 IF_DEBUG1 (DP(RINT "[main_getblk] blkno %"Q"u onblk %u pos %"Q"u "
3509 "srcpos %"Q"u\n",
3510 blkno, nread, pos, sfile->source_position));
3511
3512 return 0;
3513}
3514
3515/********************************************************************* 2959/*********************************************************************
3516 Main routines 2960 Main routines
3517 ********************************************************************/ 2961 ********************************************************************/
@@ -4015,8 +3459,6 @@ done:
4015static void 3459static void
4016main_cleanup (void) 3460main_cleanup (void)
4017{ 3461{
4018 usize_t i;
4019
4020 if (appheader_used != NULL && 3462 if (appheader_used != NULL &&
4021 appheader_used != option_appheader) 3463 appheader_used != option_appheader)
4022 { 3464 {
@@ -4033,17 +3475,7 @@ main_cleanup (void)
4033 ext_tmpfile = NULL; 3475 ext_tmpfile = NULL;
4034#endif 3476#endif
4035 3477
4036 for (i = 0; lru && i < lru_size; i += 1) 3478 main_lru_cleanup();
4037 {
4038 main_free (lru[i].blk);
4039 }
4040
4041 main_free (lru);
4042 lru = NULL;
4043
4044 lru_hits = 0;
4045 lru_misses = 0;
4046 lru_filled = 0;
4047 3479
4048 if (recode_stream != NULL) 3480 if (recode_stream != NULL)
4049 { 3481 {
@@ -4301,9 +3733,9 @@ main (int argc, char **argv)
4301 option_level = ret - '0'; 3733 option_level = ret - '0';
4302 break; 3734 break;
4303 case 'f': option_force = 1; break; 3735 case 'f': option_force = 1; break;
4304 case 'F': 3736 case 'F':
4305#if EXTERNAL_COMPRESSION 3737#if EXTERNAL_COMPRESSION
4306 option_force2 = 1; 3738 option_force2 = 1;
4307#else 3739#else
4308 XPR(NT "warning: -F option ignored, " 3740 XPR(NT "warning: -F option ignored, "
4309 "external compression support was not compiled\n"); 3741 "external compression support was not compiled\n");
generated by cgit v1.2.3 (git 2.39.1) at 2024-08-31 06:39:58 +0000