diff options
author | josh.macdonald <jmacd@users.noreply.github.com> | 2010-10-25 15:23:29 +0000 |
---|---|---|
committer | josh.macdonald <jmacd@users.noreply.github.com> | 2010-10-25 15:23:29 +0000 |
commit | 46593c36a012edda887a0072cf93949086f33b75 (patch) | |
tree | 8856f9fcb77a067a10063692ae978b3754c3b782 | |
parent | 63836024013a4b2859efc792c8ff2aaa65193641 (diff) |
Move block-cache code from -main.h to -blkcache.h. Previous merge was
incorrect - contents were branched post-code-removal. To identify diffs
in blkcache.h see previous version of -main.h
-rw-r--r-- | xdelta3/Makefile | 3 | ||||
-rw-r--r-- | xdelta3/linkxd3lib.c | 4 | ||||
-rw-r--r-- | xdelta3/xdelta3-blkcache.h | 4202 | ||||
-rw-r--r-- | xdelta3/xdelta3-main.h | 646 |
4 files changed, 422 insertions, 4433 deletions
diff --git a/xdelta3/Makefile b/xdelta3/Makefile index 08f9a03..88648e2 100644 --- a/xdelta3/Makefile +++ b/xdelta3/Makefile | |||
@@ -14,7 +14,8 @@ SWIGTGT = xdelta3module.dll | |||
14 | PYTGT = build/lib.cygwin-1.5.24-i686-$(PYVER)/xdelta3main.dll | 14 | PYTGT = build/lib.cygwin-1.5.24-i686-$(PYVER)/xdelta3main.dll |
15 | endif | 15 | endif |
16 | 16 | ||
17 | SOURCES = xdelta3-cfgs.h \ | 17 | SOURCES = xdelta3-blkcache.h \ |
18 | xdelta3-cfgs.h \ | ||
18 | xdelta3-decode.h \ | 19 | xdelta3-decode.h \ |
19 | xdelta3-djw.h \ | 20 | xdelta3-djw.h \ |
20 | xdelta3-fgk.h \ | 21 | xdelta3-fgk.h \ |
diff --git a/xdelta3/linkxd3lib.c b/xdelta3/linkxd3lib.c index 284cb0d..0f7f739 100644 --- a/xdelta3/linkxd3lib.c +++ b/xdelta3/linkxd3lib.c | |||
@@ -23,7 +23,6 @@ int main() { | |||
23 | xd3_avail_input (& stream, NULL, 0); | 23 | xd3_avail_input (& stream, NULL, 0); |
24 | xd3_consume_output (& stream); | 24 | xd3_consume_output (& stream); |
25 | 25 | ||
26 | use (xd3_bytes_on_srcblk (& source, 0)); | ||
27 | use (xd3_set_source (& stream, & source)); | 26 | use (xd3_set_source (& stream, & source)); |
28 | xd3_set_flags (& stream, 0); | 27 | xd3_set_flags (& stream, 0); |
29 | 28 | ||
@@ -31,9 +30,6 @@ int main() { | |||
31 | use (xd3_decode_input (&stream)); | 30 | use (xd3_decode_input (&stream)); |
32 | use (xd3_get_appheader (& stream, NULL, NULL)); | 31 | use (xd3_get_appheader (& stream, NULL, NULL)); |
33 | 32 | ||
34 | use ((int) xd3_errstring (& stream)); | ||
35 | use ((int) xd3_strerror (0)); | ||
36 | |||
37 | #if XD3_ENCODER | 33 | #if XD3_ENCODER |
38 | use (xd3_encode_input (&stream)); | 34 | use (xd3_encode_input (&stream)); |
39 | use (xd3_encode_stream (& stream, NULL, 0, NULL, NULL, 0)); | 35 | use (xd3_encode_stream (& stream, NULL, 0, NULL, NULL, 0)); |
diff --git a/xdelta3/xdelta3-blkcache.h b/xdelta3/xdelta3-blkcache.h index e47d708..d01322e 100644 --- a/xdelta3/xdelta3-blkcache.h +++ b/xdelta3/xdelta3-blkcache.h | |||
@@ -18,4034 +18,594 @@ | |||
18 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | 18 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
19 | */ | 19 | */ |
20 | 20 | ||
21 | /* This is all the extra stuff you need for convenience to users in a | 21 | typedef struct _main_blklru main_blklru; |
22 | * command line application. It contains these major components: | 22 | typedef struct _main_blklru_list main_blklru_list; |
23 | * | ||
24 | * 1. VCDIFF tools 2. external compression support (this is | ||
25 | * POSIX-specific). 3. a general read/write loop that handles all of | ||
26 | * the Xdelta decode/encode/VCDIFF-print functions 4. command-line | ||
27 | * interpreter 5. an Xdelta application header which stores default | ||
28 | * filename, external compression settings 6. output/error printing | ||
29 | * 7. basic file support and OS interface | ||
30 | */ | ||
31 | 23 | ||
32 | /* TODO list: 1. do exact gzip-like filename, stdout handling. make a | 24 | struct _main_blklru_list |
33 | * .vcdiff extension, refuse to encode to stdout without -cf, etc. | ||
34 | * 2. Allow the user to add a comment string to the app header without | ||
35 | * disturbing the default behavior. 3. "Source file must be seekable" | ||
36 | * is not actually true for encoding, given current behavior. Allow | ||
37 | * non-seekable sources? It would in theory let you use a fifo for | ||
38 | * the source. | ||
39 | */ | ||
40 | |||
41 | /* On error handling and printing: | ||
42 | * | ||
43 | * The xdelta library sets stream->msg to indicate what condition | ||
44 | * caused an internal failure, but many failures originate here and | ||
45 | * are printed here. The return convention is 0 for success, as | ||
46 | * throughout Xdelta code, but special attention is required here for | ||
47 | * the operating system calls with different error handling. See the | ||
48 | * main_file_* routines. All errors in this file have a message | ||
49 | * printed at the time of occurance. Since some of these calls occur | ||
50 | * within calls to the library, the error may end up being printed | ||
51 | * again with a more general error message. | ||
52 | */ | ||
53 | |||
54 | /*********************************************************************/ | ||
55 | |||
56 | #ifndef XD3_POSIX | ||
57 | #define XD3_POSIX 0 | ||
58 | #endif | ||
59 | #ifndef XD3_STDIO | ||
60 | #define XD3_STDIO 0 | ||
61 | #endif | ||
62 | #ifndef XD3_WIN32 | ||
63 | #define XD3_WIN32 0 | ||
64 | #endif | ||
65 | #ifndef NOT_MAIN | ||
66 | #define NOT_MAIN 0 | ||
67 | #endif | ||
68 | |||
69 | /* Combines xd3_strerror() and strerror() */ | ||
70 | const char* xd3_mainerror(int err_num); | ||
71 | |||
72 | /* XPRINTX (used by main) prefixes an "xdelta3: " to the output. */ | ||
73 | #define XPR fprintf | ||
74 | #define NT stderr, "xdelta3: " | ||
75 | |||
76 | /* If none are set, default to posix. */ | ||
77 | #if (XD3_POSIX + XD3_STDIO + XD3_WIN32) == 0 | ||
78 | #undef XD3_POSIX | ||
79 | #define XD3_POSIX 1 | ||
80 | #endif | ||
81 | |||
82 | /* Handle externally-compressed inputs. */ | ||
83 | #ifndef EXTERNAL_COMPRESSION | ||
84 | #define EXTERNAL_COMPRESSION 1 | ||
85 | #endif | ||
86 | |||
87 | #define PRINTHDR_SPECIAL -4378291 | ||
88 | |||
89 | /* The number of soft-config variables. */ | ||
90 | #define XD3_SOFTCFG_VARCNT 7 | ||
91 | |||
92 | /* this is used as in XPR(NT XD3_LIB_ERRMSG (stream, ret)) to print an | ||
93 | * error message from the library. */ | ||
94 | #define XD3_LIB_ERRMSG(stream, ret) "%s: %s\n", \ | ||
95 | xd3_errstring (stream), xd3_mainerror (ret) | ||
96 | |||
97 | #include <stdio.h> /* fprintf */ | ||
98 | |||
99 | #if XD3_POSIX | ||
100 | #include <unistd.h> /* close, read, write... */ | ||
101 | #include <sys/types.h> | ||
102 | #include <fcntl.h> | ||
103 | #endif | ||
104 | |||
105 | #ifndef _WIN32 | ||
106 | #include <unistd.h> /* lots */ | ||
107 | #include <sys/time.h> /* gettimeofday() */ | ||
108 | #include <sys/stat.h> /* stat() and fstat() */ | ||
109 | #else | ||
110 | #if defined(_MSC_VER) | ||
111 | #define strtoll _strtoi64 | ||
112 | #endif | ||
113 | #include <sys/types.h> | ||
114 | #include <sys/stat.h> | ||
115 | #ifndef WIFEXITED | ||
116 | # define WIFEXITED(stat) (((*((int *) &(stat))) & 0xff) == 0) | ||
117 | #endif | ||
118 | #ifndef WEXITSTATUS | ||
119 | # define WEXITSTATUS(stat) (((*((int *) &(stat))) >> 8) & 0xff) | ||
120 | #endif | ||
121 | #ifndef S_ISREG | ||
122 | //# ifdef S_IFREG | ||
123 | //# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) | ||
124 | //# else | ||
125 | # define S_ISREG(m) 1 | ||
126 | //# endif | ||
127 | #endif /* !S_ISREG */ | ||
128 | |||
129 | // For standard input/output handles | ||
130 | static STARTUPINFO winStartupInfo; | ||
131 | #endif | ||
132 | |||
133 | /********************************************************************** | ||
134 | ENUMS and TYPES | ||
135 | *********************************************************************/ | ||
136 | |||
137 | /* These flags (mainly pertaining to main_read() operations) are set | ||
138 | * in the main_file->flags variable. All are related to with external | ||
139 | * decompression support. | ||
140 | * | ||
141 | * RD_FIRST causes the external decompression check when the input is | ||
142 | * first read. | ||
143 | * | ||
144 | * RD_NONEXTERNAL disables external decompression for reading a | ||
145 | * compressed input, in the case of Xdelta inputs. Note: Xdelta is | ||
146 | * supported as an external compression type, which makes is the | ||
147 | * reason for this flag. An example to justify this is: to create a | ||
148 | * delta between two files that are VCDIFF-compressed. Two external | ||
149 | * Xdelta decoders are run to supply decompressed source and target | ||
150 | * inputs to the Xdelta encoder. */ | ||
151 | typedef enum | ||
152 | { | 25 | { |
153 | RD_FIRST = (1 << 0), | 26 | main_blklru_list *next; |
154 | RD_NONEXTERNAL = (1 << 1), | 27 | main_blklru_list *prev; |
155 | RD_DECOMPSET = (1 << 2), | ||
156 | RD_MAININPUT = (1 << 3), | ||
157 | } xd3_read_flags; | ||
158 | |||
159 | /* main_file->mode values */ | ||
160 | typedef enum | ||
161 | { | ||
162 | XO_READ = 0, | ||
163 | XO_WRITE = 1, | ||
164 | } main_file_modes; | ||
165 | |||
166 | /* Main commands. For example, CMD_PRINTHDR is the "xdelta printhdr" | ||
167 | * command. */ | ||
168 | typedef enum | ||
169 | { | ||
170 | CMD_NONE = 0, | ||
171 | CMD_PRINTHDR, | ||
172 | CMD_PRINTHDRS, | ||
173 | CMD_PRINTDELTA, | ||
174 | CMD_RECODE, | ||
175 | CMD_MERGE_ARG, | ||
176 | CMD_MERGE, | ||
177 | #if XD3_ENCODER | ||
178 | CMD_ENCODE, | ||
179 | #endif | ||
180 | CMD_DECODE, | ||
181 | CMD_TEST, | ||
182 | CMD_CONFIG, | ||
183 | } xd3_cmd; | ||
184 | |||
185 | #if XD3_ENCODER | ||
186 | #define CMD_DEFAULT CMD_ENCODE | ||
187 | #define IS_ENCODE(cmd) (cmd == CMD_ENCODE) | ||
188 | #else | ||
189 | #define CMD_DEFAULT CMD_DECODE | ||
190 | #define IS_ENCODE(cmd) (0) | ||
191 | #endif | ||
192 | |||
193 | typedef struct _main_file main_file; | ||
194 | typedef struct _main_extcomp main_extcomp; | ||
195 | typedef struct _main_merge main_merge; | ||
196 | typedef struct _main_merge_list main_merge_list; | ||
197 | |||
198 | /* The main_file object supports abstract system calls like open, | ||
199 | * close, read, write, seek, stat. The program uses these to | ||
200 | * represent both seekable files and non-seekable files. Source files | ||
201 | * must be seekable, but the target input and any output file do not | ||
202 | * require seekability. | ||
203 | */ | ||
204 | struct _main_file | ||
205 | { | ||
206 | #if XD3_STDIO | ||
207 | FILE *file; | ||
208 | #elif XD3_POSIX | ||
209 | int file; | ||
210 | #elif XD3_WIN32 | ||
211 | HANDLE file; | ||
212 | #endif | ||
213 | |||
214 | int mode; /* XO_READ and XO_WRITE */ | ||
215 | const char *filename; /* File name or /dev/stdin, | ||
216 | * /dev/stdout, /dev/stderr. */ | ||
217 | char *filename_copy; /* File name or /dev/stdin, | ||
218 | * /dev/stdout, /dev/stderr. */ | ||
219 | const char *realname; /* File name or /dev/stdin, | ||
220 | * /dev/stdout, /dev/stderr. */ | ||
221 | const main_extcomp *compressor; /* External compression struct. */ | ||
222 | int flags; /* RD_FIRST, RD_NONEXTERNAL, ... */ | ||
223 | xoff_t nread; /* for input position */ | ||
224 | xoff_t nwrite; /* for output position */ | ||
225 | uint8_t *snprintf_buf; /* internal snprintf() use */ | ||
226 | int size_known; /* Set by main_set_souze */ | ||
227 | xoff_t source_position; /* for avoiding seek in getblk_func */ | ||
228 | int seek_failed; /* after seek fails once, try FIFO */ | ||
229 | }; | 28 | }; |
230 | 29 | ||
231 | /* Various strings and magic values used to detect and call external | 30 | struct _main_blklru |
232 | * compression. See below for examples. */ | ||
233 | struct _main_extcomp | ||
234 | { | 31 | { |
235 | const char *recomp_cmdname; | 32 | uint8_t *blk; |
236 | const char *recomp_options; | 33 | xoff_t blkno; |
237 | 34 | usize_t size; | |
238 | const char *decomp_cmdname; | 35 | main_blklru_list link; |
239 | const char *decomp_options; | ||
240 | |||
241 | const char *ident; | ||
242 | const char *magic; | ||
243 | usize_t magic_size; | ||
244 | int flags; | ||
245 | }; | 36 | }; |
246 | 37 | ||
247 | /* Merge state: */ | 38 | #define MAX_LRU_SIZE 32U |
39 | #define XD3_MINSRCWINSZ XD3_ALLOCSIZE | ||
248 | 40 | ||
249 | struct _main_merge_list | 41 | XD3_MAKELIST(main_blklru_list,main_blklru,link); |
250 | { | ||
251 | main_merge_list *next; | ||
252 | main_merge_list *prev; | ||
253 | }; | ||
254 | |||
255 | struct _main_merge | ||
256 | { | ||
257 | const char *filename; | ||
258 | |||
259 | main_merge_list link; | ||
260 | }; | ||
261 | |||
262 | XD3_MAKELIST(main_merge_list,main_merge,link); | ||
263 | |||
264 | // TODO: really need to put options in a struct so that internal | ||
265 | // callers can easily reset state. | ||
266 | |||
267 | #define DEFAULT_VERBOSE 0 | ||
268 | |||
269 | /* Program options: various command line flags and options. */ | ||
270 | static int option_stdout = 0; | ||
271 | static int option_force = 0; | ||
272 | static int option_verbose = DEFAULT_VERBOSE; | ||
273 | static int option_quiet = 0; | ||
274 | static int option_use_appheader = 1; | ||
275 | static uint8_t* option_appheader = NULL; | ||
276 | static int option_use_secondary = 0; | ||
277 | static const char* option_secondary = NULL; | ||
278 | static int option_use_checksum = 1; | ||
279 | static int option_use_altcodetable = 0; | ||
280 | static const char* option_smatch_config = NULL; | ||
281 | static int option_no_compress = 0; | ||
282 | static int option_no_output = 0; /* do not write output */ | ||
283 | static const char *option_source_filename = NULL; | ||
284 | |||
285 | static int option_level = XD3_DEFAULT_LEVEL; | ||
286 | static usize_t option_iopt_size = XD3_DEFAULT_IOPT_SIZE; | ||
287 | static usize_t option_winsize = XD3_DEFAULT_WINSIZE; | ||
288 | static usize_t option_srcwinsz = XD3_DEFAULT_SRCWINSZ; | ||
289 | static usize_t option_sprevsz = XD3_DEFAULT_SPREVSZ; | ||
290 | |||
291 | /* These variables are supressed to avoid their use w/o support. main() warns | ||
292 | * appropriately when external compression is not enabled. */ | ||
293 | #if EXTERNAL_COMPRESSION | ||
294 | static int num_subprocs = 0; | ||
295 | static int option_force2 = 0; | ||
296 | static int option_decompress_inputs = 1; | ||
297 | static int option_recompress_outputs = 1; | ||
298 | #endif | ||
299 | |||
300 | /* This is for comparing "printdelta" output without attention to | ||
301 | * copy-instruction modes. */ | ||
302 | #if VCDIFF_TOOLS | ||
303 | static int option_print_cpymode = 1; /* Note: see reset_defaults(). */ | ||
304 | #endif | ||
305 | |||
306 | /* Static variables */ | ||
307 | IF_DEBUG(static int main_mallocs = 0;) | ||
308 | |||
309 | static char* program_name = NULL; | ||
310 | static uint8_t* appheader_used = NULL; | ||
311 | static uint8_t* main_bdata = NULL; | ||
312 | static usize_t main_bsize = 0; | ||
313 | |||
314 | /* Hacks for VCDIFF tools */ | ||
315 | static int allow_fake_source = 0; | ||
316 | |||
317 | /* recode_stream is used by both recode/merge for reading vcdiff inputs */ | ||
318 | static xd3_stream *recode_stream = NULL; | ||
319 | |||
320 | /* merge_stream is used by merge commands for storing the source encoding */ | ||
321 | static xd3_stream *merge_stream = NULL; | ||
322 | |||
323 | /* This array of compressor types is compiled even if EXTERNAL_COMPRESSION is | ||
324 | * false just so the program knows the mapping of IDENT->NAME. */ | ||
325 | static main_extcomp extcomp_types[] = | ||
326 | { | ||
327 | { "bzip2", "-c", "bzip2", "-dc", "B", "BZh", 3, 0 }, | ||
328 | { "gzip", "-c", "gzip", "-dc", "G", "\037\213", 2, 0 }, | ||
329 | { "compress", "-c", "uncompress", "-c", "Z", "\037\235", 2, 0 }, | ||
330 | |||
331 | /* TODO: add commandline support for magic-less formats */ | ||
332 | /*{ "lzma", "-c", "lzma", "-dc", "M", "]\000", 2, 0 },*/ | ||
333 | 42 | ||
334 | /* Xz is lzma with a magic number http://tukaani.org/xz/ */ | 43 | static usize_t lru_size = 0; |
335 | { "xz", "-c", "xz", "-dc", "Y", "\xfd\x37\x7a\x58\x5a\x00", 2, 0 }, | 44 | static main_blklru *lru = NULL; /* array of lru_size elts */ |
336 | }; | 45 | static main_blklru_list lru_list; |
337 | 46 | static main_blklru_list lru_free; | |
338 | static int main_input (xd3_cmd cmd, main_file *ifile, | 47 | static int do_src_fifo = 0; /* set to avoid lru */ |
339 | main_file *ofile, main_file *sfile); | ||
340 | static void main_get_appheader (xd3_stream *stream, main_file *ifile, | ||
341 | main_file *output, main_file *sfile); | ||
342 | |||
343 | static int main_getblk_func (xd3_stream *stream, | ||
344 | xd3_source *source, | ||
345 | xoff_t blkno); | ||
346 | static void main_free (void *ptr); | ||
347 | static void* main_malloc (usize_t size); | ||
348 | |||
349 | static int main_file_open (main_file *xfile, const char* name, int mode); | ||
350 | static int main_file_stat (main_file *xfile, xoff_t *size); | ||
351 | static int main_file_seek (main_file *xfile, xoff_t pos); | ||
352 | static int main_read_primary_input (main_file *file, | ||
353 | uint8_t *buf, | ||
354 | usize_t size, | ||
355 | usize_t *nread); | ||
356 | 48 | ||
357 | static const char* main_format_bcnt (xoff_t r, char *buf); | 49 | static int lru_hits = 0; |
358 | static int main_help (void); | 50 | static int lru_misses = 0; |
51 | static int lru_filled = 0; | ||
359 | 52 | ||
360 | /* The code in xdelta3-blk.h is essentially part of this unit, see | 53 | static void main_lru_reset() |
361 | * comments there. */ | ||
362 | #include "xdelta3-blkcache.h" | ||
363 | |||
364 | static int | ||
365 | main_version (void) | ||
366 | { | 54 | { |
367 | /* $Format: " DP(RINT \"Xdelta version $Xdelta3Version$, Copyright (C) 2007, 2008, 2009, 2010, Joshua MacDonald\n\");" $ */ | 55 | lru_size = 0; |
368 | DP(RINT "Xdelta version 3.0z, Copyright (C) 2007, 2008, 2009, 2010, Joshua MacDonald\n"); | 56 | lru = NULL; |
369 | DP(RINT "Xdelta comes with ABSOLUTELY NO WARRANTY.\n"); | 57 | do_src_fifo = 0; |
370 | DP(RINT "This is free software, and you are welcome to redistribute it\n"); | 58 | lru_hits = 0; |
371 | DP(RINT "under certain conditions; see \"COPYING\" for details.\n"); | 59 | lru_misses = 0; |
372 | return EXIT_SUCCESS; | 60 | lru_filled = 0; |
373 | } | ||
374 | |||
375 | static int | ||
376 | main_config (void) | ||
377 | { | ||
378 | main_version (); | ||
379 | |||
380 | DP(RINT "EXTERNAL_COMPRESSION=%d\n", EXTERNAL_COMPRESSION); | ||
381 | DP(RINT "GENERIC_ENCODE_TABLES=%d\n", GENERIC_ENCODE_TABLES); | ||
382 | DP(RINT "GENERIC_ENCODE_TABLES_COMPUTE=%d\n", GENERIC_ENCODE_TABLES_COMPUTE); | ||
383 | DP(RINT "REGRESSION_TEST=%d\n", REGRESSION_TEST); | ||
384 | DP(RINT "SECONDARY_DJW=%d\n", SECONDARY_DJW); | ||
385 | DP(RINT "SECONDARY_FGK=%d\n", SECONDARY_FGK); | ||
386 | DP(RINT "UNALIGNED_OK=%d\n", UNALIGNED_OK); | ||
387 | DP(RINT "VCDIFF_TOOLS=%d\n", VCDIFF_TOOLS); | ||
388 | DP(RINT "XD3_ALLOCSIZE=%d\n", XD3_ALLOCSIZE); | ||
389 | DP(RINT "XD3_DEBUG=%d\n", XD3_DEBUG); | ||
390 | DP(RINT "XD3_ENCODER=%d\n", XD3_ENCODER); | ||
391 | DP(RINT "XD3_POSIX=%d\n", XD3_POSIX); | ||
392 | DP(RINT "XD3_STDIO=%d\n", XD3_STDIO); | ||
393 | DP(RINT "XD3_WIN32=%d\n", XD3_WIN32); | ||
394 | DP(RINT "XD3_USE_LARGEFILE64=%d\n", XD3_USE_LARGEFILE64); | ||
395 | DP(RINT "XD3_DEFAULT_LEVEL=%d\n", XD3_DEFAULT_LEVEL); | ||
396 | DP(RINT "XD3_DEFAULT_IOPT_SIZE=%d\n", XD3_DEFAULT_IOPT_SIZE); | ||
397 | DP(RINT "XD3_DEFAULT_SPREVSZ=%d\n", XD3_DEFAULT_SPREVSZ); | ||
398 | DP(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ); | ||
399 | DP(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE); | ||
400 | DP(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE); | ||
401 | DP(RINT "sizeof(void*)=%d\n", (int)sizeof(void*)); | ||
402 | DP(RINT "sizeof(int)=%d\n", (int)sizeof(int)); | ||
403 | DP(RINT "sizeof(uint32_t)=%d\n", (int)sizeof(uint32_t)); | ||
404 | DP(RINT "sizeof(uint64_t)=%d\n", (int)sizeof(uint64_t)); | ||
405 | DP(RINT "sizeof(usize_t)=%d\n", (int)sizeof(usize_t)); | ||
406 | DP(RINT "sizeof(xoff_t)=%d\n", (int)sizeof(xoff_t)); | ||
407 | |||
408 | return EXIT_SUCCESS; | ||
409 | } | ||
410 | |||
411 | static void | ||
412 | reset_defaults(void) | ||
413 | { | ||
414 | option_stdout = 0; | ||
415 | option_force = 0; | ||
416 | option_verbose = DEFAULT_VERBOSE; | ||
417 | option_quiet = 0; | ||
418 | option_appheader = NULL; | ||
419 | option_use_secondary = 0; | ||
420 | option_secondary = NULL; | ||
421 | option_use_altcodetable = 0; | ||
422 | option_smatch_config = NULL; | ||
423 | option_no_compress = 0; | ||
424 | option_no_output = 0; | ||
425 | option_source_filename = NULL; | ||
426 | program_name = NULL; | ||
427 | appheader_used = NULL; | ||
428 | main_bdata = NULL; | ||
429 | main_bsize = 0; | ||
430 | allow_fake_source = 0; | ||
431 | option_smatch_config = NULL; | ||
432 | |||
433 | main_lru_reset(); | ||
434 | |||
435 | option_use_appheader = 1; | ||
436 | option_use_checksum = 1; | ||
437 | #if EXTERNAL_COMPRESSION | ||
438 | option_force2 = 0; | ||
439 | option_decompress_inputs = 1; | ||
440 | option_recompress_outputs = 1; | ||
441 | num_subprocs = 0; | ||
442 | #endif | ||
443 | #if VCDIFF_TOOLS | ||
444 | option_print_cpymode = 1; | ||
445 | #endif | ||
446 | option_level = XD3_DEFAULT_LEVEL; | ||
447 | option_iopt_size = XD3_DEFAULT_IOPT_SIZE; | ||
448 | option_winsize = XD3_DEFAULT_WINSIZE; | ||
449 | option_srcwinsz = XD3_DEFAULT_SRCWINSZ; | ||
450 | option_sprevsz = XD3_DEFAULT_SPREVSZ; | ||
451 | } | ||
452 | |||
453 | static void* | ||
454 | main_malloc1 (usize_t size) | ||
455 | { | ||
456 | void* r = malloc (size); | ||
457 | if (r == NULL) { XPR(NT "malloc: %s\n", xd3_mainerror (ENOMEM)); } | ||
458 | else if (option_verbose > 4) { XPR(NT "malloc: %u: %p\n", size, r); } | ||
459 | return r; | ||
460 | } | ||
461 | |||
462 | static void* | ||
463 | main_malloc (usize_t size) | ||
464 | { | ||
465 | void *r = main_malloc1 (size); | ||
466 | if (r) { IF_DEBUG (main_mallocs += 1); } | ||
467 | return r; | ||
468 | } | ||
469 | |||
470 | static void* | ||
471 | main_alloc (void *opaque, | ||
472 | usize_t items, | ||
473 | usize_t size) | ||
474 | { | ||
475 | return main_malloc1 (items * size); | ||
476 | } | ||
477 | |||
478 | static void | ||
479 | main_free1 (void *opaque, void *ptr) | ||
480 | { | ||
481 | if (option_verbose > 4) { XPR(NT "free: %p\n", ptr); } | ||
482 | free (ptr); | ||
483 | } | ||
484 | |||
485 | static void | ||
486 | main_free (void *ptr) | ||
487 | { | ||
488 | if (ptr) | ||
489 | { | ||
490 | IF_DEBUG (main_mallocs -= 1); | ||
491 | main_free1 (NULL, ptr); | ||
492 | IF_DEBUG (XD3_ASSERT(main_mallocs >= 0)); | ||
493 | } | ||
494 | } | ||
495 | |||
496 | /* This ensures that (ret = errno) always indicates failure, in case errno was | ||
497 | * accidentally not set. If this prints there's a bug somewhere. */ | ||
498 | static int | ||
499 | get_errno (void) | ||
500 | { | ||
501 | #ifndef _WIN32 | ||
502 | if (errno == 0) | ||
503 | { | ||
504 | XPR(NT "you found a bug: expected errno != 0\n"); | ||
505 | errno = XD3_INTERNAL; | ||
506 | } | ||
507 | return errno; | ||
508 | #else | ||
509 | DWORD err_num = GetLastError(); | ||
510 | if (err_num == NO_ERROR) | ||
511 | { | ||
512 | err_num = XD3_INTERNAL; | ||
513 | } | ||
514 | return err_num; | ||
515 | #endif | ||
516 | } | ||
517 | |||
518 | const char* | ||
519 | xd3_mainerror(int err_num) { | ||
520 | #ifndef _WIN32 | ||
521 | const char* x = xd3_strerror (err_num); | ||
522 | if (x != NULL) | ||
523 | { | ||
524 | return x; | ||
525 | } | ||
526 | return strerror(err_num); | ||
527 | #else | ||
528 | static char err_buf[256]; | ||
529 | const char* x = xd3_strerror (err_num); | ||
530 | if (x != NULL) | ||
531 | { | ||
532 | return x; | ||
533 | } | ||
534 | memset (err_buf, 0, 256); | ||
535 | FormatMessage (FORMAT_MESSAGE_FROM_SYSTEM | | ||
536 | FORMAT_MESSAGE_IGNORE_INSERTS, | ||
537 | NULL, err_num, | ||
538 | MAKELANGID (LANG_NEUTRAL, SUBLANG_DEFAULT), | ||
539 | err_buf, 256, NULL); | ||
540 | if (err_buf[0] != 0 && err_buf[strlen(err_buf) - 1] == '\n') | ||
541 | { | ||
542 | err_buf[strlen(err_buf) - 1] = 0; | ||
543 | } | ||
544 | return err_buf; | ||
545 | #endif | ||
546 | } | ||
547 | |||
548 | static long | ||
549 | get_millisecs_now (void) | ||
550 | { | ||
551 | #ifndef _WIN32 | ||
552 | struct timeval tv; | ||
553 | |||
554 | gettimeofday (& tv, NULL); | ||
555 | |||
556 | return (tv.tv_sec) * 1000L + (tv.tv_usec) / 1000; | ||
557 | #else | ||
558 | SYSTEMTIME st; | ||
559 | FILETIME ft; | ||
560 | __int64 *pi = (__int64*)&ft; | ||
561 | GetLocalTime(&st); | ||
562 | SystemTimeToFileTime(&st, &ft); | ||
563 | return (long)((*pi) / 10000); | ||
564 | #endif | ||
565 | } | ||
566 | |||
567 | /* Always >= 1 millisec, right? */ | ||
568 | static long | ||
569 | get_millisecs_since (void) | ||
570 | { | ||
571 | static long last = 0; | ||
572 | long now = get_millisecs_now(); | ||
573 | long diff = now - last; | ||
574 | last = now; | ||
575 | return diff; | ||
576 | } | ||
577 | |||
578 | static const char* | ||
579 | main_format_bcnt (xoff_t r, char *buf) | ||
580 | { | ||
581 | static const char* fmts[] = { "B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB" }; | ||
582 | usize_t i; | ||
583 | |||
584 | for (i = 0; i < SIZEOF_ARRAY(fmts) - 1; i += 1) | ||
585 | { | ||
586 | xoff_t new_r; | ||
587 | |||
588 | if (r == 0) | ||
589 | { | ||
590 | sprintf (buf, "0 %s", fmts[i]); | ||
591 | return buf; | ||
592 | } | ||
593 | |||
594 | if (r >= 1 && r < 10) | ||
595 | { | ||
596 | sprintf (buf, "%.2f %s", (double) r, fmts[i]); | ||
597 | return buf; | ||
598 | } | ||
599 | |||
600 | if (r >= 10 && r < 100) | ||
601 | { | ||
602 | sprintf (buf, "%.1f %s", (double) r, fmts[i]); | ||
603 | return buf; | ||
604 | } | ||
605 | |||
606 | if (r >= 100 && r < 1000) | ||
607 | { | ||
608 | sprintf (buf, "%"Q"u %s", r, fmts[i]); | ||
609 | return buf; | ||
610 | } | ||
611 | |||
612 | new_r = r / 1024; | ||
613 | |||
614 | if (new_r < 10) | ||
615 | { | ||
616 | sprintf (buf, "%.2f %s", (double) r / 1024.0, fmts[i + 1]); | ||
617 | return buf; | ||
618 | } | ||
619 | |||
620 | if (new_r < 100) | ||
621 | { | ||
622 | sprintf (buf, "%.1f %s", (double) r / 1024.0, fmts[i + 1]); | ||
623 | return buf; | ||
624 | } | ||
625 | |||
626 | r = new_r; | ||
627 | } | ||
628 | XD3_ASSERT (0); | ||
629 | return ""; | ||
630 | } | ||
631 | |||
632 | static char* | ||
633 | main_format_rate (xoff_t bytes, long millis, char *buf) | ||
634 | { | ||
635 | xoff_t r = (xoff_t)(1.0 * bytes / (1.0 * millis / 1000.0)); | ||
636 | static char lbuf[32]; | ||
637 | |||
638 | main_format_bcnt (r, lbuf); | ||
639 | sprintf (buf, "%s/s", lbuf); | ||
640 | return buf; | ||
641 | } | ||
642 | |||
643 | static char* | ||
644 | main_format_millis (long millis, char *buf) | ||
645 | { | ||
646 | if (millis < 1000) { sprintf (buf, "%lu ms", millis); } | ||
647 | else if (millis < 10000) { sprintf (buf, "%.1f sec", millis / 1000.0); } | ||
648 | else { sprintf (buf, "%lu sec", millis / 1000L); } | ||
649 | return buf; | ||
650 | } | ||
651 | |||
652 | /* A safe version of strtol for xoff_t. */ | ||
653 | static int | ||
654 | main_strtoxoff (const char* s, xoff_t *xo, char which) | ||
655 | { | ||
656 | char *e; | ||
657 | xoff_t x; | ||
658 | |||
659 | XD3_ASSERT(s && *s != 0); | ||
660 | |||
661 | { | ||
662 | /* Should check LONG_MIN, LONG_MAX, LLONG_MIN, LLONG_MAX? */ | ||
663 | #if SIZEOF_XOFF_T == 4 | ||
664 | long xx = strtol (s, &e, 0); | ||
665 | #else | ||
666 | long long xx = strtoll (s, &e, 0); | ||
667 | #endif | ||
668 | |||
669 | if (xx < 0) | ||
670 | { | ||
671 | XPR(NT "-%c: negative integer: %s\n", which, s); | ||
672 | return EXIT_FAILURE; | ||
673 | } | ||
674 | |||
675 | x = xx; | ||
676 | } | ||
677 | |||
678 | if (*e != 0) | ||
679 | { | ||
680 | XPR(NT "-%c: invalid integer: %s\n", which, s); | ||
681 | return EXIT_FAILURE; | ||
682 | } | ||
683 | |||
684 | (*xo) = x; | ||
685 | return 0; | ||
686 | } | ||
687 | |||
688 | static int | ||
689 | main_atou (const char* arg, usize_t *xo, usize_t low, | ||
690 | usize_t high, char which) | ||
691 | { | ||
692 | xoff_t x; | ||
693 | int ret; | ||
694 | |||
695 | if ((ret = main_strtoxoff (arg, & x, which))) { return ret; } | ||
696 | |||
697 | if (x < low) | ||
698 | { | ||
699 | XPR(NT "-%c: minimum value: %u\n", which, low); | ||
700 | return EXIT_FAILURE; | ||
701 | } | ||
702 | if (high == 0) | ||
703 | { | ||
704 | high = USIZE_T_MAX; | ||
705 | } | ||
706 | if (x > high) | ||
707 | { | ||
708 | XPR(NT "-%c: maximum value: %u\n", which, high); | ||
709 | return EXIT_FAILURE; | ||
710 | } | ||
711 | (*xo) = (usize_t)x; | ||
712 | return 0; | ||
713 | } | ||
714 | |||
715 | /****************************************************************** | ||
716 | FILE BASICS | ||
717 | ******************************************************************/ | ||
718 | |||
719 | /* With all the variation in file system-call semantics, arguments, | ||
720 | * return values and error-handling for the POSIX and STDIO file APIs, | ||
721 | * the insides of these functions make me sick, which is why these | ||
722 | * wrappers exist. */ | ||
723 | |||
724 | #define XOPEN_OPNAME (xfile->mode == XO_READ ? "read" : "write") | ||
725 | #define XOPEN_STDIO (xfile->mode == XO_READ ? "rb" : "wb") | ||
726 | #define XOPEN_POSIX (xfile->mode == XO_READ ? \ | ||
727 | O_RDONLY : O_WRONLY | O_CREAT | O_TRUNC) | ||
728 | #define XOPEN_MODE (xfile->mode == XO_READ ? 0 : 0666) | ||
729 | |||
730 | #define XF_ERROR(op, name, ret) \ | ||
731 | do { if (!option_quiet) { XPR(NT "file %s failed: %s: %s: %s\n", (op), \ | ||
732 | XOPEN_OPNAME, (name), xd3_mainerror (ret)); } } while (0) | ||
733 | |||
734 | #if XD3_STDIO | ||
735 | #define XFNO(f) fileno(f->file) | ||
736 | #define XSTDOUT_XF(f) { (f)->file = stdout; (f)->filename = "/dev/stdout"; } | ||
737 | #define XSTDIN_XF(f) { (f)->file = stdin; (f)->filename = "/dev/stdin"; } | ||
738 | |||
739 | #elif XD3_POSIX | ||
740 | #define XFNO(f) f->file | ||
741 | #define XSTDOUT_XF(f) \ | ||
742 | { (f)->file = STDOUT_FILENO; (f)->filename = "/dev/stdout"; } | ||
743 | #define XSTDIN_XF(f) \ | ||
744 | { (f)->file = STDIN_FILENO; (f)->filename = "/dev/stdin"; } | ||
745 | |||
746 | #elif XD3_WIN32 | ||
747 | #define XFNO(f) -1 | ||
748 | #define XSTDOUT_XF(f) { \ | ||
749 | (f)->file = GetStdHandle(STD_OUTPUT_HANDLE); \ | ||
750 | (f)->filename = "(stdout)"; \ | ||
751 | } | ||
752 | #define XSTDIN_XF(f) { \ | ||
753 | (f)->file = GetStdHandle(STD_INPUT_HANDLE); \ | ||
754 | (f)->filename = "(stdin)"; \ | ||
755 | } | ||
756 | #endif | ||
757 | |||
758 | static void | ||
759 | main_file_init (main_file *xfile) | ||
760 | { | ||
761 | memset (xfile, 0, sizeof (*xfile)); | ||
762 | |||
763 | #if XD3_POSIX | ||
764 | xfile->file = -1; | ||
765 | #endif | ||
766 | #if XD3_WIN32 | ||
767 | xfile->file = INVALID_HANDLE_VALUE; | ||
768 | #endif | ||
769 | } | ||
770 | |||
771 | static int | ||
772 | main_file_isopen (main_file *xfile) | ||
773 | { | ||
774 | #if XD3_STDIO | ||
775 | return xfile->file != NULL; | ||
776 | |||
777 | #elif XD3_POSIX | ||
778 | return xfile->file != -1; | ||
779 | |||
780 | #elif XD3_WIN32 | ||
781 | return xfile->file != INVALID_HANDLE_VALUE; | ||
782 | #endif | ||
783 | } | ||
784 | |||
785 | static int | ||
786 | main_file_close (main_file *xfile) | ||
787 | { | ||
788 | int ret = 0; | ||
789 | |||
790 | if (! main_file_isopen (xfile)) | ||
791 | { | ||
792 | return 0; | ||
793 | } | ||
794 | |||
795 | #if XD3_STDIO | ||
796 | ret = fclose (xfile->file); | ||
797 | xfile->file = NULL; | ||
798 | |||
799 | #elif XD3_POSIX | ||
800 | ret = close (xfile->file); | ||
801 | xfile->file = -1; | ||
802 | |||
803 | #elif XD3_WIN32 | ||
804 | if (!CloseHandle(xfile->file)) { | ||
805 | ret = get_errno (); | ||
806 | } | ||
807 | xfile->file = INVALID_HANDLE_VALUE; | ||
808 | #endif | ||
809 | |||
810 | if (ret != 0) { XF_ERROR ("close", xfile->filename, ret = get_errno ()); } | ||
811 | return ret; | ||
812 | } | ||
813 | |||
814 | static void | ||
815 | main_file_cleanup (main_file *xfile) | ||
816 | { | ||
817 | XD3_ASSERT (xfile != NULL); | ||
818 | |||
819 | if (main_file_isopen (xfile)) | ||
820 | { | ||
821 | main_file_close (xfile); | ||
822 | } | ||
823 | |||
824 | if (xfile->snprintf_buf != NULL) | ||
825 | { | ||
826 | main_free(xfile->snprintf_buf); | ||
827 | xfile->snprintf_buf = NULL; | ||
828 | } | ||
829 | |||
830 | if (xfile->filename_copy != NULL) | ||
831 | { | ||
832 | main_free(xfile->filename_copy); | ||
833 | xfile->filename_copy = NULL; | ||
834 | } | ||
835 | } | ||
836 | |||
837 | static int | ||
838 | main_file_open (main_file *xfile, const char* name, int mode) | ||
839 | { | ||
840 | int ret = 0; | ||
841 | |||
842 | xfile->mode = mode; | ||
843 | |||
844 | XD3_ASSERT (name != NULL); | ||
845 | XD3_ASSERT (! main_file_isopen (xfile)); | ||
846 | if (name[0] == 0) | ||
847 | { | ||
848 | XPR(NT "invalid file name: empty string\n"); | ||
849 | return XD3_INVALID; | ||
850 | } | ||
851 | |||
852 | #if XD3_STDIO | ||
853 | xfile->file = fopen (name, XOPEN_STDIO); | ||
854 | |||
855 | ret = (xfile->file == NULL) ? get_errno () : 0; | ||
856 | |||
857 | #elif XD3_POSIX | ||
858 | if ((ret = open (name, XOPEN_POSIX, XOPEN_MODE)) < 0) | ||
859 | { | ||
860 | ret = get_errno (); | ||
861 | } | ||
862 | else | ||
863 | { | ||
864 | xfile->file = ret; | ||
865 | ret = 0; | ||
866 | } | ||
867 | |||
868 | #elif XD3_WIN32 | ||
869 | xfile->file = CreateFile(name, | ||
870 | (mode == XO_READ) ? GENERIC_READ : GENERIC_WRITE, | ||
871 | FILE_SHARE_READ, | ||
872 | NULL, | ||
873 | (mode == XO_READ) ? | ||
874 | OPEN_EXISTING : | ||
875 | (option_force ? CREATE_ALWAYS : CREATE_NEW), | ||
876 | FILE_ATTRIBUTE_NORMAL, | ||
877 | NULL); | ||
878 | if (xfile->file == INVALID_HANDLE_VALUE) | ||
879 | { | ||
880 | ret = get_errno (); | ||
881 | } | ||
882 | #endif | ||
883 | if (ret) { XF_ERROR ("open", name, ret); } | ||
884 | else { xfile->realname = name; xfile->nread = 0; } | ||
885 | return ret; | ||
886 | } | ||
887 | |||
888 | static int | ||
889 | main_file_stat (main_file *xfile, xoff_t *size) | ||
890 | { | ||
891 | int ret = 0; | ||
892 | #if XD3_WIN32 | ||
893 | if (GetFileType(xfile->file) != FILE_TYPE_DISK) | ||
894 | { | ||
895 | return -1; | ||
896 | } | ||
897 | # if (_WIN32_WINNT >= 0x0500) | ||
898 | { | ||
899 | LARGE_INTEGER li; | ||
900 | if (GetFileSizeEx(xfile->file, &li) == 0) | ||
901 | { | ||
902 | return get_errno (); | ||
903 | } | ||
904 | *size = li.QuadPart; | ||
905 | } | ||
906 | # else | ||
907 | { | ||
908 | DWORD filesize = GetFileSize(xfile->file, NULL); | ||
909 | if (filesize == INVALID_FILE_SIZE) | ||
910 | { | ||
911 | return get_errno () | ||
912 | } | ||
913 | *size = filesize; | ||
914 | } | ||
915 | # endif | ||
916 | #else | ||
917 | struct stat sbuf; | ||
918 | if (fstat (XFNO (xfile), & sbuf) < 0) | ||
919 | { | ||
920 | ret = get_errno (); | ||
921 | return ret; | ||
922 | } | ||
923 | |||
924 | if (! S_ISREG (sbuf.st_mode)) | ||
925 | { | ||
926 | return ESPIPE; | ||
927 | } | ||
928 | (*size) = sbuf.st_size; | ||
929 | #endif | ||
930 | return ret; | ||
931 | } | ||
932 | |||
933 | static int | ||
934 | main_file_exists (main_file *xfile) | ||
935 | { | ||
936 | struct stat sbuf; | ||
937 | return stat (xfile->filename, & sbuf) == 0 && S_ISREG (sbuf.st_mode); | ||
938 | } | ||
939 | |||
940 | #if (XD3_POSIX || EXTERNAL_COMPRESSION) | ||
941 | /* POSIX-generic code takes a function pointer to read() or write(). | ||
942 | * This calls the function repeatedly until the buffer is full or EOF. | ||
943 | * The NREAD parameter is not set for write, NULL is passed. Return | ||
944 | * is signed, < 0 indicate errors, otherwise byte count. */ | ||
945 | typedef int (xd3_posix_func) (int fd, uint8_t *buf, usize_t size); | ||
946 | |||
947 | static int | ||
948 | xd3_posix_io (int fd, uint8_t *buf, usize_t size, | ||
949 | xd3_posix_func *func, usize_t *nread) | ||
950 | { | ||
951 | int ret; | ||
952 | usize_t nproc = 0; | ||
953 | |||
954 | while (nproc < size) | ||
955 | { | ||
956 | int result = (*func) (fd, buf + nproc, size - nproc); | ||
957 | |||
958 | if (result < 0) | ||
959 | { | ||
960 | ret = get_errno (); | ||
961 | if (ret != EAGAIN && ret != EINTR) | ||
962 | { | ||
963 | return ret; | ||
964 | } | ||
965 | continue; | ||
966 | } | ||
967 | |||
968 | if (nread != NULL && result == 0) { break; } | ||
969 | |||
970 | nproc += result; | ||
971 | } | ||
972 | if (nread != NULL) { (*nread) = nproc; } | ||
973 | return 0; | ||
974 | } | ||
975 | #endif | ||
976 | |||
977 | #if XD3_WIN32 | ||
978 | static int | ||
979 | xd3_win32_io (HANDLE file, uint8_t *buf, usize_t size, | ||
980 | int is_read, usize_t *nread) | ||
981 | { | ||
982 | int ret = 0; | ||
983 | usize_t nproc = 0; | ||
984 | |||
985 | while (nproc < size) | ||
986 | { | ||
987 | DWORD nproc2; | ||
988 | if ((is_read ? | ||
989 | ReadFile (file, buf + nproc, size - nproc, &nproc2, NULL) : | ||
990 | WriteFile (file, buf + nproc, size - nproc, &nproc2, NULL)) == 0) | ||
991 | { | ||
992 | ret = get_errno(); | ||
993 | if (ret != ERROR_HANDLE_EOF && ret != ERROR_BROKEN_PIPE) | ||
994 | { | ||
995 | return ret; | ||
996 | } | ||
997 | /* By falling through here, we'll break this loop in the | ||
998 | * read case in case of eof or broken pipe. */ | ||
999 | } | ||
1000 | |||
1001 | nproc += (usize_t) nproc2; | ||
1002 | |||
1003 | if (nread != NULL && nproc2 == 0) { break; } | ||
1004 | } | ||
1005 | if (nread != NULL) { (*nread) = nproc; } | ||
1006 | return 0; | ||
1007 | } | 61 | } |
1008 | #endif | ||
1009 | 62 | ||
1010 | /* POSIX is unbuffered, while STDIO is buffered. main_file_read() | 63 | static void main_lru_cleanup() |
1011 | * should always be called on blocks. */ | ||
1012 | static int | ||
1013 | main_file_read (main_file *ifile, | ||
1014 | uint8_t *buf, | ||
1015 | usize_t size, | ||
1016 | usize_t *nread, | ||
1017 | const char *msg) | ||
1018 | { | 64 | { |
1019 | int ret = 0; | 65 | int i; |
1020 | 66 | /* TODO(jmacd): HERE YOU ARE | |
1021 | #if XD3_STDIO | 67 | * Remove this loop, free only lru[0].blk. |
1022 | usize_t result; | 68 | */ |
1023 | 69 | for (i = 0; lru && i < lru_size; i += 1) | |
1024 | result = fread (buf, 1, size, ifile->file); | ||
1025 | |||
1026 | if (result < size && ferror (ifile->file)) | ||
1027 | { | ||
1028 | ret = get_errno (); | ||
1029 | } | ||
1030 | else | ||
1031 | { | 70 | { |
1032 | *nread = result; | 71 | main_free (lru[i].blk); |
1033 | } | 72 | } |
1034 | 73 | ||
1035 | #elif XD3_POSIX | 74 | main_free (lru); |
1036 | ret = xd3_posix_io (ifile->file, buf, size, (xd3_posix_func*) &read, nread); | 75 | lru = NULL; |
1037 | #elif XD3_WIN32 | ||
1038 | ret = xd3_win32_io (ifile->file, buf, size, 1 /* is_read */, nread); | ||
1039 | #endif | ||
1040 | |||
1041 | if (ret) | ||
1042 | { | ||
1043 | XPR(NT "%s: %s: %s\n", msg, ifile->filename, xd3_mainerror (ret)); | ||
1044 | } | ||
1045 | else | ||
1046 | { | ||
1047 | if (option_verbose > 4) { XPR(NT "read %s: %u bytes\n", | ||
1048 | ifile->filename, (*nread)); } | ||
1049 | ifile->nread += (*nread); | ||
1050 | } | ||
1051 | 76 | ||
1052 | return ret; | 77 | lru_hits = 0; |
78 | lru_misses = 0; | ||
79 | lru_filled = 0; | ||
1053 | } | 80 | } |
1054 | 81 | ||
82 | /* This is called at different times for encoding and decoding. The | ||
83 | * encoder calls it immediately, the decoder delays until the | ||
84 | * application header is received. */ | ||
1055 | static int | 85 | static int |
1056 | main_file_write (main_file *ofile, uint8_t *buf, usize_t size, const char *msg) | 86 | main_set_source (xd3_stream *stream, xd3_cmd cmd, |
87 | main_file *sfile, xd3_source *source) | ||
1057 | { | 88 | { |
1058 | int ret = 0; | 89 | int ret = 0; |
90 | usize_t i; | ||
91 | usize_t blksize; | ||
92 | xoff_t source_size = 0; | ||
93 | main_blklru block0; | ||
1059 | 94 | ||
1060 | #if XD3_STDIO | 95 | XD3_ASSERT (lru == NULL); |
1061 | usize_t result; | 96 | XD3_ASSERT (stream->src == NULL); |
1062 | 97 | XD3_ASSERT (option_srcwinsz >= XD3_MINSRCWINSZ); | |
1063 | result = fwrite (buf, 1, size, ofile->file); | ||
1064 | |||
1065 | if (result != size) { ret = get_errno (); } | ||
1066 | |||
1067 | #elif XD3_POSIX | ||
1068 | ret = xd3_posix_io (ofile->file, buf, size, (xd3_posix_func*) &write, NULL); | ||
1069 | |||
1070 | #elif XD3_WIN32 | ||
1071 | ret = xd3_win32_io (ofile->file, buf, size, 0, NULL); | ||
1072 | 98 | ||
1073 | #endif | 99 | main_blklru_list_init (& lru_list); |
100 | main_blklru_list_init (& lru_free); | ||
1074 | 101 | ||
1075 | if (ret) | 102 | if (allow_fake_source) |
1076 | { | 103 | { |
1077 | XPR(NT "%s: %s: %s\n", msg, ofile->filename, xd3_mainerror (ret)); | 104 | sfile->mode = XO_READ; |
105 | sfile->realname = sfile->filename; | ||
106 | sfile->nread = 0; | ||
1078 | } | 107 | } |
1079 | else | 108 | else |
1080 | { | 109 | { |
1081 | if (option_verbose > 4) { XPR(NT "write %s: %u bytes\n", | 110 | if ((ret = main_file_open (sfile, sfile->filename, XO_READ))) |
1082 | ofile->filename, size); } | ||
1083 | ofile->nwrite += size; | ||
1084 | } | ||
1085 | |||
1086 | return ret; | ||
1087 | } | ||
1088 | |||
1089 | static int | ||
1090 | main_file_seek (main_file *xfile, xoff_t pos) | ||
1091 | { | ||
1092 | int ret = 0; | ||
1093 | |||
1094 | #if XD3_STDIO | ||
1095 | if (fseek (xfile->file, pos, SEEK_SET) != 0) { ret = get_errno (); } | ||
1096 | |||
1097 | #elif XD3_POSIX | ||
1098 | if ((xoff_t) lseek (xfile->file, pos, SEEK_SET) != pos) | ||
1099 | { ret = get_errno (); } | ||
1100 | |||
1101 | #elif XD3_WIN32 | ||
1102 | # if (_WIN32_WINNT >= 0x0500) | ||
1103 | LARGE_INTEGER move, out; | ||
1104 | move.QuadPart = pos; | ||
1105 | if (SetFilePointerEx(xfile->file, move, &out, FILE_BEGIN) == 0) | ||
1106 | { | ||
1107 | ret = get_errno (); | ||
1108 | } | ||
1109 | # else | ||
1110 | if (SetFilePointer(xfile->file, (LONG)pos, NULL, FILE_BEGIN) == | ||
1111 | INVALID_SET_FILE_POINTER) | ||
1112 | { | ||
1113 | ret = get_errno (); | ||
1114 | } | ||
1115 | # endif | ||
1116 | #endif | ||
1117 | |||
1118 | return ret; | ||
1119 | } | ||
1120 | |||
1121 | /* This function simply writes the stream output buffer, if there is | ||
1122 | * any, for encode, decode and recode commands. (The VCDIFF tools use | ||
1123 | * main_print_func()). */ | ||
1124 | static int | ||
1125 | main_write_output (xd3_stream* stream, main_file *ofile) | ||
1126 | { | ||
1127 | int ret; | ||
1128 | |||
1129 | if (option_no_output) | ||
1130 | { | ||
1131 | return 0; | ||
1132 | } | ||
1133 | |||
1134 | if (stream->avail_out > 0 && | ||
1135 | (ret = main_file_write (ofile, stream->next_out, | ||
1136 | stream->avail_out, "write failed"))) | ||
1137 | { | ||
1138 | return ret; | ||
1139 | } | ||
1140 | |||
1141 | return 0; | ||
1142 | } | ||
1143 | |||
1144 | static int | ||
1145 | main_set_secondary_flags (xd3_config *config) | ||
1146 | { | ||
1147 | int ret; | ||
1148 | if (option_use_secondary) | ||
1149 | { | ||
1150 | /* The default secondary compressor is DJW, if it's compiled. */ | ||
1151 | if (option_secondary == NULL) | ||
1152 | { | ||
1153 | if (SECONDARY_DJW) | ||
1154 | { | ||
1155 | config->flags |= XD3_SEC_DJW; | ||
1156 | } | ||
1157 | } | ||
1158 | else | ||
1159 | { | 111 | { |
1160 | if (strcmp (option_secondary, "fgk") == 0 && SECONDARY_FGK) | ||
1161 | { | ||
1162 | config->flags |= XD3_SEC_FGK; | ||
1163 | } | ||
1164 | else if (strncmp (option_secondary, "djw", 3) == 0 && SECONDARY_DJW) | ||
1165 | { | ||
1166 | usize_t level = XD3_DEFAULT_SECONDARY_LEVEL; | ||
1167 | |||
1168 | config->flags |= XD3_SEC_DJW; | ||
1169 | |||
1170 | if (strlen (option_secondary) > 3 && | ||
1171 | (ret = main_atou (option_secondary + 3, | ||
1172 | &level, | ||
1173 | 0, 9, 'S')) != 0 && | ||
1174 | !option_quiet) | ||
1175 | { | ||
1176 | return XD3_INVALID; | ||
1177 | } | ||
1178 | |||
1179 | /* XD3_SEC_NOXXXX flags disable secondary compression on | ||
1180 | * a per-section basis. For djw, ngroups=1 indicates | ||
1181 | * minimum work, ngroups=0 uses default settings, which | ||
1182 | * is > 1 groups by default. */ | ||
1183 | if (level < 1) { config->flags |= XD3_SEC_NODATA; } | ||
1184 | if (level < 7) { config->sec_data.ngroups = 1; } | ||
1185 | else { config->sec_data.ngroups = 0; } | ||
1186 | |||
1187 | if (level < 3) { config->flags |= XD3_SEC_NOINST; } | ||
1188 | if (level < 8) { config->sec_inst.ngroups = 1; } | ||
1189 | else { config->sec_inst.ngroups = 0; } | ||
1190 | |||
1191 | if (level < 5) { config->flags |= XD3_SEC_NOADDR; } | ||
1192 | if (level < 9) { config->sec_addr.ngroups = 1; } | ||
1193 | else { config->sec_addr.ngroups = 0; } | ||
1194 | } | ||
1195 | else if (strcmp (option_secondary, "none") == 0 && SECONDARY_DJW) | ||
1196 | { | ||
1197 | /* No secondary */ | ||
1198 | } | ||
1199 | else | ||
1200 | { | ||
1201 | if (!option_quiet) | ||
1202 | { | ||
1203 | XPR(NT "unrecognized secondary compressor type: %s\n", | ||
1204 | option_secondary); | ||
1205 | return XD3_INVALID; | ||
1206 | } | ||
1207 | } | ||
1208 | } | ||
1209 | } | ||
1210 | |||
1211 | return 0; | ||
1212 | } | ||
1213 | |||
1214 | /****************************************************************** | ||
1215 | VCDIFF TOOLS | ||
1216 | *****************************************************************/ | ||
1217 | |||
1218 | #if VCDIFF_TOOLS | ||
1219 | #include "xdelta3-merge.h" | ||
1220 | |||
1221 | /* According to the internet, Windows vsnprintf() differs from most | ||
1222 | * Unix implementations regarding the terminating 0 when the boundary | ||
1223 | * condition is met. It doesn't matter here, we don't rely on the | ||
1224 | * trailing 0. Besides, both Windows and DJGPP vsnprintf return -1 | ||
1225 | * upon truncation, which isn't C99 compliant. To overcome this, | ||
1226 | * recent MinGW runtimes provided their own vsnprintf (notice the | ||
1227 | * absence of the '_' prefix) but they were initially buggy. So, | ||
1228 | * always use the native '_'-prefixed version with Win32. */ | ||
1229 | #include <stdarg.h> | ||
1230 | #ifdef _WIN32 | ||
1231 | #define vsnprintf_func _vsnprintf | ||
1232 | #else | ||
1233 | #define vsnprintf_func vsnprintf | ||
1234 | #endif | ||
1235 | |||
1236 | /* Prior to SVN 303 this function was only defined in DJGPP and WIN32 | ||
1237 | * environments and other platforms would use the builtin snprintf() | ||
1238 | * with an arrangement of macros below. In OS X 10.6, Apply made | ||
1239 | * snprintf() a macro, which defeated those macros (since snprintf | ||
1240 | * would be evaluated before its argument macros were expanded, | ||
1241 | * therefore always define xsnprintf_func. */ | ||
1242 | #undef PRINTF_ATTRIBUTE | ||
1243 | #ifdef __GNUC__ | ||
1244 | /* Let's just assume no one uses gcc 2.x! */ | ||
1245 | #define PRINTF_ATTRIBUTE(x,y) __attribute__ ((__format__ (__printf__, x, y))) | ||
1246 | #else | ||
1247 | #define PRINTF_ATTRIBUTE(x,y) | ||
1248 | #endif | ||
1249 | |||
1250 | static int | ||
1251 | xsnprintf_func (char *str, int n, const char *fmt, ...) | ||
1252 | PRINTF_ATTRIBUTE(3,4); | ||
1253 | |||
1254 | int | ||
1255 | xsnprintf_func (char *str, int n, const char *fmt, ...) | ||
1256 | { | ||
1257 | va_list a; | ||
1258 | int ret; | ||
1259 | va_start (a, fmt); | ||
1260 | ret = vsnprintf_func (str, n, fmt, a); | ||
1261 | va_end (a); | ||
1262 | if (ret < 0) | ||
1263 | { | ||
1264 | ret = n; | ||
1265 | } | ||
1266 | return ret; | ||
1267 | } | ||
1268 | |||
1269 | /* The following macros let VCDIFF print using main_file_write(), | ||
1270 | * for example: | ||
1271 | * | ||
1272 | * VC(UT "trying to be portable: %d\n", x)VE; | ||
1273 | */ | ||
1274 | #define SNPRINTF_BUFSIZE 1024 | ||
1275 | #define VC do { if (((ret = xsnprintf_func | ||
1276 | #define UT (char*)xfile->snprintf_buf, SNPRINTF_BUFSIZE, | ||
1277 | #define VE ) >= SNPRINTF_BUFSIZE \ | ||
1278 | && (ret = main_print_overflow(ret)) != 0) \ | ||
1279 | || (ret = main_file_write(xfile, xfile->snprintf_buf, \ | ||
1280 | (usize_t)ret, "print")) != 0) \ | ||
1281 | { return ret; } } while (0) | ||
1282 | |||
1283 | static int | ||
1284 | main_print_overflow (int x) | ||
1285 | { | ||
1286 | XPR(NT "internal print buffer overflow: %d bytes\n", x); | ||
1287 | return XD3_INTERNAL; | ||
1288 | } | ||
1289 | |||
1290 | /* This function prints a single VCDIFF window. */ | ||
1291 | static int | ||
1292 | main_print_window (xd3_stream* stream, main_file *xfile) | ||
1293 | { | ||
1294 | int ret; | ||
1295 | usize_t size = 0; | ||
1296 | |||
1297 | VC(UT " Offset Code Type1 Size1 @Addr1 + Type2 Size2 @Addr2\n")VE; | ||
1298 | |||
1299 | while (stream->inst_sect.buf < stream->inst_sect.buf_max) | ||
1300 | { | ||
1301 | usize_t code = stream->inst_sect.buf[0]; | ||
1302 | const uint8_t *addr_before = stream->addr_sect.buf; | ||
1303 | const uint8_t *inst_before = stream->inst_sect.buf; | ||
1304 | usize_t addr_bytes; | ||
1305 | usize_t inst_bytes; | ||
1306 | usize_t size_before = size; | ||
1307 | |||
1308 | if ((ret = xd3_decode_instruction (stream))) | ||
1309 | { | ||
1310 | XPR(NT "instruction decode error at %"Q"u: %s\n", | ||
1311 | stream->dec_winstart + size, stream->msg); | ||
1312 | return ret; | 112 | return ret; |
1313 | } | 113 | } |
1314 | 114 | ||
1315 | addr_bytes = (usize_t)(stream->addr_sect.buf - addr_before); | 115 | /* Allow non-seekable sources from the start. If the file |
1316 | inst_bytes = (usize_t)(stream->inst_sect.buf - inst_before); | 116 | * turns out to be externally compressed, size_known may change. */ |
1317 | 117 | sfile->size_known = (main_file_stat (sfile, &source_size) == 0); | |
1318 | VC(UT " %06"Q"u %03u %s %6u", stream->dec_winstart + size, | 118 | } |
1319 | option_print_cpymode ? code : 0, | 119 | |
1320 | xd3_rtype_to_string ((xd3_rtype) stream->dec_current1.type, | 120 | /* The API requires power-of-two blocksize, */ |
1321 | option_print_cpymode), | 121 | blksize = xd3_pow2_roundup (max (option_srcwinsz / MAX_LRU_SIZE, |
1322 | (usize_t) stream->dec_current1.size)VE; | 122 | XD3_MINSRCWINSZ)); |
1323 | 123 | ||
1324 | if (stream->dec_current1.type != XD3_NOOP) | 124 | /* TODO(jmacd): The organization of this code and the implementation |
1325 | { | 125 | * of the LRU cache could be improved. This is too convoluted, the |
1326 | if (stream->dec_current1.type >= XD3_CPY) | 126 | * code uses main_getblk_func() to read the first block, which may |
1327 | { | 127 | * trigger external-decompression, in large part so that the |
1328 | if (stream->dec_current1.addr >= stream->dec_cpylen) | 128 | * verbose-printing and counters maintained by getblk_func are |
1329 | { | 129 | * consistent. There used to be additional optimizations going on |
1330 | VC(UT " T@%-6u", | 130 | * here: (1) if the source size is known we would like to lower |
1331 | stream->dec_current1.addr - stream->dec_cpylen)VE; | 131 | * option_srcwinsz, if possible, (2) avoid allocating more memory |
1332 | } | 132 | * than needed, and (3) also want to use a single block when |
1333 | else | 133 | * source_size < option_srcwinsz, this because compression is better |
1334 | { | 134 | * for larger blocks (especially due to the blockwise-reverse |
1335 | VC(UT " S@%-6"Q"u", | 135 | * insertion of checksums). |
1336 | stream->dec_cpyoff + stream->dec_current1.addr)VE; | 136 | * |
1337 | } | 137 | * (3) is no longer implemented. (2) is only implemented for files |
1338 | } | 138 | * that are shorter than the default blocksize, and (1) is not |
1339 | else | 139 | * implemented. These optimizations are not taken because the code |
1340 | { | 140 | * is already too complicated. |
1341 | VC(UT " ")VE; | 141 | * |
1342 | } | 142 | * The ideal solution may be to allocate a block of memory equal to |
1343 | 143 | * half of the option_srcwinsz. Read as much data as possible into | |
1344 | size += stream->dec_current1.size; | 144 | * that block. If the entire file fits, pass one block to the |
1345 | } | 145 | * library for best compression. If not, copy the 50% block into |
1346 | 146 | * smaller (option_srcwinsz / MAX_LRU_SIZE) blocks and proceed. Too | |
1347 | if (stream->dec_current2.type != XD3_NOOP) | 147 | * much effort for too little payback. */ |
1348 | { | 148 | |
1349 | VC(UT " %s %6u", | 149 | memset (&block0, 0, sizeof (block0)); |
1350 | xd3_rtype_to_string ((xd3_rtype) stream->dec_current2.type, | 150 | block0.blkno = (xoff_t) -1; |
1351 | option_print_cpymode), | 151 | |
1352 | (usize_t)stream->dec_current2.size)VE; | 152 | /* TODO(jmacd): HERE YOU ARE |
1353 | 153 | * Allocate only one block of option_srcwinsz. | |
1354 | if (stream->dec_current2.type >= XD3_CPY) | 154 | */ |
1355 | { | 155 | /* Allocate the first block. Even if the size is known at this |
1356 | if (stream->dec_current2.addr >= stream->dec_cpylen) | 156 | * point, we do not lower it because decompression may happen. */ |
1357 | { | 157 | if ((block0.blk = (uint8_t*) main_malloc (blksize)) == NULL) |
1358 | VC(UT " T@%-6u", | ||
1359 | stream->dec_current2.addr - stream->dec_cpylen)VE; | ||
1360 | } | ||
1361 | else | ||
1362 | { | ||
1363 | VC(UT " S@%-6"Q"u", | ||
1364 | stream->dec_cpyoff + stream->dec_current2.addr)VE; | ||
1365 | } | ||
1366 | } | ||
1367 | |||
1368 | size += stream->dec_current2.size; | ||
1369 | } | ||
1370 | |||
1371 | VC(UT "\n")VE; | ||
1372 | |||
1373 | if (option_verbose && | ||
1374 | addr_bytes + inst_bytes >= (size - size_before) && | ||
1375 | (stream->dec_current1.type >= XD3_CPY || | ||
1376 | stream->dec_current2.type >= XD3_CPY)) | ||
1377 | { | ||
1378 | VC(UT " %06"Q"u (inefficiency) %u encoded as %u bytes\n", | ||
1379 | stream->dec_winstart + size_before, | ||
1380 | size - size_before, | ||
1381 | addr_bytes + inst_bytes)VE; | ||
1382 | } | ||
1383 | } | ||
1384 | |||
1385 | if (stream->dec_tgtlen != size && (stream->flags & XD3_SKIP_WINDOW) == 0) | ||
1386 | { | ||
1387 | XPR(NT "target window size inconsistency"); | ||
1388 | return XD3_INTERNAL; | ||
1389 | } | ||
1390 | |||
1391 | if (stream->dec_position != stream->dec_maxpos) | ||
1392 | { | ||
1393 | XPR(NT "target window position inconsistency"); | ||
1394 | return XD3_INTERNAL; | ||
1395 | } | ||
1396 | |||
1397 | if (stream->addr_sect.buf != stream->addr_sect.buf_max) | ||
1398 | { | 158 | { |
1399 | XPR(NT "address section inconsistency"); | 159 | ret = ENOMEM; |
1400 | return XD3_INTERNAL; | 160 | return ret; |
1401 | } | 161 | } |
1402 | 162 | ||
1403 | return 0; | 163 | source->blksize = blksize; |
1404 | } | 164 | source->name = sfile->filename; |
165 | source->ioh = sfile; | ||
166 | source->curblkno = (xoff_t) -1; | ||
167 | source->curblk = NULL; | ||
168 | |||
169 | /* We have to read the first block into the cache now, because | ||
170 | * size_known can still change (due to secondary decompression). | ||
171 | * Calls main_secondary_decompress_check() via | ||
172 | * main_read_primary_input(). */ | ||
173 | lru_size = 1; | ||
174 | lru = &block0; | ||
175 | XD3_ASSERT (main_blklru_list_empty (& lru_free)); | ||
176 | XD3_ASSERT (main_blklru_list_empty (& lru_list)); | ||
177 | main_blklru_list_push_back (& lru_free, & lru[0]); | ||
178 | /* This call is partly so that the diagnostics printed by | ||
179 | * this function appear to happen normally for the first read, | ||
180 | * which is special. */ | ||
181 | ret = main_getblk_func (stream, source, 0); | ||
182 | main_blklru_list_remove (& lru[0]); | ||
183 | XD3_ASSERT (main_blklru_list_empty (& lru_free)); | ||
184 | XD3_ASSERT (main_blklru_list_empty (& lru_list)); | ||
185 | lru = NULL; | ||
186 | |||
187 | if (ret != 0) | ||
188 | { | ||
189 | main_free (block0.blk); | ||
190 | |||
191 | XPR(NT "error reading source: %s: %s\n", | ||
192 | sfile->filename, | ||
193 | xd3_mainerror (ret)); | ||
1405 | 194 | ||
1406 | static int | 195 | return ret; |
1407 | main_print_vcdiff_file (main_file *xfile, main_file *file, const char *type) | ||
1408 | { | ||
1409 | int ret; /* Used by above macros */ | ||
1410 | if (file->filename) | ||
1411 | { | ||
1412 | VC(UT "XDELTA filename (%s): %s\n", type, | ||
1413 | file->filename)VE; | ||
1414 | } | ||
1415 | if (file->compressor) | ||
1416 | { | ||
1417 | VC(UT "XDELTA ext comp (%s): %s\n", type, | ||
1418 | file->compressor->recomp_cmdname)VE; | ||
1419 | } | 196 | } |
1420 | return 0; | ||
1421 | } | ||
1422 | 197 | ||
1423 | /* This function prints a VCDIFF input, mainly for debugging purposes. */ | 198 | source->onblk = block0.size; |
1424 | static int | ||
1425 | main_print_func (xd3_stream* stream, main_file *xfile) | ||
1426 | { | ||
1427 | int ret; | ||
1428 | 199 | ||
1429 | if (option_no_output) | 200 | /* If the file is smaller than a block, size is known. */ |
201 | if (!sfile->size_known && source->onblk < blksize) | ||
1430 | { | 202 | { |
1431 | return 0; | 203 | source_size = source->onblk; |
204 | sfile->size_known = 1; | ||
1432 | } | 205 | } |
1433 | 206 | ||
1434 | if (xfile->snprintf_buf == NULL) | 207 | /* We update lru_size accordingly, and modify option_srcwinsz, which |
208 | * will be passed via xd3_config. */ | ||
209 | if (sfile->size_known && source_size <= option_srcwinsz) | ||
1435 | { | 210 | { |
1436 | if ((xfile->snprintf_buf = | 211 | lru_size = (usize_t) ((source_size + blksize - 1) / blksize); |
1437 | (uint8_t*)main_malloc(SNPRINTF_BUFSIZE)) == NULL) | ||
1438 | { | ||
1439 | return ENOMEM; | ||
1440 | } | ||
1441 | } | ||
1442 | |||
1443 | if (stream->dec_winstart == 0) | ||
1444 | { | ||
1445 | VC(UT "VCDIFF version: 0\n")VE; | ||
1446 | VC(UT "VCDIFF header size: %d\n", | ||
1447 | stream->dec_hdrsize)VE; | ||
1448 | VC(UT "VCDIFF header indicator: ")VE; | ||
1449 | if ((stream->dec_hdr_ind & VCD_SECONDARY) != 0) | ||
1450 | VC(UT "VCD_SECONDARY ")VE; | ||
1451 | if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) | ||
1452 | VC(UT "VCD_CODETABLE ")VE; | ||
1453 | if ((stream->dec_hdr_ind & VCD_APPHEADER) != 0) | ||
1454 | VC(UT "VCD_APPHEADER ")VE; | ||
1455 | if (stream->dec_hdr_ind == 0) | ||
1456 | VC(UT "none")VE; | ||
1457 | VC(UT "\n")VE; | ||
1458 | |||
1459 | IF_SEC(VC(UT "VCDIFF secondary compressor: %s\n", | ||
1460 | stream->sec_type ? stream->sec_type->name : "none")VE); | ||
1461 | IF_NSEC(VC(UT "VCDIFF secondary compressor: unsupported\n")VE); | ||
1462 | |||
1463 | if (stream->dec_hdr_ind & VCD_APPHEADER) | ||
1464 | { | ||
1465 | uint8_t *apphead; | ||
1466 | usize_t appheadsz; | ||
1467 | ret = xd3_get_appheader (stream, & apphead, & appheadsz); | ||
1468 | |||
1469 | if (ret == 0 && appheadsz > 0) | ||
1470 | { | ||
1471 | int sq = option_quiet; | ||
1472 | main_file i, o, s; | ||
1473 | XD3_ASSERT (apphead != NULL); | ||
1474 | VC(UT "VCDIFF application header: ")VE; | ||
1475 | if ((ret = main_file_write (xfile, apphead, | ||
1476 | appheadsz, "print")) != 0) | ||
1477 | { return ret; } | ||
1478 | VC(UT "\n")VE; | ||
1479 | |||
1480 | main_file_init (& i); | ||
1481 | main_file_init (& o); | ||
1482 | main_file_init (& s); | ||
1483 | option_quiet = 1; | ||
1484 | main_get_appheader (stream, &i, & o, & s); | ||
1485 | option_quiet = sq; | ||
1486 | if ((ret = main_print_vcdiff_file (xfile, & o, "output"))) | ||
1487 | { return ret; } | ||
1488 | if ((ret = main_print_vcdiff_file (xfile, & s, "source"))) | ||
1489 | { return ret; } | ||
1490 | main_file_cleanup (& i); | ||
1491 | main_file_cleanup (& o); | ||
1492 | main_file_cleanup (& s); | ||
1493 | } | ||
1494 | } | ||
1495 | } | 212 | } |
1496 | else | 213 | else |
1497 | { | 214 | { |
1498 | VC(UT "\n")VE; | 215 | lru_size = (option_srcwinsz + blksize - 1) / blksize; |
1499 | } | ||
1500 | |||
1501 | VC(UT "VCDIFF window number: %"Q"u\n", stream->current_window)VE; | ||
1502 | VC(UT "VCDIFF window indicator: ")VE; | ||
1503 | if ((stream->dec_win_ind & VCD_SOURCE) != 0) VC(UT "VCD_SOURCE ")VE; | ||
1504 | if ((stream->dec_win_ind & VCD_TARGET) != 0) VC(UT "VCD_TARGET ")VE; | ||
1505 | if ((stream->dec_win_ind & VCD_ADLER32) != 0) VC(UT "VCD_ADLER32 ")VE; | ||
1506 | if (stream->dec_win_ind == 0) VC(UT "none")VE; | ||
1507 | VC(UT "\n")VE; | ||
1508 | |||
1509 | if ((stream->dec_win_ind & VCD_ADLER32) != 0) | ||
1510 | { | ||
1511 | VC(UT "VCDIFF adler32 checksum: %08X\n", | ||
1512 | (usize_t)stream->dec_adler32)VE; | ||
1513 | } | ||
1514 | |||
1515 | if (stream->dec_del_ind != 0) | ||
1516 | { | ||
1517 | VC(UT "VCDIFF delta indicator: ")VE; | ||
1518 | if ((stream->dec_del_ind & VCD_DATACOMP) != 0) VC(UT "VCD_DATACOMP ")VE; | ||
1519 | if ((stream->dec_del_ind & VCD_INSTCOMP) != 0) VC(UT "VCD_INSTCOMP ")VE; | ||
1520 | if ((stream->dec_del_ind & VCD_ADDRCOMP) != 0) VC(UT "VCD_ADDRCOMP ")VE; | ||
1521 | if (stream->dec_del_ind == 0) VC(UT "none")VE; | ||
1522 | VC(UT "\n")VE; | ||
1523 | } | ||
1524 | |||
1525 | if (stream->dec_winstart != 0) | ||
1526 | { | ||
1527 | VC(UT "VCDIFF window at offset: %"Q"u\n", stream->dec_winstart)VE; | ||
1528 | } | ||
1529 | |||
1530 | if (SRCORTGT (stream->dec_win_ind)) | ||
1531 | { | ||
1532 | VC(UT "VCDIFF copy window length: %u\n", | ||
1533 | (usize_t)stream->dec_cpylen)VE; | ||
1534 | VC(UT "VCDIFF copy window offset: %"Q"u\n", | ||
1535 | stream->dec_cpyoff)VE; | ||
1536 | } | ||
1537 | |||
1538 | VC(UT "VCDIFF delta encoding length: %u\n", | ||
1539 | (usize_t)stream->dec_enclen)VE; | ||
1540 | VC(UT "VCDIFF target window length: %u\n", | ||
1541 | (usize_t)stream->dec_tgtlen)VE; | ||
1542 | |||
1543 | VC(UT "VCDIFF data section length: %u\n", | ||
1544 | (usize_t)stream->data_sect.size)VE; | ||
1545 | VC(UT "VCDIFF inst section length: %u\n", | ||
1546 | (usize_t)stream->inst_sect.size)VE; | ||
1547 | VC(UT "VCDIFF addr section length: %u\n", | ||
1548 | (usize_t)stream->addr_sect.size)VE; | ||
1549 | |||
1550 | ret = 0; | ||
1551 | if ((stream->flags & XD3_JUST_HDR) != 0) | ||
1552 | { | ||
1553 | /* Print a header -- finished! */ | ||
1554 | ret = PRINTHDR_SPECIAL; | ||
1555 | } | ||
1556 | else if ((stream->flags & XD3_SKIP_WINDOW) == 0) | ||
1557 | { | ||
1558 | ret = main_print_window (stream, xfile); | ||
1559 | } | ||
1560 | |||
1561 | return ret; | ||
1562 | } | ||
1563 | |||
1564 | static int | ||
1565 | main_recode_copy (xd3_stream* stream, | ||
1566 | xd3_output* output, | ||
1567 | xd3_desect* input) | ||
1568 | { | ||
1569 | int ret; | ||
1570 | |||
1571 | XD3_ASSERT(output != NULL); | ||
1572 | XD3_ASSERT(output->next_page == NULL); | ||
1573 | |||
1574 | if ((ret = xd3_decode_allocate (recode_stream, | ||
1575 | input->size, | ||
1576 | &output->base, | ||
1577 | &output->avail))) | ||
1578 | { | ||
1579 | XPR(NT XD3_LIB_ERRMSG (stream, ret)); | ||
1580 | return ret; | ||
1581 | } | 216 | } |
1582 | 217 | ||
1583 | memcpy (output->base, | 218 | XD3_ASSERT (lru_size >= 1); |
1584 | /* Note: decoder advances buf, so get base of buffer with | 219 | option_srcwinsz = lru_size * blksize; |
1585 | * buf_max - size */ | ||
1586 | input->buf_max - input->size, | ||
1587 | input->size); | ||
1588 | output->next = input->size; | ||
1589 | return 0; | ||
1590 | } | ||
1591 | 220 | ||
1592 | // Re-encode one window | 221 | if ((lru = (main_blklru*) main_malloc (lru_size * sizeof (main_blklru))) |
1593 | static int | 222 | == NULL) |
1594 | main_recode_func (xd3_stream* stream, main_file *ofile) | ||
1595 | { | ||
1596 | int ret; | ||
1597 | xd3_source decode_source; | ||
1598 | |||
1599 | XD3_ASSERT(stream->dec_state == DEC_FINISH); | ||
1600 | XD3_ASSERT(recode_stream->enc_state == ENC_INIT || | ||
1601 | recode_stream->enc_state == ENC_INPUT); | ||
1602 | |||
1603 | // Copy partial decoder output to partial encoder inputs | ||
1604 | if ((ret = main_recode_copy (recode_stream, | ||
1605 | DATA_HEAD(recode_stream), | ||
1606 | &stream->data_sect)) || | ||
1607 | (ret = main_recode_copy (recode_stream, | ||
1608 | INST_HEAD(recode_stream), | ||
1609 | &stream->inst_sect)) || | ||
1610 | (ret = main_recode_copy (recode_stream, | ||
1611 | ADDR_HEAD(recode_stream), | ||
1612 | &stream->addr_sect))) | ||
1613 | { | 223 | { |
224 | ret = ENOMEM; | ||
1614 | return ret; | 225 | return ret; |
1615 | } | 226 | } |
1616 | 227 | ||
1617 | // This jumps to xd3_emit_hdr() | 228 | lru[0].blk = block0.blk; |
1618 | recode_stream->enc_state = ENC_FLUSH; | 229 | lru[0].blkno = 0; |
1619 | recode_stream->avail_in = stream->dec_tgtlen; | 230 | lru[0].size = source->onblk; |
1620 | |||
1621 | if (SRCORTGT (stream->dec_win_ind)) | ||
1622 | { | ||
1623 | recode_stream->src = & decode_source; | ||
1624 | decode_source.srclen = stream->dec_cpylen; | ||
1625 | decode_source.srcbase = stream->dec_cpyoff; | ||
1626 | } | ||
1627 | |||
1628 | if (option_use_checksum && | ||
1629 | (stream->dec_win_ind & VCD_ADLER32) != 0) | ||
1630 | { | ||
1631 | recode_stream->flags |= XD3_ADLER32_RECODE; | ||
1632 | recode_stream->recode_adler32 = stream->dec_adler32; | ||
1633 | } | ||
1634 | 231 | ||
1635 | if (option_use_appheader != 0 && | 232 | if (! sfile->size_known) |
1636 | option_appheader != NULL) | ||
1637 | { | 233 | { |
1638 | xd3_set_appheader (recode_stream, option_appheader, | 234 | do_src_fifo = 1; |
1639 | (usize_t) strlen ((char*) option_appheader)); | ||
1640 | } | 235 | } |
1641 | else if (option_use_appheader != 0 && | 236 | else if (! do_src_fifo) |
1642 | option_appheader == NULL) | ||
1643 | { | 237 | { |
1644 | if (stream->dec_appheader != NULL) | 238 | main_blklru_list_push_back (& lru_list, & lru[0]); |
1645 | { | ||
1646 | xd3_set_appheader (recode_stream, | ||
1647 | stream->dec_appheader, stream->dec_appheadsz); | ||
1648 | } | ||
1649 | } | 239 | } |
1650 | 240 | ||
1651 | // Output loop | 241 | for (i = 1; i < lru_size; i += 1) |
1652 | for (;;) | ||
1653 | { | 242 | { |
1654 | switch((ret = xd3_encode_input (recode_stream))) | 243 | lru[i].blkno = (xoff_t) -1; |
1655 | { | ||
1656 | case XD3_INPUT: { | ||
1657 | /* finished recoding one window */ | ||
1658 | stream->total_out = recode_stream->total_out; | ||
1659 | return 0; | ||
1660 | } | ||
1661 | case XD3_OUTPUT: { | ||
1662 | /* main_file_write below */ | ||
1663 | break; | ||
1664 | } | ||
1665 | case XD3_GOTHEADER: | ||
1666 | case XD3_WINSTART: | ||
1667 | case XD3_WINFINISH: { | ||
1668 | /* ignore */ | ||
1669 | continue; | ||
1670 | } | ||
1671 | case XD3_GETSRCBLK: | ||
1672 | case 0: { | ||
1673 | return XD3_INTERNAL; | ||
1674 | } | ||
1675 | default: | ||
1676 | return ret; | ||
1677 | } | ||
1678 | 244 | ||
1679 | if ((ret = main_write_output (recode_stream, ofile))) | 245 | if ((lru[i].blk = (uint8_t*) main_malloc (source->blksize)) == NULL) |
1680 | { | 246 | { |
247 | ret = ENOMEM; | ||
1681 | return ret; | 248 | return ret; |
1682 | } | 249 | } |
1683 | 250 | ||
1684 | xd3_consume_output (recode_stream); | 251 | if (! do_src_fifo) |
1685 | } | ||
1686 | } | ||
1687 | #endif /* VCDIFF_TOOLS */ | ||
1688 | |||
1689 | /******************************************************************* | ||
1690 | VCDIFF merging | ||
1691 | ******************************************************************/ | ||
1692 | |||
1693 | #if VCDIFF_TOOLS | ||
1694 | /* Modifies static state. */ | ||
1695 | static int | ||
1696 | main_init_recode_stream (void) | ||
1697 | { | ||
1698 | int ret; | ||
1699 | int stream_flags = XD3_ADLER32_NOVER | XD3_SKIP_EMIT; | ||
1700 | int recode_flags; | ||
1701 | xd3_config recode_config; | ||
1702 | |||
1703 | XD3_ASSERT (recode_stream == NULL); | ||
1704 | |||
1705 | if ((recode_stream = (xd3_stream*) main_malloc(sizeof(xd3_stream))) == NULL) | ||
1706 | { | ||
1707 | return ENOMEM; | ||
1708 | } | ||
1709 | |||
1710 | recode_flags = (stream_flags & XD3_SEC_TYPE); | ||
1711 | |||
1712 | recode_config.alloc = main_alloc; | ||
1713 | recode_config.freef = main_free1; | ||
1714 | |||
1715 | xd3_init_config(&recode_config, recode_flags); | ||
1716 | |||
1717 | if ((ret = main_set_secondary_flags (&recode_config)) || | ||
1718 | (ret = xd3_config_stream (recode_stream, &recode_config)) || | ||
1719 | (ret = xd3_encode_init_partial (recode_stream)) || | ||
1720 | (ret = xd3_whole_state_init (recode_stream))) | ||
1721 | { | ||
1722 | XPR(NT XD3_LIB_ERRMSG (recode_stream, ret)); | ||
1723 | xd3_free_stream (recode_stream); | ||
1724 | recode_stream = NULL; | ||
1725 | return ret; | ||
1726 | } | ||
1727 | |||
1728 | return 0; | ||
1729 | } | ||
1730 | |||
1731 | /* This processes the sequence of -m arguments. The final input | ||
1732 | * is processed as part of the ordinary main_input() loop. */ | ||
1733 | static int | ||
1734 | main_merge_arguments (main_merge_list* merges) | ||
1735 | { | ||
1736 | int ret = 0; | ||
1737 | int count = 0; | ||
1738 | main_merge *merge = NULL; | ||
1739 | xd3_stream merge_input; | ||
1740 | |||
1741 | if (main_merge_list_empty (merges)) | ||
1742 | { | ||
1743 | return 0; | ||
1744 | } | ||
1745 | |||
1746 | if ((ret = xd3_config_stream (& merge_input, NULL)) || | ||
1747 | (ret = xd3_whole_state_init (& merge_input))) | ||
1748 | { | ||
1749 | XPR(NT XD3_LIB_ERRMSG (& merge_input, ret)); | ||
1750 | return ret; | ||
1751 | } | ||
1752 | |||
1753 | merge = main_merge_list_front (merges); | ||
1754 | while (!main_merge_list_end (merges, merge)) | ||
1755 | { | ||
1756 | main_file mfile; | ||
1757 | main_file_init (& mfile); | ||
1758 | mfile.filename = merge->filename; | ||
1759 | mfile.flags = RD_NONEXTERNAL; | ||
1760 | |||
1761 | if ((ret = main_file_open (& mfile, merge->filename, XO_READ))) | ||
1762 | { | ||
1763 | goto error; | ||
1764 | } | ||
1765 | |||
1766 | ret = main_input (CMD_MERGE_ARG, & mfile, NULL, NULL); | ||
1767 | |||
1768 | if (ret == 0) | ||
1769 | { | 252 | { |
1770 | if (count++ == 0) | 253 | main_blklru_list_push_back (& lru_free, & lru[i]); |
1771 | { | ||
1772 | /* The first merge source is the next merge input. */ | ||
1773 | xd3_swap_whole_state (& recode_stream->whole_target, | ||
1774 | & merge_input.whole_target); | ||
1775 | } | ||
1776 | else | ||
1777 | { | ||
1778 | /* Merge the recode_stream with merge_input. */ | ||
1779 | ret = xd3_merge_input_output (recode_stream, | ||
1780 | & merge_input.whole_target); | ||
1781 | |||
1782 | /* Save the next merge source in merge_input. */ | ||
1783 | xd3_swap_whole_state (& recode_stream->whole_target, | ||
1784 | & merge_input.whole_target); | ||
1785 | } | ||
1786 | } | 254 | } |
1787 | |||
1788 | main_file_cleanup (& mfile); | ||
1789 | |||
1790 | if (recode_stream != NULL) | ||
1791 | { | ||
1792 | xd3_free_stream (recode_stream); | ||
1793 | main_free (recode_stream); | ||
1794 | recode_stream = NULL; | ||
1795 | } | ||
1796 | |||
1797 | if (main_bdata != NULL) | ||
1798 | { | ||
1799 | main_free (main_bdata); | ||
1800 | main_bdata = NULL; | ||
1801 | main_bsize = 0; | ||
1802 | } | ||
1803 | |||
1804 | if (ret != 0) | ||
1805 | { | ||
1806 | goto error; | ||
1807 | } | ||
1808 | |||
1809 | merge = main_merge_list_next (merge); | ||
1810 | } | ||
1811 | |||
1812 | XD3_ASSERT (merge_stream == NULL); | ||
1813 | |||
1814 | if ((merge_stream = (xd3_stream*) main_malloc (sizeof(xd3_stream))) == NULL) | ||
1815 | { | ||
1816 | ret = ENOMEM; | ||
1817 | goto error; | ||
1818 | } | 255 | } |
1819 | 256 | ||
1820 | if ((ret = xd3_config_stream (merge_stream, NULL)) || | 257 | if (sfile->size_known) |
1821 | (ret = xd3_whole_state_init (merge_stream))) | ||
1822 | { | 258 | { |
1823 | XPR(NT XD3_LIB_ERRMSG (& merge_input, ret)); | 259 | ret = xd3_set_source_and_size (stream, source, source_size); |
1824 | goto error; | ||
1825 | } | 260 | } |
1826 | 261 | else | |
1827 | xd3_swap_whole_state (& merge_stream->whole_target, | ||
1828 | & merge_input.whole_target); | ||
1829 | ret = 0; | ||
1830 | error: | ||
1831 | xd3_free_stream (& merge_input); | ||
1832 | return ret; | ||
1833 | } | ||
1834 | |||
1835 | /* This processes each window of the final merge input. This routine | ||
1836 | * does not output, it buffers the entire delta into memory. */ | ||
1837 | static int | ||
1838 | main_merge_func (xd3_stream* stream, main_file *no_write) | ||
1839 | { | ||
1840 | int ret; | ||
1841 | |||
1842 | if ((ret = xd3_whole_append_window (stream))) | ||
1843 | { | 262 | { |
1844 | return ret; | 263 | ret = xd3_set_source (stream, source); |
1845 | } | 264 | } |
1846 | 265 | ||
1847 | return 0; | 266 | if (ret) |
1848 | } | ||
1849 | |||
1850 | |||
1851 | /* This is called after all windows have been read, as a final step in | ||
1852 | * main_input(). This is only called for the final merge step. */ | ||
1853 | static int | ||
1854 | main_merge_output (xd3_stream *stream, main_file *ofile) | ||
1855 | { | ||
1856 | int ret; | ||
1857 | usize_t inst_pos = 0; | ||
1858 | xoff_t output_pos = 0; | ||
1859 | xd3_source recode_source; | ||
1860 | usize_t window_num = 0; | ||
1861 | int at_least_once = 0; | ||
1862 | |||
1863 | /* merge_stream is set if there were arguments. this stream's input | ||
1864 | * needs to be applied to the merge_stream source. */ | ||
1865 | if ((merge_stream != NULL) && | ||
1866 | (ret = xd3_merge_input_output (stream, | ||
1867 | & merge_stream->whole_target))) | ||
1868 | { | 267 | { |
1869 | XPR(NT XD3_LIB_ERRMSG (stream, ret)); | 268 | XPR(NT XD3_LIB_ERRMSG (stream, ret)); |
1870 | return ret; | 269 | return ret; |
1871 | } | 270 | } |
1872 | 271 | ||
1873 | if (option_use_appheader != 0 && | 272 | XD3_ASSERT (stream->src == source); |
1874 | option_appheader != NULL) | 273 | XD3_ASSERT (source->blksize == blksize); |
1875 | { | ||
1876 | xd3_set_appheader (recode_stream, option_appheader, | ||
1877 | (usize_t) strlen ((char*) option_appheader)); | ||
1878 | } | ||
1879 | |||
1880 | /* Enter the ENC_INPUT state and bypass the next_in == NULL test | ||
1881 | * and (leftover) input buffering logic. */ | ||
1882 | XD3_ASSERT(recode_stream->enc_state == ENC_INIT); | ||
1883 | recode_stream->enc_state = ENC_INPUT; | ||
1884 | recode_stream->next_in = main_bdata; | ||
1885 | recode_stream->flags |= XD3_FLUSH; | ||
1886 | 274 | ||
1887 | /* This encodes the entire target. */ | 275 | if (option_verbose) |
1888 | while (inst_pos < stream->whole_target.instlen || !at_least_once) | ||
1889 | { | 276 | { |
1890 | xoff_t window_start = output_pos; | 277 | static char srcszbuf[32]; |
1891 | int window_srcset = 0; | 278 | static char srccntbuf[32]; |
1892 | xoff_t window_srcmin = 0; | 279 | static char winszbuf[32]; |
1893 | xoff_t window_srcmax = 0; | 280 | static char blkszbuf[32]; |
1894 | usize_t window_pos = 0; | 281 | static char nbufs[32]; |
1895 | usize_t window_size; | ||
1896 | |||
1897 | /* at_least_once ensures that we encode at least one window, | ||
1898 | * which handles the 0-byte case. */ | ||
1899 | at_least_once = 1; | ||
1900 | |||
1901 | XD3_ASSERT (recode_stream->enc_state == ENC_INPUT); | ||
1902 | |||
1903 | if ((ret = xd3_encode_input (recode_stream)) != XD3_WINSTART) | ||
1904 | { | ||
1905 | XPR(NT "invalid merge state: %s\n", xd3_mainerror (ret)); | ||
1906 | return XD3_INVALID; | ||
1907 | } | ||
1908 | |||
1909 | /* Window sizes must match from the input to the output, so that | ||
1910 | * target copies are in-range (and so that checksums carry | ||
1911 | * over). */ | ||
1912 | XD3_ASSERT (window_num < stream->whole_target.wininfolen); | ||
1913 | window_size = stream->whole_target.wininfo[window_num].length; | ||
1914 | 282 | ||
1915 | /* Output position should also match. */ | 283 | if (sfile->size_known) |
1916 | if (output_pos != stream->whole_target.wininfo[window_num].offset) | ||
1917 | { | 284 | { |
1918 | XPR(NT "internal merge error: offset mismatch\n"); | 285 | sprintf (srcszbuf, "source size %s [%"Q"u]", |
1919 | return XD3_INVALID; | 286 | main_format_bcnt (source_size, srccntbuf), |
287 | source_size); | ||
1920 | } | 288 | } |
1921 | 289 | else | |
1922 | if (option_use_checksum && | ||
1923 | (stream->dec_win_ind & VCD_ADLER32) != 0) | ||
1924 | { | 290 | { |
1925 | recode_stream->flags |= XD3_ADLER32_RECODE; | 291 | strcpy(srcszbuf, "source size unknown"); |
1926 | recode_stream->recode_adler32 = | ||
1927 | stream->whole_target.wininfo[window_num].adler32; | ||
1928 | } | 292 | } |
1929 | 293 | ||
1930 | window_num++; | 294 | nbufs[0] = 0; |
1931 | 295 | ||
1932 | if (main_bsize < window_size) | ||
1933 | { | ||
1934 | main_free (main_bdata); | ||
1935 | main_bdata = NULL; | ||
1936 | main_bsize = 0; | ||
1937 | if ((main_bdata = (uint8_t*) | ||
1938 | main_malloc (window_size)) == NULL) | ||
1939 | { | ||
1940 | return ENOMEM; | ||
1941 | } | ||
1942 | main_bsize = window_size; | ||
1943 | } | ||
1944 | |||
1945 | /* This encodes a single target window. */ | ||
1946 | while (window_pos < window_size && | ||
1947 | inst_pos < stream->whole_target.instlen) | ||
1948 | { | ||
1949 | xd3_winst *inst = &stream->whole_target.inst[inst_pos]; | ||
1950 | usize_t take = min(inst->size, window_size - window_pos); | ||
1951 | xoff_t addr; | ||
1952 | |||
1953 | switch (inst->type) | ||
1954 | { | ||
1955 | case XD3_RUN: | ||
1956 | if ((ret = xd3_emit_run (recode_stream, window_pos, take, | ||
1957 | &stream->whole_target.adds[inst->addr]))) | ||
1958 | { | ||
1959 | return ret; | ||
1960 | } | ||
1961 | break; | ||
1962 | |||
1963 | case XD3_ADD: | ||
1964 | /* Adds are implicit, put them into the input buffer. */ | ||
1965 | memcpy (main_bdata + window_pos, | ||
1966 | stream->whole_target.adds + inst->addr, take); | ||
1967 | break; | ||
1968 | |||
1969 | default: /* XD3_COPY + copy mode */ | ||
1970 | if (inst->mode != 0) | ||
1971 | { | ||
1972 | if (window_srcset) { | ||
1973 | window_srcmin = min(window_srcmin, inst->addr); | ||
1974 | window_srcmax = max(window_srcmax, inst->addr + take); | ||
1975 | } else { | ||
1976 | window_srcset = 1; | ||
1977 | window_srcmin = inst->addr; | ||
1978 | window_srcmax = inst->addr + take; | ||
1979 | } | ||
1980 | addr = inst->addr; | ||
1981 | } | ||
1982 | else | ||
1983 | { | ||
1984 | XD3_ASSERT (inst->addr >= window_start); | ||
1985 | addr = inst->addr - window_start; | ||
1986 | } | ||
1987 | IF_DEBUG2 (DP(RINT "[merge copy] winpos %u take %u addr %"Q"u mode %u\n", | ||
1988 | window_pos, take, addr, inst->mode)); | ||
1989 | if ((ret = xd3_found_match (recode_stream, window_pos, take, | ||
1990 | addr, inst->mode != 0))) | ||
1991 | { | ||
1992 | return ret; | ||
1993 | } | ||
1994 | break; | ||
1995 | } | ||
1996 | |||
1997 | window_pos += take; | ||
1998 | output_pos += take; | ||
1999 | |||
2000 | if (take == inst->size) | ||
2001 | { | ||
2002 | inst_pos += 1; | ||
2003 | } | ||
2004 | else | ||
2005 | { | ||
2006 | /* Modify the instruction for the next pass. */ | ||
2007 | if (inst->type != XD3_RUN) | ||
2008 | { | ||
2009 | inst->addr += take; | ||
2010 | } | ||
2011 | inst->size -= take; | ||
2012 | } | ||
2013 | } | ||
2014 | |||
2015 | xd3_avail_input (recode_stream, main_bdata, window_pos); | ||
2016 | |||
2017 | recode_stream->enc_state = ENC_INSTR; | ||
2018 | |||
2019 | if (window_srcset) { | ||
2020 | recode_stream->srcwin_decided = 1; | ||
2021 | recode_stream->src = &recode_source; | ||
2022 | recode_source.srclen = (usize_t)(window_srcmax - window_srcmin); | ||
2023 | recode_source.srcbase = window_srcmin; | ||
2024 | recode_stream->taroff = recode_source.srclen; | ||
2025 | |||
2026 | XD3_ASSERT (recode_source.srclen != 0); | ||
2027 | } else { | ||
2028 | recode_stream->srcwin_decided = 0; | ||
2029 | recode_stream->src = NULL; | ||
2030 | recode_stream->taroff = 0; | ||
2031 | } | ||
2032 | |||
2033 | for (;;) | ||
2034 | { | ||
2035 | switch ((ret = xd3_encode_input (recode_stream))) | ||
2036 | { | ||
2037 | case XD3_INPUT: { | ||
2038 | goto done_window; | ||
2039 | } | ||
2040 | case XD3_OUTPUT: { | ||
2041 | /* main_file_write below */ | ||
2042 | break; | ||
2043 | } | ||
2044 | case XD3_GOTHEADER: | ||
2045 | case XD3_WINSTART: | ||
2046 | case XD3_WINFINISH: { | ||
2047 | /* ignore */ | ||
2048 | continue; | ||
2049 | } | ||
2050 | case XD3_GETSRCBLK: | ||
2051 | case 0: { | ||
2052 | return XD3_INTERNAL; | ||
2053 | } | ||
2054 | default: | ||
2055 | return ret; | ||
2056 | } | ||
2057 | |||
2058 | if ((ret = main_write_output(recode_stream, ofile))) | ||
2059 | { | ||
2060 | return ret; | ||
2061 | } | ||
2062 | |||
2063 | xd3_consume_output (recode_stream); | ||
2064 | } | ||
2065 | done_window: | ||
2066 | (void) 0; | ||
2067 | } | ||
2068 | |||
2069 | return 0; | ||
2070 | } | ||
2071 | #endif | ||
2072 | |||
2073 | /******************************************************************* | ||
2074 | Input decompression, output recompression | ||
2075 | ******************************************************************/ | ||
2076 | |||
2077 | #if EXTERNAL_COMPRESSION | ||
2078 | /* This is tricky POSIX-specific code with lots of fork(), pipe(), | ||
2079 | * dup(), waitpid(), and exec() business. Most of this code | ||
2080 | * originated in PRCS1, which did automatic package-file | ||
2081 | * decompression. It works with both XD3_POSIX and XD3_STDIO file | ||
2082 | * disciplines. | ||
2083 | * | ||
2084 | * To automatically detect compressed inputs requires a child process | ||
2085 | * to reconstruct the input stream, which was advanced in order to | ||
2086 | * detect compression, because it may not be seekable. In other | ||
2087 | * words, the main program reads part of the input stream, and if it | ||
2088 | * detects a compressed input it then forks a pipe copier process, | ||
2089 | * which copies the first-read block out of the main-program's memory, | ||
2090 | * then streams the remaining compressed input into the | ||
2091 | * input-decompression pipe. | ||
2092 | */ | ||
2093 | |||
2094 | #include <signal.h> | ||
2095 | #include <unistd.h> | ||
2096 | #include <sys/stat.h> | ||
2097 | #include <sys/wait.h> | ||
2098 | |||
2099 | /* Remember which pipe FD is which. */ | ||
2100 | #define PIPE_READ_FD 0 | ||
2101 | #define PIPE_WRITE_FD 1 | ||
2102 | #define MAX_SUBPROCS 4 /* max(source + copier + output, | ||
2103 | source + copier + input + copier). */ | ||
2104 | static pid_t ext_subprocs[MAX_SUBPROCS]; | ||
2105 | static char* ext_tmpfile = NULL; | ||
2106 | |||
2107 | /* Like write(), applies to a fd instead of a main_file, for the pipe | ||
2108 | * copier subprocess. Does not print an error, to facilitate ignoring | ||
2109 | * trailing garbage, see main_pipe_copier(). */ | ||
2110 | static int | ||
2111 | main_pipe_write (int outfd, uint8_t *exist_buf, usize_t remain) | ||
2112 | { | ||
2113 | int ret; | ||
2114 | |||
2115 | if ((ret = xd3_posix_io (outfd, exist_buf, remain, | ||
2116 | (xd3_posix_func*) &write, NULL))) | ||
2117 | { | ||
2118 | return ret; | ||
2119 | } | ||
2120 | |||
2121 | return 0; | ||
2122 | } | ||
2123 | |||
2124 | /* A simple error-reporting waitpid interface. */ | ||
2125 | static int | ||
2126 | main_waitpid_check(pid_t pid) | ||
2127 | { | ||
2128 | int status; | ||
2129 | int ret = 0; | ||
2130 | |||
2131 | if (waitpid (pid, & status, 0) < 0) | ||
2132 | { | ||
2133 | ret = get_errno (); | ||
2134 | XPR(NT "external compression [pid %d] wait: %s\n", | ||
2135 | pid, xd3_mainerror (ret)); | ||
2136 | } | ||
2137 | else if (! WIFEXITED (status)) | ||
2138 | { | ||
2139 | ret = ECHILD; | ||
2140 | XPR(NT "external compression [pid %d] signal %d\n", | ||
2141 | pid, WIFSIGNALED (status) ? WTERMSIG (status) : WSTOPSIG (status)); | ||
2142 | } | ||
2143 | else if (WEXITSTATUS (status) != 0) | ||
2144 | { | ||
2145 | ret = ECHILD; | ||
2146 | if (option_verbose > 1) | 296 | if (option_verbose > 1) |
2147 | { | 297 | { |
2148 | /* Presumably, the error was printed by the subprocess. */ | 298 | sprintf(nbufs, " #bufs %u", lru_size); |
2149 | XPR(NT "external compression [pid %d] exit %d\n", | ||
2150 | pid, WEXITSTATUS (status)); | ||
2151 | } | ||
2152 | } | ||
2153 | |||
2154 | return ret; | ||
2155 | } | ||
2156 | |||
2157 | /* Wait for any existing child processes to check for abnormal exit. */ | ||
2158 | static int | ||
2159 | main_external_compression_finish (void) | ||
2160 | { | ||
2161 | int i; | ||
2162 | int ret; | ||
2163 | |||
2164 | for (i = 0; i < num_subprocs; i += 1) | ||
2165 | { | ||
2166 | if (! ext_subprocs[i]) { continue; } | ||
2167 | |||
2168 | if ((ret = main_waitpid_check (ext_subprocs[i]))) | ||
2169 | { | ||
2170 | return ret; | ||
2171 | } | ||
2172 | } | ||
2173 | |||
2174 | return 0; | ||
2175 | } | ||
2176 | |||
2177 | /* This runs as a forked process of main_input_decompress_setup() to | ||
2178 | * copy input to the decompression process. First, the available | ||
2179 | * input is copied out of the existing buffer, then the buffer is | ||
2180 | * reused to continue reading from the compressed input file. */ | ||
2181 | static int | ||
2182 | main_pipe_copier (uint8_t *pipe_buf, | ||
2183 | usize_t pipe_bufsize, | ||
2184 | usize_t nread, | ||
2185 | main_file *ifile, | ||
2186 | int outfd) | ||
2187 | { | ||
2188 | int ret; | ||
2189 | xoff_t garbage = 0; | ||
2190 | |||
2191 | /* Prevent SIGPIPE signals, allow EPIPE return values instead. This | ||
2192 | * is safe to comment-out, except that the -F flag will not work | ||
2193 | * properly (the parent would need to treat WTERMSIG(status) == | ||
2194 | * SIGPIPE). */ | ||
2195 | struct sigaction sa; | ||
2196 | sa.sa_handler = SIG_IGN; | ||
2197 | sigaction (SIGPIPE, &sa, NULL); | ||
2198 | |||
2199 | for (;;) | ||
2200 | { | ||
2201 | /* force_drain will be set when option_force and EPIPE cause us | ||
2202 | * to skip data. This is reset each time through the loop, so | ||
2203 | * the break condition below works. */ | ||
2204 | int force_drain = 0; | ||
2205 | if (nread > 0 && (ret = main_pipe_write (outfd, pipe_buf, nread))) | ||
2206 | { | ||
2207 | if (option_force && ret == EPIPE) | ||
2208 | { | ||
2209 | /* This causes the loop to continue reading until nread | ||
2210 | * == 0. */ | ||
2211 | garbage += nread; | ||
2212 | force_drain = 1; | ||
2213 | } | ||
2214 | else if (ret == EPIPE) | ||
2215 | { | ||
2216 | XPR(NT "external compression closed the pipe\n"); | ||
2217 | if (option_verbose) | ||
2218 | { | ||
2219 | if (!option_force2) | ||
2220 | { | ||
2221 | XPR(NT "use -F to force the subprocess\n"); | ||
2222 | } | ||
2223 | if (!option_force) | ||
2224 | { | ||
2225 | XPR(NT "use -f to force this process\n"); | ||
2226 | } | ||
2227 | } | ||
2228 | return ret; | ||
2229 | } | ||
2230 | else | ||
2231 | { | ||
2232 | XPR(NT "pipe write failed: %s\n", xd3_mainerror (ret)); | ||
2233 | return ret; | ||
2234 | } | ||
2235 | } | ||
2236 | |||
2237 | if (nread < pipe_bufsize && !force_drain) | ||
2238 | { | ||
2239 | break; | ||
2240 | } | 299 | } |
2241 | 300 | ||
2242 | if ((ret = main_file_read (ifile, pipe_buf, pipe_bufsize, | 301 | XPR(NT "source %s %s blksize %s window %s%s%s\n", |
2243 | & nread, "pipe read failed")) < 0) | 302 | sfile->filename, |
2244 | { | 303 | srcszbuf, |
2245 | return ret; | 304 | main_format_bcnt (blksize, blkszbuf), |
2246 | } | 305 | main_format_bcnt (option_srcwinsz, winszbuf), |
306 | nbufs, | ||
307 | do_src_fifo ? " (FIFO)" : ""); | ||
2247 | } | 308 | } |
2248 | 309 | ||
2249 | if (garbage != 0) | ||
2250 | { | ||
2251 | XPR(NT "trailing garbage ignored in %s (%"Q"u bytes)\n", | ||
2252 | ifile->filename, garbage); | ||
2253 | } | ||
2254 | return 0; | 310 | return 0; |
2255 | } | 311 | } |
2256 | 312 | ||
2257 | /* This function is called after we have read some amount of data from | ||
2258 | * the input file and detected a compressed input. Here we start a | ||
2259 | * decompression subprocess by forking twice. The first process runs | ||
2260 | * the decompression command, the second process copies data to the | ||
2261 | * input of the first. */ | ||
2262 | static int | 313 | static int |
2263 | main_input_decompress_setup (const main_extcomp *decomp, | 314 | main_getblk_lru (xd3_source *source, xoff_t blkno, |
2264 | main_file *ifile, | 315 | main_blklru** blrup, int *is_new) |
2265 | uint8_t *input_buf, | ||
2266 | usize_t input_bufsize, | ||
2267 | uint8_t *pipe_buf, | ||
2268 | usize_t pipe_bufsize, | ||
2269 | usize_t pipe_avail, | ||
2270 | usize_t *nread) | ||
2271 | { | 316 | { |
2272 | /* The two pipes: input and output file descriptors. */ | 317 | main_blklru *blru = NULL; |
2273 | int outpipefd[2], inpipefd[2]; | 318 | usize_t i; |
2274 | int input_fd = -1; /* The resulting input_fd (output of decompression). */ | ||
2275 | pid_t decomp_id, copier_id; /* The two subprocs. */ | ||
2276 | int ret; | ||
2277 | |||
2278 | outpipefd[0] = outpipefd[1] = -1; | ||
2279 | inpipefd[0] = inpipefd[1] = -1; | ||
2280 | |||
2281 | if (pipe (outpipefd) || pipe (inpipefd)) | ||
2282 | { | ||
2283 | XPR(NT "pipe failed: %s\n", xd3_mainerror (ret = get_errno ())); | ||
2284 | goto pipe_cleanup; | ||
2285 | } | ||
2286 | 319 | ||
2287 | if ((decomp_id = fork ()) < 0) | 320 | (*is_new) = 0; |
2288 | { | ||
2289 | XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ())); | ||
2290 | goto pipe_cleanup; | ||
2291 | } | ||
2292 | 321 | ||
2293 | /* The first child runs the decompression process: */ | 322 | if (do_src_fifo) |
2294 | if (decomp_id == 0) | ||
2295 | { | 323 | { |
2296 | if (option_verbose > 2) | 324 | /* Direct lookup assumes sequential scan w/o skipping blocks. */ |
325 | int idx = blkno % lru_size; | ||
326 | blru = & lru[idx]; | ||
327 | if (blru->blkno == blkno) | ||
2297 | { | 328 | { |
2298 | XPR(NT "external decompression pid %d\n", getpid ()); | 329 | (*blrup) = blru; |
330 | return 0; | ||
2299 | } | 331 | } |
2300 | 332 | ||
2301 | /* Setup pipes: write to the outpipe, read from the inpipe. */ | 333 | if (blru->blkno != (xoff_t)-1 && |
2302 | if (dup2 (outpipefd[PIPE_WRITE_FD], STDOUT_FILENO) < 0 || | 334 | blru->blkno != (xoff_t)(blkno - lru_size)) |
2303 | dup2 (inpipefd[PIPE_READ_FD], STDIN_FILENO) < 0 || | ||
2304 | close (outpipefd[PIPE_READ_FD]) || | ||
2305 | close (outpipefd[PIPE_WRITE_FD]) || | ||
2306 | close (inpipefd[PIPE_READ_FD]) || | ||
2307 | close (inpipefd[PIPE_WRITE_FD]) || | ||
2308 | execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, | ||
2309 | decomp->decomp_options, | ||
2310 | option_force2 ? "-f" : NULL, | ||
2311 | NULL)) | ||
2312 | { | 335 | { |
2313 | XPR(NT "child process %s failed to execute: %s\n", | 336 | return XD3_TOOFARBACK; |
2314 | decomp->decomp_cmdname, xd3_mainerror (get_errno ())); | ||
2315 | } | 337 | } |
2316 | |||
2317 | _exit (127); | ||
2318 | } | ||
2319 | |||
2320 | XD3_ASSERT(num_subprocs < MAX_SUBPROCS); | ||
2321 | ext_subprocs[num_subprocs++] = decomp_id; | ||
2322 | |||
2323 | if ((copier_id = fork ()) < 0) | ||
2324 | { | ||
2325 | XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ())); | ||
2326 | goto pipe_cleanup; | ||
2327 | } | ||
2328 | |||
2329 | /* The second child runs the copier process: */ | ||
2330 | if (copier_id == 0) | ||
2331 | { | ||
2332 | int exitval = 0; | ||
2333 | |||
2334 | if (option_verbose > 2) | ||
2335 | { | ||
2336 | XPR(NT "child pipe-copier pid %d\n", getpid ()); | ||
2337 | } | ||
2338 | |||
2339 | if (close (inpipefd[PIPE_READ_FD]) || | ||
2340 | main_pipe_copier (pipe_buf, pipe_bufsize, pipe_avail, | ||
2341 | ifile, inpipefd[PIPE_WRITE_FD]) || | ||
2342 | close (inpipefd[PIPE_WRITE_FD])) | ||
2343 | { | ||
2344 | XPR(NT "child copier process failed: %s\n", | ||
2345 | xd3_mainerror (get_errno ())); | ||
2346 | exitval = 1; | ||
2347 | } | ||
2348 | |||
2349 | _exit (exitval); | ||
2350 | } | ||
2351 | |||
2352 | XD3_ASSERT(num_subprocs < MAX_SUBPROCS); | ||
2353 | ext_subprocs[num_subprocs++] = copier_id; | ||
2354 | |||
2355 | /* The parent closes both pipes after duplicating the output of | ||
2356 | * compression. */ | ||
2357 | input_fd = dup (outpipefd[PIPE_READ_FD]); | ||
2358 | |||
2359 | if (input_fd < 0 || | ||
2360 | main_file_close (ifile) || | ||
2361 | close (outpipefd[PIPE_READ_FD]) || | ||
2362 | close (outpipefd[PIPE_WRITE_FD]) || | ||
2363 | close (inpipefd[PIPE_READ_FD]) || | ||
2364 | close (inpipefd[PIPE_WRITE_FD])) | ||
2365 | { | ||
2366 | XPR(NT "dup/close failed: %s\n", xd3_mainerror (ret = get_errno ())); | ||
2367 | goto pipe_cleanup; | ||
2368 | } | ||
2369 | |||
2370 | #if XD3_STDIO | ||
2371 | /* Note: fdopen() acquires the fd, closes it when finished. */ | ||
2372 | if ((ifile->file = fdopen (input_fd, "r")) == NULL) | ||
2373 | { | ||
2374 | XPR(NT "fdopen failed: %s\n", xd3_mainerror (ret = get_errno ())); | ||
2375 | goto pipe_cleanup; | ||
2376 | } | ||
2377 | |||
2378 | #elif XD3_POSIX | ||
2379 | ifile->file = input_fd; | ||
2380 | #endif | ||
2381 | |||
2382 | ifile->compressor = decomp; | ||
2383 | |||
2384 | /* Now the input file is decompressed. */ | ||
2385 | return main_file_read (ifile, input_buf, input_bufsize, | ||
2386 | nread, "input decompression failed"); | ||
2387 | |||
2388 | pipe_cleanup: | ||
2389 | close (input_fd); | ||
2390 | close (outpipefd[PIPE_READ_FD]); | ||
2391 | close (outpipefd[PIPE_WRITE_FD]); | ||
2392 | close (inpipefd[PIPE_READ_FD]); | ||
2393 | close (inpipefd[PIPE_WRITE_FD]); | ||
2394 | return ret; | ||
2395 | } | ||
2396 | |||
2397 | |||
2398 | /* This routine is called when the first buffer of input data is read | ||
2399 | * by the main program (unless input decompression is disabled by | ||
2400 | * command-line option). If it recognizes the magic number of a known | ||
2401 | * input type it invokes decompression. | ||
2402 | * | ||
2403 | * Skips decompression if the decompression type or the file type is | ||
2404 | * RD_NONEXTERNAL. | ||
2405 | * | ||
2406 | * Behaves exactly like main_file_read, otherwise. | ||
2407 | * | ||
2408 | * This function uses a separate buffer to read the first small block | ||
2409 | * of input. If a compressed input is detected, the separate buffer | ||
2410 | * is passed to the pipe copier. This avoids using the same size | ||
2411 | * buffer in both cases. */ | ||
2412 | static int | ||
2413 | main_secondary_decompress_check (main_file *file, | ||
2414 | uint8_t *input_buf, | ||
2415 | usize_t input_size, | ||
2416 | usize_t *nread) | ||
2417 | { | ||
2418 | int ret; | ||
2419 | usize_t i; | ||
2420 | usize_t try_read = min (input_size, XD3_ALLOCSIZE); | ||
2421 | usize_t check_nread; | ||
2422 | uint8_t check_buf[XD3_ALLOCSIZE]; /* TODO: stack limit */ | ||
2423 | const main_extcomp *decompressor = NULL; | ||
2424 | |||
2425 | if ((ret = main_file_read (file, check_buf, | ||
2426 | try_read, | ||
2427 | & check_nread, "input read failed"))) | ||
2428 | { | ||
2429 | return ret; | ||
2430 | } | ||
2431 | |||
2432 | if (file->flags & RD_DECOMPSET) | ||
2433 | { | ||
2434 | /* This allows the application header to override the magic | ||
2435 | * number, for whatever reason. */ | ||
2436 | decompressor = file->compressor; | ||
2437 | } | 338 | } |
2438 | else | 339 | else |
2439 | { | 340 | { |
2440 | for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) | 341 | /* Sequential search through LRU. */ |
2441 | { | 342 | for (i = 0; i < lru_size; i += 1) |
2442 | const main_extcomp *decomp = & extcomp_types[i]; | ||
2443 | |||
2444 | if (check_nread > decomp->magic_size) | ||
2445 | { | ||
2446 | /* The following expr checks if we are trying to read a | ||
2447 | * VCDIFF input, in which case do not treat it as | ||
2448 | * "secondary" decompression. */ | ||
2449 | int skip_this_type = (decomp->flags & RD_NONEXTERNAL) && | ||
2450 | (file->flags & RD_NONEXTERNAL); | ||
2451 | |||
2452 | if (skip_this_type) | ||
2453 | { | ||
2454 | continue; | ||
2455 | } | ||
2456 | |||
2457 | if (memcmp (check_buf, decomp->magic, decomp->magic_size) == 0) | ||
2458 | { | ||
2459 | decompressor = decomp; | ||
2460 | break; | ||
2461 | } | ||
2462 | } | ||
2463 | } | ||
2464 | } | ||
2465 | |||
2466 | if (decompressor != NULL) | ||
2467 | { | ||
2468 | if (! option_quiet) | ||
2469 | { | 343 | { |
2470 | XPR(NT "externally compressed input: %s %s%s < %s\n", | 344 | blru = & lru[i]; |
2471 | decompressor->decomp_cmdname, | 345 | if (blru->blkno == blkno) |
2472 | decompressor->decomp_options, | ||
2473 | (option_force2 ? " -f" : ""), | ||
2474 | file->filename); | ||
2475 | if (file->flags & RD_MAININPUT) | ||
2476 | { | 346 | { |
2477 | XPR(NT | 347 | main_blklru_list_remove (blru); |
2478 | "WARNING: the encoder is automatically decompressing the input file;\n"); | 348 | main_blklru_list_push_back (& lru_list, blru); |
2479 | XPR(NT | 349 | (*blrup) = blru; |
2480 | "WARNING: the decoder will automatically recompress the output file;\n"); | 350 | return 0; |
2481 | XPR(NT | ||
2482 | "WARNING: this may result in different compressed data and checksums\n"); | ||
2483 | XPR(NT | ||
2484 | "WARNING: despite being identical data; if this is an issue, use -D\n"); | ||
2485 | XPR(NT | ||
2486 | "WARNING: to avoid decompression and/or use -R to avoid recompression\n"); | ||
2487 | XPR(NT | ||
2488 | "WARNING: and/or manually decompress the input file; if you know the\n"); | ||
2489 | XPR(NT | ||
2490 | "WARNING: compression settings that will produce identical output\n"); | ||
2491 | XPR(NT | ||
2492 | "WARNING: you may set those flags using the environment (e.g., GZIP=-9)\n"); | ||
2493 | } | 351 | } |
2494 | } | 352 | } |
2495 | |||
2496 | file->size_known = 0; | ||
2497 | return main_input_decompress_setup (decompressor, file, | ||
2498 | input_buf, input_size, | ||
2499 | check_buf, XD3_ALLOCSIZE, | ||
2500 | check_nread, nread); | ||
2501 | } | ||
2502 | |||
2503 | /* Now read the rest of the input block. */ | ||
2504 | (*nread) = 0; | ||
2505 | |||
2506 | if (check_nread == try_read) | ||
2507 | { | ||
2508 | ret = main_file_read (file, | ||
2509 | input_buf + try_read, | ||
2510 | input_size - try_read, | ||
2511 | nread, | ||
2512 | "input read failed"); | ||
2513 | } | ||
2514 | |||
2515 | memcpy (input_buf, check_buf, check_nread); | ||
2516 | |||
2517 | (*nread) += check_nread; | ||
2518 | |||
2519 | return 0; | ||
2520 | } | ||
2521 | |||
2522 | /* Initiate re-compression of the output stream. This is easier than | ||
2523 | * input decompression because we know beforehand that the stream will | ||
2524 | * be compressed, whereas the input has already been read when we | ||
2525 | * decide it should be decompressed. Thus, it only requires one | ||
2526 | * subprocess and one pipe. */ | ||
2527 | static int | ||
2528 | main_recompress_output (main_file *ofile) | ||
2529 | { | ||
2530 | pid_t recomp_id; /* One subproc. */ | ||
2531 | int pipefd[2]; /* One pipe. */ | ||
2532 | int output_fd = -1; | ||
2533 | int ret; | ||
2534 | const main_extcomp *recomp = ofile->compressor; | ||
2535 | |||
2536 | pipefd[0] = pipefd[1] = -1; | ||
2537 | |||
2538 | if (pipe (pipefd)) | ||
2539 | { | ||
2540 | XPR(NT "pipe failed: %s\n", xd3_mainerror (ret = get_errno ())); | ||
2541 | goto pipe_cleanup; | ||
2542 | } | ||
2543 | |||
2544 | if ((recomp_id = fork ()) < 0) | ||
2545 | { | ||
2546 | XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ())); | ||
2547 | goto pipe_cleanup; | ||
2548 | } | ||
2549 | |||
2550 | /* The child runs the recompression process: */ | ||
2551 | if (recomp_id == 0) | ||
2552 | { | ||
2553 | if (option_verbose > 2) | ||
2554 | { | ||
2555 | XPR(NT "external recompression pid %d\n", getpid ()); | ||
2556 | } | ||
2557 | |||
2558 | /* Setup pipes: write to the output file, read from the pipe. */ | ||
2559 | if (dup2 (XFNO (ofile), STDOUT_FILENO) < 0 || | ||
2560 | dup2 (pipefd[PIPE_READ_FD], STDIN_FILENO) < 0 || | ||
2561 | close (pipefd[PIPE_READ_FD]) || | ||
2562 | close (pipefd[PIPE_WRITE_FD]) || | ||
2563 | execlp (recomp->recomp_cmdname, recomp->recomp_cmdname, | ||
2564 | recomp->recomp_options, | ||
2565 | option_force2 ? "-f" : NULL, | ||
2566 | NULL)) | ||
2567 | { | ||
2568 | XPR(NT "child process %s failed to execute: %s\n", | ||
2569 | recomp->recomp_cmdname, xd3_mainerror (get_errno ())); | ||
2570 | } | ||
2571 | |||
2572 | _exit (127); | ||
2573 | } | ||
2574 | |||
2575 | XD3_ASSERT(num_subprocs < MAX_SUBPROCS); | ||
2576 | ext_subprocs[num_subprocs++] = recomp_id; | ||
2577 | |||
2578 | /* The parent closes both pipes after duplicating the output-fd for | ||
2579 | * writing to the compression pipe. */ | ||
2580 | output_fd = dup (pipefd[PIPE_WRITE_FD]); | ||
2581 | |||
2582 | if (output_fd < 0 || | ||
2583 | main_file_close (ofile) || | ||
2584 | close (pipefd[PIPE_READ_FD]) || | ||
2585 | close (pipefd[PIPE_WRITE_FD])) | ||
2586 | { | ||
2587 | XPR(NT "close failed: %s\n", xd3_mainerror (ret = get_errno ())); | ||
2588 | goto pipe_cleanup; | ||
2589 | } | ||
2590 | |||
2591 | #if XD3_STDIO | ||
2592 | /* Note: fdopen() acquires the fd, closes it when finished. */ | ||
2593 | if ((ofile->file = fdopen (output_fd, "w")) == NULL) | ||
2594 | { | ||
2595 | XPR(NT "fdopen failed: %s\n", xd3_mainerror (ret = get_errno ())); | ||
2596 | goto pipe_cleanup; | ||
2597 | } | ||
2598 | |||
2599 | #elif XD3_POSIX | ||
2600 | ofile->file = output_fd; | ||
2601 | #endif | ||
2602 | |||
2603 | /* Now the output file will be compressed. */ | ||
2604 | return 0; | ||
2605 | |||
2606 | pipe_cleanup: | ||
2607 | close (output_fd); | ||
2608 | close (pipefd[PIPE_READ_FD]); | ||
2609 | close (pipefd[PIPE_WRITE_FD]); | ||
2610 | return ret; | ||
2611 | } | ||
2612 | #endif /* EXTERNAL_COMPRESSION */ | ||
2613 | |||
2614 | /* Identify the compressor that was used based on its ident string, | ||
2615 | * which is passed in the application header. */ | ||
2616 | static const main_extcomp* | ||
2617 | main_ident_compressor (const char *ident) | ||
2618 | { | ||
2619 | usize_t i; | ||
2620 | |||
2621 | for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) | ||
2622 | { | ||
2623 | if (strcmp (extcomp_types[i].ident, ident) == 0) | ||
2624 | { | ||
2625 | return & extcomp_types[i]; | ||
2626 | } | ||
2627 | } | ||
2628 | |||
2629 | return NULL; | ||
2630 | } | ||
2631 | |||
2632 | /* Return the main_extcomp record to use for this identifier, if possible. */ | ||
2633 | static const main_extcomp* | ||
2634 | main_get_compressor (const char *ident) | ||
2635 | { | ||
2636 | const main_extcomp *ext = main_ident_compressor (ident); | ||
2637 | |||
2638 | if (ext == NULL) | ||
2639 | { | ||
2640 | if (! option_quiet) | ||
2641 | { | ||
2642 | XPR(NT "warning: cannot recompress output: " | ||
2643 | "unrecognized external compression ID: %s\n", ident); | ||
2644 | } | ||
2645 | return NULL; | ||
2646 | } | ||
2647 | else if (! EXTERNAL_COMPRESSION) | ||
2648 | { | ||
2649 | if (! option_quiet) | ||
2650 | { | ||
2651 | XPR(NT "warning: external support not compiled: " | ||
2652 | "original input was compressed: %s\n", ext->recomp_cmdname); | ||
2653 | } | ||
2654 | return NULL; | ||
2655 | } | ||
2656 | else | ||
2657 | { | ||
2658 | return ext; | ||
2659 | } | 353 | } |
2660 | } | ||
2661 | |||
2662 | /********************************************************************* | ||
2663 | APPLICATION HEADER | ||
2664 | *******************************************************************/ | ||
2665 | |||
2666 | #if XD3_ENCODER | ||
2667 | static const char* | ||
2668 | main_apphead_string (const char* x) | ||
2669 | { | ||
2670 | const char *y; | ||
2671 | |||
2672 | if (x == NULL) { return ""; } | ||
2673 | |||
2674 | if (strcmp (x, "/dev/stdin") == 0 || | ||
2675 | strcmp (x, "/dev/stdout") == 0 || | ||
2676 | strcmp (x, "/dev/stderr") == 0) { return "-"; } | ||
2677 | |||
2678 | // TODO: this is not portable | ||
2679 | return (y = strrchr (x, '/')) == NULL ? x : y + 1; | ||
2680 | } | ||
2681 | |||
2682 | static int | ||
2683 | main_set_appheader (xd3_stream *stream, main_file *input, main_file *sfile) | ||
2684 | { | ||
2685 | /* The user may disable the application header. Once the appheader | ||
2686 | * is set, this disables setting it again. */ | ||
2687 | if (appheader_used || ! option_use_appheader) { return 0; } | ||
2688 | 354 | ||
2689 | /* The user may specify the application header, otherwise format the | 355 | if (do_src_fifo) |
2690 | default header. */ | ||
2691 | if (option_appheader) | ||
2692 | { | 356 | { |
2693 | appheader_used = option_appheader; | 357 | int idx = blkno % lru_size; |
358 | blru = & lru[idx]; | ||
2694 | } | 359 | } |
2695 | else | 360 | else |
2696 | { | 361 | { |
2697 | const char *iname; | 362 | if (! main_blklru_list_empty (& lru_free)) |
2698 | const char *icomp; | ||
2699 | const char *sname; | ||
2700 | const char *scomp; | ||
2701 | usize_t len; | ||
2702 | |||
2703 | iname = main_apphead_string (input->filename); | ||
2704 | icomp = (input->compressor == NULL) ? "" : input->compressor->ident; | ||
2705 | len = (usize_t) strlen (iname) + (usize_t) strlen (icomp) + 2; | ||
2706 | |||
2707 | if (sfile->filename != NULL) | ||
2708 | { | ||
2709 | sname = main_apphead_string (sfile->filename); | ||
2710 | scomp = (sfile->compressor == NULL) ? "" : sfile->compressor->ident; | ||
2711 | len += (usize_t) strlen (sname) + (usize_t) strlen (scomp) + 2; | ||
2712 | } | ||
2713 | else | ||
2714 | { | ||
2715 | sname = scomp = ""; | ||
2716 | } | ||
2717 | |||
2718 | if ((appheader_used = (uint8_t*) main_malloc (len)) == NULL) | ||
2719 | { | ||
2720 | return ENOMEM; | ||
2721 | } | ||
2722 | |||
2723 | if (sfile->filename == NULL) | ||
2724 | { | 363 | { |
2725 | sprintf ((char*)appheader_used, "%s/%s", iname, icomp); | 364 | blru = main_blklru_list_pop_front (& lru_free); |
365 | main_blklru_list_push_back (& lru_list, blru); | ||
2726 | } | 366 | } |
2727 | else | 367 | else |
2728 | { | 368 | { |
2729 | sprintf ((char*)appheader_used, "%s/%s/%s/%s", | 369 | XD3_ASSERT (! main_blklru_list_empty (& lru_list)); |
2730 | iname, icomp, sname, scomp); | 370 | blru = main_blklru_list_pop_front (& lru_list); |
371 | main_blklru_list_push_back (& lru_list, blru); | ||
2731 | } | 372 | } |
2732 | } | 373 | } |
2733 | 374 | ||
2734 | xd3_set_appheader (stream, appheader_used, | 375 | lru_filled += 1; |
2735 | (usize_t) strlen ((char*)appheader_used)); | 376 | (*is_new) = 1; |
2736 | 377 | (*blrup) = blru; | |
378 | blru->blkno = blkno; | ||
2737 | return 0; | 379 | return 0; |
2738 | } | 380 | } |
2739 | #endif | ||
2740 | |||
2741 | static void | ||
2742 | main_get_appheader_params (main_file *file, char **parsed, | ||
2743 | int output, const char *type, | ||
2744 | main_file *other) | ||
2745 | { | ||
2746 | /* Set the filename if it was not specified. If output, option_stdout (-c) | ||
2747 | * overrides. */ | ||
2748 | if (file->filename == NULL && | ||
2749 | ! (output && option_stdout) && | ||
2750 | strcmp (parsed[0], "-") != 0) | ||
2751 | { | ||
2752 | file->filename = parsed[0]; | ||
2753 | |||
2754 | if (other->filename != NULL) { | ||
2755 | /* Take directory from the other file, if it has one. */ | ||
2756 | /* TODO: This results in nonsense names like /dev/foo.tar.gz | ||
2757 | * and probably the filename-default logic interferes with | ||
2758 | * multi-file operation and the standard file extension? | ||
2759 | * Possibly the name header is bad, should be off by default. | ||
2760 | * Possibly we just want to remember external/compression | ||
2761 | * settings. */ | ||
2762 | const char *last_slash = strrchr(other->filename, '/'); | ||
2763 | |||
2764 | if (last_slash != NULL) { | ||
2765 | usize_t dlen = (usize_t) (last_slash - other->filename); | ||
2766 | |||
2767 | XD3_ASSERT(file->filename_copy == NULL); | ||
2768 | file->filename_copy = | ||
2769 | (char*) main_malloc(dlen + 2 + (usize_t) strlen(file->filename)); | ||
2770 | |||
2771 | strncpy(file->filename_copy, other->filename, dlen); | ||
2772 | file->filename_copy[dlen] = '/'; | ||
2773 | strcpy(file->filename_copy + dlen + 1, parsed[0]); | ||
2774 | |||
2775 | file->filename = file->filename_copy; | ||
2776 | } | ||
2777 | } | ||
2778 | |||
2779 | if (! option_quiet) | ||
2780 | { | ||
2781 | XPR(NT "using default %s filename: %s\n", type, file->filename); | ||
2782 | } | ||
2783 | } | ||
2784 | |||
2785 | /* Set the compressor, initiate de/recompression later. */ | ||
2786 | if (file->compressor == NULL && *parsed[1] != 0) | ||
2787 | { | ||
2788 | file->flags |= RD_DECOMPSET; | ||
2789 | file->compressor = main_get_compressor (parsed[1]); | ||
2790 | } | ||
2791 | } | ||
2792 | |||
2793 | static void | ||
2794 | main_get_appheader (xd3_stream *stream, main_file *ifile, | ||
2795 | main_file *output, main_file *sfile) | ||
2796 | { | ||
2797 | uint8_t *apphead; | ||
2798 | usize_t appheadsz; | ||
2799 | int ret; | ||
2800 | |||
2801 | /* The user may disable the application header. Once the appheader | ||
2802 | * is set, this disables setting it again. */ | ||
2803 | if (! option_use_appheader) { return; } | ||
2804 | |||
2805 | ret = xd3_get_appheader (stream, & apphead, & appheadsz); | ||
2806 | |||
2807 | /* Ignore failure, it only means we haven't received a header yet. */ | ||
2808 | if (ret != 0) { return; } | ||
2809 | |||
2810 | if (appheadsz > 0) | ||
2811 | { | ||
2812 | char *start = (char*)apphead; | ||
2813 | char *slash; | ||
2814 | int place = 0; | ||
2815 | char *parsed[4]; | ||
2816 | |||
2817 | memset (parsed, 0, sizeof (parsed)); | ||
2818 | 381 | ||
2819 | while ((slash = strchr (start, '/')) != NULL) | ||
2820 | { | ||
2821 | *slash = 0; | ||
2822 | parsed[place++] = start; | ||
2823 | start = slash + 1; | ||
2824 | } | ||
2825 | |||
2826 | parsed[place++] = start; | ||
2827 | |||
2828 | /* First take the output parameters. */ | ||
2829 | if (place == 2 || place == 4) | ||
2830 | { | ||
2831 | main_get_appheader_params (output, parsed, 1, "output", ifile); | ||
2832 | } | ||
2833 | |||
2834 | /* Then take the source parameters. */ | ||
2835 | if (place == 4) | ||
2836 | { | ||
2837 | main_get_appheader_params (sfile, parsed+2, 0, "source", ifile); | ||
2838 | } | ||
2839 | } | ||
2840 | |||
2841 | option_use_appheader = 0; | ||
2842 | return; | ||
2843 | } | ||
2844 | |||
2845 | /********************************************************************* | ||
2846 | Main I/O routines | ||
2847 | **********************************************************************/ | ||
2848 | |||
2849 | /* This function acts like the above except it may also try to | ||
2850 | * recognize a compressed input (source or target) when the first | ||
2851 | * buffer of data is read. The EXTERNAL_COMPRESSION code is called to | ||
2852 | * search for magic numbers. */ | ||
2853 | static int | 382 | static int |
2854 | main_read_primary_input (main_file *file, | 383 | main_read_seek_source (xd3_stream *stream, |
2855 | uint8_t *buf, | 384 | xd3_source *source, |
2856 | usize_t size, | 385 | xoff_t blkno) { |
2857 | usize_t *nread) | 386 | xoff_t pos = blkno * source->blksize; |
2858 | { | 387 | main_file *sfile = (main_file*) source->ioh; |
2859 | #if EXTERNAL_COMPRESSION | 388 | main_blklru *blru; |
2860 | if (option_decompress_inputs && file->flags & RD_FIRST) | 389 | int is_new; |
2861 | { | 390 | usize_t nread = 0; |
2862 | file->flags &= ~RD_FIRST; | 391 | int ret = 0; |
2863 | return main_secondary_decompress_check (file, buf, size, nread); | ||
2864 | } | ||
2865 | #endif | ||
2866 | |||
2867 | return main_file_read (file, buf, size, nread, "input read failed"); | ||
2868 | } | ||
2869 | |||
2870 | /* Open the main output file, sets a default file name, initiate | ||
2871 | * recompression. This function is expected to fprint any error | ||
2872 | * messages. */ | ||
2873 | static int | ||
2874 | main_open_output (xd3_stream *stream, main_file *ofile) | ||
2875 | { | ||
2876 | int ret; | ||
2877 | |||
2878 | if (option_no_output) | ||
2879 | { | ||
2880 | return 0; | ||
2881 | } | ||
2882 | 392 | ||
2883 | if (ofile->filename == NULL) | 393 | if (!sfile->seek_failed) |
2884 | { | 394 | { |
2885 | XSTDOUT_XF (ofile); | 395 | ret = main_file_seek (sfile, pos); |
2886 | 396 | ||
2887 | if (option_verbose > 1) | 397 | if (ret == 0) |
2888 | { | 398 | { |
2889 | XPR(NT "using standard output: %s\n", ofile->filename); | 399 | sfile->source_position = pos; |
2890 | } | 400 | } |
2891 | } | 401 | } |
2892 | else | 402 | |
403 | if (sfile->seek_failed || ret != 0) | ||
2893 | { | 404 | { |
2894 | /* Stat the file to check for overwrite. */ | 405 | /* For an unseekable file (or other seek error, does it |
2895 | if (option_force == 0 && main_file_exists (ofile)) | 406 | * matter?) */ |
407 | if (sfile->source_position > pos) | ||
2896 | { | 408 | { |
409 | /* Could assert !IS_ENCODE(), this shouldn't happen | ||
410 | * because of do_src_fifo during encode. */ | ||
2897 | if (!option_quiet) | 411 | if (!option_quiet) |
2898 | { | 412 | { |
2899 | XPR(NT "to overwrite output file specify -f: %s\n", | 413 | XPR(NT "source can't seek backwards; requested block offset " |
2900 | ofile->filename); | 414 | "%"Q"u source position is %"Q"u\n", |
415 | pos, sfile->source_position); | ||
2901 | } | 416 | } |
2902 | return EEXIST; | ||
2903 | } | ||
2904 | 417 | ||
2905 | if ((ret = main_file_open (ofile, ofile->filename, XO_WRITE))) | 418 | sfile->seek_failed = 1; |
2906 | { | 419 | stream->msg = "non-seekable source: " |
2907 | return ret; | 420 | "copy is too far back (try raising -B)"; |
421 | return XD3_TOOFARBACK; | ||
2908 | } | 422 | } |
2909 | 423 | ||
2910 | if (option_verbose > 1) { XPR(NT "output %s\n", ofile->filename); } | 424 | /* There's a chance here, that an genuine lseek error will cause |
2911 | } | 425 | * xdelta3 to shift into non-seekable mode, entering a degraded |
2912 | 426 | * condition. */ | |
2913 | #if EXTERNAL_COMPRESSION | 427 | if (!sfile->seek_failed && option_verbose) |
2914 | /* Do output recompression. */ | ||
2915 | if (ofile->compressor != NULL && option_recompress_outputs == 1) | ||
2916 | { | ||
2917 | if (! option_quiet) | ||
2918 | { | 428 | { |
2919 | XPR(NT "externally compressed output: %s %s%s > %s\n", | 429 | XPR(NT "source can't seek, will use FIFO for %s\n", |
2920 | ofile->compressor->recomp_cmdname, | 430 | sfile->filename); |
2921 | ofile->compressor->recomp_options, | ||
2922 | (option_force2 ? " -f" : ""), | ||
2923 | ofile->filename); | ||
2924 | } | ||
2925 | 431 | ||
2926 | if ((ret = main_recompress_output (ofile))) | 432 | if (option_verbose > 1) |
2927 | { | 433 | { |
2928 | return ret; | 434 | XPR(NT "seek error at offset %"Q"u: %s\n", |
435 | pos, xd3_mainerror (ret)); | ||
436 | } | ||
2929 | } | 437 | } |
2930 | } | ||
2931 | #endif | ||
2932 | |||
2933 | return 0; | ||
2934 | } | ||
2935 | 438 | ||
2936 | static usize_t | 439 | sfile->seek_failed = 1; |
2937 | main_get_winsize (main_file *ifile) { | ||
2938 | xoff_t file_size = 0; | ||
2939 | usize_t size = option_winsize; | ||
2940 | static char iszbuf[32]; | ||
2941 | 440 | ||
2942 | if (main_file_stat (ifile, &file_size) == 0) | 441 | while (sfile->source_position < pos) |
2943 | { | ||
2944 | size = (usize_t) min(file_size, (xoff_t) size); | ||
2945 | } | ||
2946 | |||
2947 | size = max(size, XD3_ALLOCSIZE); | ||
2948 | |||
2949 | if (option_verbose > 1) | ||
2950 | { | ||
2951 | XPR(NT "input %s window size %s\n", | ||
2952 | ifile->filename, | ||
2953 | main_format_bcnt (size, iszbuf)); | ||
2954 | } | ||
2955 | |||
2956 | return size; | ||
2957 | } | ||
2958 | |||
2959 | /********************************************************************* | ||
2960 | Main routines | ||
2961 | ********************************************************************/ | ||
2962 | |||
2963 | /* This is a generic input function. It calls the xd3_encode_input or | ||
2964 | * xd3_decode_input functions and makes calls to the various input | ||
2965 | * handling routines above, which coordinate external decompression. | ||
2966 | */ | ||
2967 | static int | ||
2968 | main_input (xd3_cmd cmd, | ||
2969 | main_file *ifile, | ||
2970 | main_file *ofile, | ||
2971 | main_file *sfile) | ||
2972 | { | ||
2973 | int ret; | ||
2974 | xd3_stream stream; | ||
2975 | usize_t nread = 0; | ||
2976 | usize_t winsize; | ||
2977 | int stream_flags = 0; | ||
2978 | xd3_config config; | ||
2979 | xd3_source source; | ||
2980 | xoff_t last_total_in = 0; | ||
2981 | xoff_t last_total_out = 0; | ||
2982 | long start_time; | ||
2983 | int stdout_only = 0; | ||
2984 | int (*input_func) (xd3_stream*); | ||
2985 | int (*output_func) (xd3_stream*, main_file *); | ||
2986 | |||
2987 | memset (& stream, 0, sizeof (stream)); | ||
2988 | memset (& source, 0, sizeof (source)); | ||
2989 | memset (& config, 0, sizeof (config)); | ||
2990 | |||
2991 | config.alloc = main_alloc; | ||
2992 | config.freef = main_free1; | ||
2993 | |||
2994 | config.iopt_size = option_iopt_size; | ||
2995 | config.sprevsz = option_sprevsz; | ||
2996 | |||
2997 | do_src_fifo = 0; | ||
2998 | |||
2999 | start_time = get_millisecs_now (); | ||
3000 | |||
3001 | if (option_use_checksum) { stream_flags |= XD3_ADLER32; } | ||
3002 | |||
3003 | /* main_input setup. */ | ||
3004 | switch ((int) cmd) | ||
3005 | { | ||
3006 | #if VCDIFF_TOOLS | ||
3007 | if (1) { case CMD_PRINTHDR: stream_flags |= XD3_JUST_HDR; } | ||
3008 | else if (1) { case CMD_PRINTHDRS: stream_flags |= XD3_SKIP_WINDOW; } | ||
3009 | else { case CMD_PRINTDELTA: stream_flags |= XD3_SKIP_EMIT; } | ||
3010 | ifile->flags |= RD_NONEXTERNAL; | ||
3011 | input_func = xd3_decode_input; | ||
3012 | output_func = main_print_func; | ||
3013 | stream_flags |= XD3_ADLER32_NOVER; | ||
3014 | stdout_only = 1; | ||
3015 | break; | ||
3016 | |||
3017 | case CMD_RECODE: | ||
3018 | case CMD_MERGE: | ||
3019 | case CMD_MERGE_ARG: | ||
3020 | /* No source will be read */ | ||
3021 | stream_flags |= XD3_ADLER32_NOVER | XD3_SKIP_EMIT; | ||
3022 | ifile->flags |= RD_NONEXTERNAL; | ||
3023 | input_func = xd3_decode_input; | ||
3024 | |||
3025 | if ((ret = main_init_recode_stream ())) | ||
3026 | { | ||
3027 | return EXIT_FAILURE; | ||
3028 | } | ||
3029 | |||
3030 | if (cmd == CMD_RECODE) { output_func = main_recode_func; } | ||
3031 | else { output_func = main_merge_func; } | ||
3032 | break; | ||
3033 | #endif /* VCDIFF_TOOLS */ | ||
3034 | |||
3035 | #if XD3_ENCODER | ||
3036 | case CMD_ENCODE: | ||
3037 | do_src_fifo = 1; | ||
3038 | input_func = xd3_encode_input; | ||
3039 | output_func = main_write_output; | ||
3040 | |||
3041 | if (option_no_compress) { stream_flags |= XD3_NOCOMPRESS; } | ||
3042 | if (option_use_altcodetable) { stream_flags |= XD3_ALT_CODE_TABLE; } | ||
3043 | if (option_smatch_config) | ||
3044 | { | 442 | { |
3045 | const char *s = option_smatch_config; | 443 | xoff_t skip_blkno; |
3046 | char *e; | 444 | usize_t skip_offset; |
3047 | int values[XD3_SOFTCFG_VARCNT]; | 445 | |
3048 | int got; | 446 | xd3_blksize_div (sfile->source_position, source, |
447 | &skip_blkno, &skip_offset); | ||
3049 | 448 | ||
3050 | config.smatch_cfg = XD3_SMATCH_SOFT; | 449 | /* Read past unused data */ |
450 | XD3_ASSERT (pos - sfile->source_position >= source->blksize); | ||
451 | XD3_ASSERT (skip_offset == 0); | ||
3051 | 452 | ||
3052 | for (got = 0; got < XD3_SOFTCFG_VARCNT; got += 1, s = e + 1) | 453 | if ((ret = main_getblk_lru (source, skip_blkno, |
454 | & blru, & is_new))) | ||
3053 | { | 455 | { |
3054 | values[got] = strtol (s, &e, 10); | 456 | return ret; |
3055 | |||
3056 | if ((values[got] < 0) || | ||
3057 | (e == s) || | ||
3058 | (got < XD3_SOFTCFG_VARCNT-1 && *e == 0) || | ||
3059 | (got == XD3_SOFTCFG_VARCNT-1 && *e != 0)) | ||
3060 | { | ||
3061 | XPR(NT "invalid string match specifier (-C) %d: %s\n", | ||
3062 | got, s); | ||
3063 | return EXIT_FAILURE; | ||
3064 | } | ||
3065 | } | 457 | } |
3066 | 458 | ||
3067 | config.smatcher_soft.large_look = values[0]; | 459 | XD3_ASSERT (is_new); |
3068 | config.smatcher_soft.large_step = values[1]; | 460 | |
3069 | config.smatcher_soft.small_look = values[2]; | 461 | if (option_verbose > 1) |
3070 | config.smatcher_soft.small_chain = values[3]; | ||
3071 | config.smatcher_soft.small_lchain = values[4]; | ||
3072 | config.smatcher_soft.max_lazy = values[5]; | ||
3073 | config.smatcher_soft.long_enough = values[6]; | ||
3074 | } | ||
3075 | else | ||
3076 | { | ||
3077 | if (option_verbose > 2) | ||
3078 | { | 462 | { |
3079 | XPR(NT "compression level: %d\n", option_level); | 463 | XPR(NT "non-seekable source skipping %"Q"u bytes @ %"Q"u\n", |
464 | pos - sfile->source_position, | ||
465 | sfile->source_position); | ||
3080 | } | 466 | } |
3081 | if (option_level == 0) | 467 | |
468 | if ((ret = main_read_primary_input (sfile, | ||
469 | (uint8_t*) blru->blk, | ||
470 | source->blksize, | ||
471 | & nread))) | ||
3082 | { | 472 | { |
3083 | stream_flags |= XD3_NOCOMPRESS; | 473 | return ret; |
3084 | config.smatch_cfg = XD3_SMATCH_FASTEST; | ||
3085 | } | 474 | } |
3086 | else if (option_level == 1) | ||
3087 | { config.smatch_cfg = XD3_SMATCH_FASTEST; } | ||
3088 | else if (option_level == 2) | ||
3089 | { config.smatch_cfg = XD3_SMATCH_FASTER; } | ||
3090 | else if (option_level <= 5) | ||
3091 | { config.smatch_cfg = XD3_SMATCH_FAST; } | ||
3092 | else if (option_level == 6) | ||
3093 | { config.smatch_cfg = XD3_SMATCH_DEFAULT; } | ||
3094 | else | ||
3095 | { config.smatch_cfg = XD3_SMATCH_SLOW; } | ||
3096 | } | ||
3097 | break; | ||
3098 | #endif | ||
3099 | case CMD_DECODE: | ||
3100 | if (option_use_checksum == 0) { stream_flags |= XD3_ADLER32_NOVER; } | ||
3101 | ifile->flags |= RD_NONEXTERNAL; | ||
3102 | input_func = xd3_decode_input; | ||
3103 | output_func = main_write_output; | ||
3104 | break; | ||
3105 | default: | ||
3106 | XPR(NT "internal error\n"); | ||
3107 | return EXIT_FAILURE; | ||
3108 | } | ||
3109 | |||
3110 | main_bsize = winsize = main_get_winsize (ifile); | ||
3111 | |||
3112 | if ((main_bdata = (uint8_t*) main_malloc (winsize)) == NULL) | ||
3113 | { | ||
3114 | return EXIT_FAILURE; | ||
3115 | } | ||
3116 | 475 | ||
3117 | config.winsize = winsize; | 476 | if (nread != source->blksize) |
3118 | config.srcwin_maxsz = option_srcwinsz; | ||
3119 | config.getblk = main_getblk_func; | ||
3120 | config.flags = stream_flags; | ||
3121 | |||
3122 | if ((ret = main_set_secondary_flags (&config)) || | ||
3123 | (ret = xd3_config_stream (& stream, & config))) | ||
3124 | { | ||
3125 | XPR(NT XD3_LIB_ERRMSG (& stream, ret)); | ||
3126 | return EXIT_FAILURE; | ||
3127 | } | ||
3128 | |||
3129 | #if VCDIFF_TOOLS | ||
3130 | if ((cmd == CMD_MERGE || cmd == CMD_MERGE_ARG) && | ||
3131 | (ret = xd3_whole_state_init (& stream))) | ||
3132 | { | ||
3133 | XPR(NT XD3_LIB_ERRMSG (& stream, ret)); | ||
3134 | return EXIT_FAILURE; | ||
3135 | } | ||
3136 | #endif | ||
3137 | |||
3138 | if (cmd != CMD_DECODE) | ||
3139 | { | ||
3140 | /* When not decoding, set source now. The decoder delays this | ||
3141 | * step until XD3_GOTHEADER. */ | ||
3142 | if (sfile && sfile->filename != NULL) | ||
3143 | { | ||
3144 | if ((ret = main_set_source (& stream, cmd, sfile, & source))) | ||
3145 | { | 477 | { |
3146 | return EXIT_FAILURE; | 478 | IF_DEBUG1 (DP(RINT "[getblk] short skip block nread = %u\n", |
479 | nread)); | ||
480 | stream->msg = "non-seekable input is short"; | ||
481 | return XD3_INVALID_INPUT; | ||
3147 | } | 482 | } |
3148 | 483 | ||
3149 | XD3_ASSERT(stream.src != NULL); | 484 | sfile->source_position += nread; |
3150 | } | 485 | blru->size = nread; |
3151 | } | ||
3152 | |||
3153 | if (cmd == CMD_PRINTHDR || | ||
3154 | cmd == CMD_PRINTHDRS || | ||
3155 | cmd == CMD_PRINTDELTA || | ||
3156 | cmd == CMD_RECODE) | ||
3157 | { | ||
3158 | if (sfile->filename == NULL) | ||
3159 | { | ||
3160 | allow_fake_source = 1; | ||
3161 | sfile->filename = "<placeholder>"; | ||
3162 | main_set_source (& stream, cmd, sfile, & source); | ||
3163 | } | ||
3164 | } | ||
3165 | |||
3166 | /* This times each window. */ | ||
3167 | get_millisecs_since (); | ||
3168 | |||
3169 | /* Main input loop. */ | ||
3170 | do | ||
3171 | { | ||
3172 | xoff_t input_offset; | ||
3173 | xoff_t input_remain; | ||
3174 | usize_t try_read; | ||
3175 | |||
3176 | input_offset = ifile->nread; | ||
3177 | |||
3178 | input_remain = XOFF_T_MAX - input_offset; | ||
3179 | |||
3180 | try_read = (usize_t) min ((xoff_t) config.winsize, input_remain); | ||
3181 | |||
3182 | if ((ret = main_read_primary_input (ifile, main_bdata, | ||
3183 | try_read, & nread))) | ||
3184 | { | ||
3185 | return EXIT_FAILURE; | ||
3186 | } | ||
3187 | 486 | ||
3188 | /* If we've reached EOF tell the stream to flush. */ | 487 | IF_DEBUG1 (DP(RINT "[getblk] skip blkno %"Q"u size %u\n", |
3189 | if (nread < try_read) | 488 | skip_blkno, blru->size)); |
3190 | { | ||
3191 | stream.flags |= XD3_FLUSH; | ||
3192 | } | ||
3193 | 489 | ||
3194 | #if XD3_ENCODER | 490 | XD3_ASSERT (sfile->source_position <= pos); |
3195 | /* After the first main_read_primary_input completes, we know | ||
3196 | * all the information needed to encode the application | ||
3197 | * header. */ | ||
3198 | if (cmd == CMD_ENCODE && | ||
3199 | (ret = main_set_appheader (& stream, ifile, sfile))) | ||
3200 | { | ||
3201 | return EXIT_FAILURE; | ||
3202 | } | 491 | } |
3203 | #endif | ||
3204 | xd3_avail_input (& stream, main_bdata, nread); | ||
3205 | |||
3206 | /* If we read zero bytes after encoding at least one window... */ | ||
3207 | if (nread == 0 && stream.current_window > 0) { | ||
3208 | break; | ||
3209 | } | ||
3210 | |||
3211 | again: | ||
3212 | ret = input_func (& stream); | ||
3213 | |||
3214 | switch (ret) | ||
3215 | { | ||
3216 | case XD3_INPUT: | ||
3217 | continue; | ||
3218 | |||
3219 | case XD3_GOTHEADER: | ||
3220 | { | ||
3221 | XD3_ASSERT (stream.current_window == 0); | ||
3222 | |||
3223 | /* Need to process the appheader as soon as possible. It may | ||
3224 | * contain a suggested default filename/decompression routine for | ||
3225 | * the ofile, and it may contain default/decompression routine for | ||
3226 | * the sources. */ | ||
3227 | if (cmd == CMD_DECODE) | ||
3228 | { | ||
3229 | /* May need to set the sfile->filename if none was given. */ | ||
3230 | main_get_appheader (& stream, ifile, ofile, sfile); | ||
3231 | |||
3232 | /* Now open the source file. */ | ||
3233 | if ((sfile->filename != NULL) && | ||
3234 | (ret = main_set_source (& stream, cmd, sfile, & source))) | ||
3235 | { | ||
3236 | return EXIT_FAILURE; | ||
3237 | } | ||
3238 | } | ||
3239 | } | ||
3240 | /* FALLTHROUGH */ | ||
3241 | case XD3_WINSTART: | ||
3242 | { | ||
3243 | /* e.g., set or unset XD3_SKIP_WINDOW. */ | ||
3244 | goto again; | ||
3245 | } | ||
3246 | |||
3247 | case XD3_OUTPUT: | ||
3248 | { | ||
3249 | /* Defer opening the output file until the stream produces its | ||
3250 | * first output for both encoder and decoder, this way we | ||
3251 | * delay long enough for the decoder to receive the | ||
3252 | * application header. (Or longer if there are skipped | ||
3253 | * windows, but I can't think of any reason not to delay | ||
3254 | * open.) */ | ||
3255 | if (ofile != NULL && | ||
3256 | ! main_file_isopen (ofile) && | ||
3257 | (ret = main_open_output (& stream, ofile)) != 0) | ||
3258 | { | ||
3259 | return EXIT_FAILURE; | ||
3260 | } | ||
3261 | |||
3262 | if ((ret = output_func (& stream, ofile)) && | ||
3263 | (ret != PRINTHDR_SPECIAL)) | ||
3264 | { | ||
3265 | return EXIT_FAILURE; | ||
3266 | } | ||
3267 | |||
3268 | if (ret == PRINTHDR_SPECIAL) | ||
3269 | { | ||
3270 | xd3_abort_stream (& stream); | ||
3271 | ret = EXIT_SUCCESS; | ||
3272 | goto done; | ||
3273 | } | ||
3274 | |||
3275 | ret = 0; | ||
3276 | |||
3277 | xd3_consume_output (& stream); | ||
3278 | goto again; | ||
3279 | } | ||
3280 | |||
3281 | case XD3_WINFINISH: | ||
3282 | { | ||
3283 | if (IS_ENCODE (cmd) || cmd == CMD_DECODE || cmd == CMD_RECODE) | ||
3284 | { | ||
3285 | if (! option_quiet && IS_ENCODE (cmd) && | ||
3286 | main_file_isopen (sfile)) | ||
3287 | { | ||
3288 | /* Warn when no source copies are found */ | ||
3289 | if (option_verbose && ! xd3_encoder_used_source (& stream)) | ||
3290 | { | ||
3291 | XPR(NT "warning: input window %"Q"u..%"Q"u has " | ||
3292 | "no source copies\n", | ||
3293 | stream.current_window * winsize, | ||
3294 | (stream.current_window+1) * winsize); | ||
3295 | XD3_ASSERT (stream.src != NULL); | ||
3296 | } | ||
3297 | |||
3298 | /* Limited i-buffer size affects source copies | ||
3299 | * when the sourcewin is decided early. */ | ||
3300 | if (option_verbose && | ||
3301 | stream.srcwin_decided_early && | ||
3302 | stream.i_slots_used > stream.iopt_size) | ||
3303 | { | ||
3304 | XPR(NT "warning: input position %"Q"u overflowed " | ||
3305 | "instruction buffer, needed %u (vs. %u), " | ||
3306 | "consider changing -I\n", | ||
3307 | stream.current_window * winsize, | ||
3308 | stream.i_slots_used, stream.iopt_size); | ||
3309 | } | ||
3310 | } | ||
3311 | |||
3312 | if (option_verbose) | ||
3313 | { | ||
3314 | char rrateavg[32], wrateavg[32], tm[32]; | ||
3315 | char rdb[32], wdb[32]; | ||
3316 | char trdb[32], twdb[32]; | ||
3317 | char srcpos[32]; | ||
3318 | long millis = get_millisecs_since (); | ||
3319 | usize_t this_read = (usize_t)(stream.total_in - | ||
3320 | last_total_in); | ||
3321 | usize_t this_write = (usize_t)(stream.total_out - | ||
3322 | last_total_out); | ||
3323 | last_total_in = stream.total_in; | ||
3324 | last_total_out = stream.total_out; | ||
3325 | |||
3326 | if (option_verbose > 1) | ||
3327 | { | ||
3328 | XPR(NT "%"Q"u: in %s (%s): out %s (%s): " | ||
3329 | "total in %s: out %s: %s: srcpos %s\n", | ||
3330 | stream.current_window, | ||
3331 | main_format_bcnt (this_read, rdb), | ||
3332 | main_format_rate (this_read, millis, rrateavg), | ||
3333 | main_format_bcnt (this_write, wdb), | ||
3334 | main_format_rate (this_write, millis, wrateavg), | ||
3335 | main_format_bcnt (stream.total_in, trdb), | ||
3336 | main_format_bcnt (stream.total_out, twdb), | ||
3337 | main_format_millis (millis, tm), | ||
3338 | main_format_bcnt (sfile->source_position, srcpos)); | ||
3339 | } | ||
3340 | else | ||
3341 | { | ||
3342 | XPR(NT "%"Q"u: in %s: out %s: total in %s: " | ||
3343 | "out %s: %s\n", | ||
3344 | stream.current_window, | ||
3345 | main_format_bcnt (this_read, rdb), | ||
3346 | main_format_bcnt (this_write, wdb), | ||
3347 | main_format_bcnt (stream.total_in, trdb), | ||
3348 | main_format_bcnt (stream.total_out, twdb), | ||
3349 | main_format_millis (millis, tm)); | ||
3350 | } | ||
3351 | } | ||
3352 | } | ||
3353 | goto again; | ||
3354 | } | ||
3355 | |||
3356 | default: | ||
3357 | /* input_func() error */ | ||
3358 | XPR(NT XD3_LIB_ERRMSG (& stream, ret)); | ||
3359 | return EXIT_FAILURE; | ||
3360 | } | ||
3361 | } | ||
3362 | while (nread == config.winsize); | ||
3363 | done: | ||
3364 | /* Close the inputs. (ifile must be open, sfile may be open) */ | ||
3365 | main_file_close (ifile); | ||
3366 | if (sfile != NULL) | ||
3367 | { | ||
3368 | main_file_close (sfile); | ||
3369 | } | ||
3370 | |||
3371 | #if VCDIFF_TOOLS | ||
3372 | if (cmd == CMD_MERGE && | ||
3373 | (ret = main_merge_output (& stream, ofile))) | ||
3374 | { | ||
3375 | return EXIT_FAILURE; | ||
3376 | } | 492 | } |
3377 | 493 | ||
3378 | if (cmd == CMD_MERGE_ARG) | 494 | return 0; |
3379 | { | 495 | } |
3380 | xd3_swap_whole_state (& stream.whole_target, | ||
3381 | & recode_stream->whole_target); | ||
3382 | } | ||
3383 | #endif /* VCDIFF_TOOLS */ | ||
3384 | |||
3385 | /* If output file is not open yet because of delayed-open, it means | ||
3386 | * we never encountered a window in the delta, but it could have had | ||
3387 | * a VCDIFF header? TODO: solve this elsewhere. For now, it prints | ||
3388 | * "nothing to output" below, but the check doesn't happen in case | ||
3389 | * of option_no_output. */ | ||
3390 | if (! option_no_output && ofile != NULL) | ||
3391 | { | ||
3392 | if (!stdout_only && ! main_file_isopen (ofile)) | ||
3393 | { | ||
3394 | XPR(NT "nothing to output: %s\n", ifile->filename); | ||
3395 | return EXIT_FAILURE; | ||
3396 | } | ||
3397 | 496 | ||
3398 | /* Have to close the output before calling | 497 | /* This is the callback for reading a block of source. This function |
3399 | * main_external_compression_finish, or else it hangs. */ | 498 | * is blocking and it implements a small LRU. |
3400 | if (main_file_close (ofile) != 0) | 499 | * |
3401 | { | 500 | * Note that it is possible for main_input() to handle getblk requests |
3402 | return EXIT_FAILURE; | 501 | * in a non-blocking manner. If the callback is NULL then the caller |
3403 | } | 502 | * of xd3_*_input() must handle the XD3_GETSRCBLK return value and |
503 | * fill the source in the same way. See xd3_getblk for details. To | ||
504 | * see an example of non-blocking getblk, see xdelta-test.h. */ | ||
505 | static int | ||
506 | main_getblk_func (xd3_stream *stream, | ||
507 | xd3_source *source, | ||
508 | xoff_t blkno) | ||
509 | { | ||
510 | int ret = 0; | ||
511 | xoff_t pos = blkno * source->blksize; | ||
512 | main_file *sfile = (main_file*) source->ioh; | ||
513 | main_blklru *blru; | ||
514 | int is_new; | ||
515 | int did_seek = 0; | ||
516 | usize_t nread = 0; | ||
517 | |||
518 | if (allow_fake_source) | ||
519 | { | ||
520 | source->curblkno = blkno; | ||
521 | source->onblk = 0; | ||
522 | source->curblk = lru[0].blk; | ||
523 | lru[0].size = 0; | ||
524 | return 0; | ||
3404 | } | 525 | } |
3405 | 526 | ||
3406 | #if EXTERNAL_COMPRESSION | 527 | if ((ret = main_getblk_lru (source, blkno, & blru, & is_new))) |
3407 | if ((ret = main_external_compression_finish ())) | ||
3408 | { | 528 | { |
3409 | XPR(NT "external compression commands failed\n"); | 529 | return ret; |
3410 | return EXIT_FAILURE; | ||
3411 | } | 530 | } |
3412 | #endif | ||
3413 | 531 | ||
3414 | if ((ret = xd3_close_stream (& stream))) | 532 | if (!is_new) |
3415 | { | 533 | { |
3416 | XPR(NT XD3_LIB_ERRMSG (& stream, ret)); | 534 | source->curblkno = blkno; |
3417 | return EXIT_FAILURE; | 535 | source->onblk = blru->size; |
536 | source->curblk = blru->blk; | ||
537 | lru_hits++; | ||
538 | return 0; | ||
3418 | } | 539 | } |
3419 | 540 | ||
3420 | #if XD3_ENCODER | 541 | lru_misses += 1; |
3421 | if (option_verbose > 1 && cmd == CMD_ENCODE) | 542 | |
543 | if (pos != sfile->source_position) | ||
3422 | { | 544 | { |
3423 | XPR(NT "scanner configuration: %s\n", stream.smatcher.name); | 545 | /* Only try to seek when the position is wrong. This means the |
3424 | XPR(NT "target hash table size: %u\n", stream.small_hash.size); | 546 | * decoder will fail when the source buffer is too small, but |
3425 | if (sfile != NULL && sfile->filename != NULL) | 547 | * only when the input is non-seekable. */ |
548 | if ((ret = main_read_seek_source (stream, source, blkno))) | ||
3426 | { | 549 | { |
3427 | XPR(NT "source hash table size: %u\n", stream.large_hash.size); | 550 | return ret; |
3428 | } | 551 | } |
3429 | } | ||
3430 | |||
3431 | if (option_verbose > 2 && cmd == CMD_ENCODE) | ||
3432 | { | ||
3433 | XPR(NT "source copies: %"Q"u (%"Q"u bytes)\n", | ||
3434 | stream.n_scpy, stream.l_scpy); | ||
3435 | XPR(NT "target copies: %"Q"u (%"Q"u bytes)\n", | ||
3436 | stream.n_tcpy, stream.l_tcpy); | ||
3437 | XPR(NT "adds: %"Q"u (%"Q"u bytes)\n", stream.n_add, stream.l_add); | ||
3438 | XPR(NT "runs: %"Q"u (%"Q"u bytes)\n", stream.n_run, stream.l_run); | ||
3439 | } | ||
3440 | #endif | ||
3441 | |||
3442 | xd3_free_stream (& stream); | ||
3443 | |||
3444 | if (option_verbose) | ||
3445 | { | ||
3446 | char tm[32]; | ||
3447 | long end_time = get_millisecs_now (); | ||
3448 | xoff_t nwrite = ofile != NULL ? ofile->nwrite : 0; | ||
3449 | 552 | ||
3450 | XPR(NT "finished in %s; input %"Q"u output %"Q"u bytes (%0.2f%%)\n", | 553 | /* Indicates that another call to main_getblk_lru() may be |
3451 | main_format_millis (end_time - start_time, tm), | 554 | * needed */ |
3452 | ifile->nread, nwrite, 100.0 * nwrite / ifile->nread); | 555 | did_seek = 1; |
3453 | } | 556 | } |
3454 | 557 | ||
3455 | return EXIT_SUCCESS; | 558 | XD3_ASSERT (sfile->source_position == pos); |
3456 | } | ||
3457 | |||
3458 | /* free memory before exit, reset single-use variables. */ | ||
3459 | static void | ||
3460 | main_cleanup (void) | ||
3461 | { | ||
3462 | if (appheader_used != NULL && | ||
3463 | appheader_used != option_appheader) | ||
3464 | { | ||
3465 | main_free (appheader_used); | ||
3466 | appheader_used = NULL; | ||
3467 | } | ||
3468 | |||
3469 | main_free (main_bdata); | ||
3470 | main_bdata = NULL; | ||
3471 | main_bsize = 0; | ||
3472 | |||
3473 | #if EXTERNAL_COMPRESSION | ||
3474 | main_free (ext_tmpfile); | ||
3475 | ext_tmpfile = NULL; | ||
3476 | #endif | ||
3477 | |||
3478 | main_lru_cleanup(); | ||
3479 | 559 | ||
3480 | if (recode_stream != NULL) | 560 | if (did_seek && |
561 | (ret = main_getblk_lru (source, blkno, & blru, & is_new))) | ||
3481 | { | 562 | { |
3482 | xd3_free_stream (recode_stream); | 563 | return ret; |
3483 | main_free (recode_stream); | ||
3484 | recode_stream = NULL; | ||
3485 | } | 564 | } |
3486 | 565 | ||
3487 | if (merge_stream != NULL) | 566 | if ((ret = main_read_primary_input (sfile, |
567 | (uint8_t*) blru->blk, | ||
568 | source->blksize, | ||
569 | & nread))) | ||
3488 | { | 570 | { |
3489 | xd3_free_stream (merge_stream); | 571 | return ret; |
3490 | main_free (merge_stream); | ||
3491 | merge_stream = NULL; | ||
3492 | } | ||
3493 | |||
3494 | XD3_ASSERT (main_mallocs == 0); | ||
3495 | } | ||
3496 | |||
3497 | static void | ||
3498 | setup_environment (int argc, | ||
3499 | char **argv, | ||
3500 | int *argc_out, | ||
3501 | char ***argv_out, | ||
3502 | char ***argv_free, | ||
3503 | char **env_free) | ||
3504 | { | ||
3505 | int n, i, i0; | ||
3506 | char *p, *v = getenv("XDELTA"); | ||
3507 | if (v == NULL) { | ||
3508 | (*argc_out) = argc; | ||
3509 | (*argv_out) = argv; | ||
3510 | (*argv_free) = NULL; | ||
3511 | (*env_free) = NULL; | ||
3512 | return; | ||
3513 | } | ||
3514 | |||
3515 | (*env_free) = (char*) main_malloc((usize_t) strlen(v) + 1); | ||
3516 | strcpy(*env_free, v); | ||
3517 | |||
3518 | /* Space needed for extra args, at least # of spaces */ | ||
3519 | n = argc + 1; | ||
3520 | for (p = *env_free; *p != 0; ) { | ||
3521 | if (*p++ == ' ') { | ||
3522 | n++; | ||
3523 | } | ||
3524 | } | ||
3525 | |||
3526 | (*argv_free) = (char**) main_malloc(sizeof(char*) * (n + 1)); | ||
3527 | (*argv_out) = (*argv_free); | ||
3528 | (*argv_out)[0] = argv[0]; | ||
3529 | (*argv_out)[n] = NULL; | ||
3530 | |||
3531 | i = 1; | ||
3532 | for (p = *env_free; *p != 0; ) { | ||
3533 | (*argv_out)[i++] = p; | ||
3534 | while (*p != ' ' && *p != 0) { | ||
3535 | p++; | ||
3536 | } | ||
3537 | while (*p == ' ') { | ||
3538 | *p++ = 0; | ||
3539 | } | 572 | } |
3540 | } | ||
3541 | |||
3542 | for (i0 = 1; i0 < argc; i0++) { | ||
3543 | (*argv_out)[i++] = argv[i0]; | ||
3544 | } | ||
3545 | 573 | ||
3546 | /* Counting spaces is an upper bound, argv stays NULL terminated. */ | 574 | /* Save the last block read, used to handle non-seekable files. */ |
3547 | (*argc_out) = i; | 575 | sfile->source_position = pos + nread; |
3548 | while (i <= n) { | ||
3549 | (*argv_out)[i++] = NULL; | ||
3550 | } | ||
3551 | } | ||
3552 | 576 | ||
3553 | int | 577 | if (option_verbose > 3) |
3554 | #if PYTHON_MODULE || SWIG_MODULE || NOT_MAIN | ||
3555 | xd3_main_cmdline (int argc, char **argv) | ||
3556 | #else | ||
3557 | main (int argc, char **argv) | ||
3558 | #endif | ||
3559 | { | ||
3560 | static const char *flags = | ||
3561 | "0123456789cdefhnqvDFJNORTVs:m:B:C:E:I:L:O:M:P:W:A::S::"; | ||
3562 | xd3_cmd cmd; | ||
3563 | main_file ifile; | ||
3564 | main_file ofile; | ||
3565 | main_file sfile; | ||
3566 | main_merge_list merge_order; | ||
3567 | main_merge *merge; | ||
3568 | int my_optind; | ||
3569 | const char *my_optarg; | ||
3570 | const char *my_optstr; | ||
3571 | const char *sfilename; | ||
3572 | int env_argc; | ||
3573 | char **env_argv; | ||
3574 | char **free_argv; /* malloc() in setup_environment() */ | ||
3575 | char *free_value; /* malloc() in setup_environment() */ | ||
3576 | int ret; | ||
3577 | |||
3578 | #ifdef _WIN32 | ||
3579 | GetStartupInfo(&winStartupInfo); | ||
3580 | setvbuf(stderr, NULL, _IONBF, 0); /* Do not buffer stderr */ | ||
3581 | #endif | ||
3582 | |||
3583 | main_file_init (& ifile); | ||
3584 | main_file_init (& ofile); | ||
3585 | main_file_init (& sfile); | ||
3586 | main_merge_list_init (& merge_order); | ||
3587 | |||
3588 | reset_defaults(); | ||
3589 | |||
3590 | free_argv = NULL; | ||
3591 | free_value = NULL; | ||
3592 | setup_environment(argc, argv, &env_argc, &env_argv, | ||
3593 | &free_argv, &free_value); | ||
3594 | cmd = CMD_NONE; | ||
3595 | sfilename = NULL; | ||
3596 | my_optind = 1; | ||
3597 | argv = env_argv; | ||
3598 | argc = env_argc; | ||
3599 | program_name = env_argv[0]; | ||
3600 | |||
3601 | takearg: | ||
3602 | my_optarg = NULL; | ||
3603 | my_optstr = argv[my_optind]; | ||
3604 | |||
3605 | /* This doesn't use getopt() because it makes trouble for -P & python which | ||
3606 | * reenter main() and thus care about freeing all memory. I never had much | ||
3607 | * trust for getopt anyway, it's too opaque. This implements a fairly | ||
3608 | * standard non-long-option getopt with support for named operations (e.g., | ||
3609 | * "xdelta3 [encode|decode|printhdr...] < in > out"). */ | ||
3610 | if (my_optstr) | ||
3611 | { | 578 | { |
3612 | if (*my_optstr == '-') { my_optstr += 1; } | 579 | if (blru->blkno != (xoff_t)-1) |
3613 | else if (cmd == CMD_NONE) { goto nonflag; } | ||
3614 | else { my_optstr = NULL; } | ||
3615 | } | ||
3616 | while (my_optstr) | ||
3617 | { | ||
3618 | const char *s; | ||
3619 | my_optarg = NULL; | ||
3620 | if ((ret = *my_optstr++) == 0) { my_optind += 1; goto takearg; } | ||
3621 | |||
3622 | /* Option handling: first check for one ':' following the option in | ||
3623 | * flags, then check for two. The syntax allows: | ||
3624 | * | ||
3625 | * 1. -Afoo defines optarg="foo" | ||
3626 | * 2. -A foo defines optarg="foo" | ||
3627 | * 3. -A "" defines optarg="" (allows empty-string) | ||
3628 | * 4. -A [EOA or -moreargs] error (mandatory case) | ||
3629 | * 5. -A [EOA -moreargs] defines optarg=NULL (optional case) | ||
3630 | * 6. -A=foo defines optarg="foo" | ||
3631 | * 7. -A= defines optarg="" (mandatory case) | ||
3632 | * 8. -A= defines optarg=NULL (optional case) | ||
3633 | * | ||
3634 | * See tests in test_command_line_arguments(). | ||
3635 | */ | ||
3636 | s = strchr (flags, ret); | ||
3637 | if (s && s[1] && s[1] == ':') | ||
3638 | { | ||
3639 | int eqcase = 0; | ||
3640 | int option = s[2] && s[2] == ':'; | ||
3641 | |||
3642 | /* Case 1, set optarg to the remaining characters. */ | ||
3643 | my_optarg = my_optstr; | ||
3644 | my_optstr = ""; | ||
3645 | |||
3646 | /* Case 2-5 */ | ||
3647 | if (*my_optarg == 0) | ||
3648 | { | ||
3649 | /* Condition 4-5 */ | ||
3650 | int have_arg = (my_optind < (argc - 1) && | ||
3651 | *argv[my_optind+1] != '-'); | ||
3652 | |||
3653 | if (! have_arg) | ||
3654 | { | ||
3655 | if (! option) | ||
3656 | { | ||
3657 | /* Case 4 */ | ||
3658 | XPR(NT "-%c: requires an argument\n", ret); | ||
3659 | ret = EXIT_FAILURE; | ||
3660 | goto cleanup; | ||
3661 | } | ||
3662 | /* Case 5. */ | ||
3663 | my_optarg = NULL; | ||
3664 | } | ||
3665 | else | ||
3666 | { | ||
3667 | /* Case 2-3. */ | ||
3668 | my_optarg = argv[++my_optind]; | ||
3669 | } | ||
3670 | } | ||
3671 | /* Case 6-8. */ | ||
3672 | else if (*my_optarg == '=') | ||
3673 | { | ||
3674 | /* Remove the = in all cases. */ | ||
3675 | my_optarg += 1; | ||
3676 | eqcase = 1; | ||
3677 | |||
3678 | if (option && *my_optarg == 0) | ||
3679 | { | ||
3680 | /* Case 8. */ | ||
3681 | my_optarg = NULL; | ||
3682 | } | ||
3683 | } | ||
3684 | } | ||
3685 | |||
3686 | switch (ret) | ||
3687 | { | 580 | { |
3688 | /* case: if no '-' was found, maybe check for a command name. */ | 581 | if (blru->blkno != blkno) |
3689 | nonflag: | ||
3690 | if (strcmp (my_optstr, "decode") == 0) { cmd = CMD_DECODE; } | ||
3691 | else if (strcmp (my_optstr, "encode") == 0) | ||
3692 | { | ||
3693 | #if XD3_ENCODER | ||
3694 | cmd = CMD_ENCODE; | ||
3695 | #else | ||
3696 | XPR(NT "encoder support not compiled\n"); | ||
3697 | return EXIT_FAILURE; | ||
3698 | #endif | ||
3699 | } | ||
3700 | else if (strcmp (my_optstr, "config") == 0) { cmd = CMD_CONFIG; } | ||
3701 | #if REGRESSION_TEST | ||
3702 | else if (strcmp (my_optstr, "test") == 0) { cmd = CMD_TEST; } | ||
3703 | #endif | ||
3704 | #if VCDIFF_TOOLS | ||
3705 | else if (strcmp (my_optstr, "printhdr") == 0) { cmd = CMD_PRINTHDR; } | ||
3706 | else if (strcmp (my_optstr, "printhdrs") == 0) | ||
3707 | { cmd = CMD_PRINTHDRS; } | ||
3708 | else if (strcmp (my_optstr, "printdelta") == 0) | ||
3709 | { cmd = CMD_PRINTDELTA; } | ||
3710 | else if (strcmp (my_optstr, "recode") == 0) { cmd = CMD_RECODE; } | ||
3711 | else if (strcmp (my_optstr, "merge") == 0) { cmd = CMD_MERGE; } | ||
3712 | #endif | ||
3713 | |||
3714 | /* If no option was found and still no command, let the default | ||
3715 | * command be encode. The remaining args are treated as | ||
3716 | * filenames. */ | ||
3717 | if (cmd == CMD_NONE) | ||
3718 | { | ||
3719 | cmd = CMD_DEFAULT; | ||
3720 | my_optstr = NULL; | ||
3721 | break; | ||
3722 | } | ||
3723 | else | ||
3724 | { | 582 | { |
3725 | /* But if we find a command name, continue the getopt loop. */ | 583 | XPR(NT "source block %"Q"u ejects %"Q"u (lru_hits=%u, " |
3726 | my_optind += 1; | 584 | "lru_misses=%u, lru_filled=%u)\n", |
3727 | goto takearg; | 585 | blkno, blru->blkno, lru_hits, lru_misses, lru_filled); |
3728 | } | ||
3729 | |||
3730 | /* gzip-like options */ | ||
3731 | case '0': case '1': case '2': case '3': case '4': | ||
3732 | case '5': case '6': case '7': case '8': case '9': | ||
3733 | option_level = ret - '0'; | ||
3734 | break; | ||
3735 | case 'f': option_force = 1; break; | ||
3736 | case 'F': | ||
3737 | #if EXTERNAL_COMPRESSION | ||
3738 | option_force2 = 1; | ||
3739 | #else | ||
3740 | XPR(NT "warning: -F option ignored, " | ||
3741 | "external compression support was not compiled\n"); | ||
3742 | break; | ||
3743 | #endif | ||
3744 | case 'v': option_verbose += 1; option_quiet = 0; break; | ||
3745 | case 'q': option_quiet = 1; option_verbose = 0; break; | ||
3746 | case 'c': option_stdout = 1; break; | ||
3747 | case 'd': | ||
3748 | if (cmd == CMD_NONE) { cmd = CMD_DECODE; } | ||
3749 | else { ret = main_help (); goto exit; } | ||
3750 | break; | ||
3751 | case 'e': | ||
3752 | #if XD3_ENCODER | ||
3753 | if (cmd == CMD_NONE) { cmd = CMD_ENCODE; } | ||
3754 | else { ret = main_help (); goto exit; } | ||
3755 | break; | ||
3756 | #else | ||
3757 | XPR(NT "encoder support not compiled\n"); | ||
3758 | return EXIT_FAILURE; | ||
3759 | #endif | ||
3760 | |||
3761 | case 'n': option_use_checksum = 0; break; | ||
3762 | case 'N': option_no_compress = 1; break; | ||
3763 | case 'T': option_use_altcodetable = 1; break; | ||
3764 | case 'C': option_smatch_config = my_optarg; break; | ||
3765 | case 'J': option_no_output = 1; break; | ||
3766 | case 'S': if (my_optarg == NULL) | ||
3767 | { | ||
3768 | option_use_secondary = 1; | ||
3769 | option_secondary = "none"; | ||
3770 | } | 586 | } |
3771 | else | 587 | else |
3772 | { | 588 | { |
3773 | option_use_secondary = 1; | 589 | XPR(NT "source block %"Q"u read (lru_hits=%u, " |
3774 | option_secondary = my_optarg; | 590 | "lru_misses=%u, lru_filled=%u)\n", |
3775 | } | 591 | blkno, lru_hits, lru_misses, lru_filled); |
3776 | break; | ||
3777 | case 'A': if (my_optarg == NULL) { option_use_appheader = 0; } | ||
3778 | else { option_appheader = (uint8_t*) my_optarg; } break; | ||
3779 | case 'B': | ||
3780 | if ((ret = main_atou (my_optarg, & option_srcwinsz, XD3_MINSRCWINSZ, | ||
3781 | 0, 'B'))) | ||
3782 | { | ||
3783 | goto exit; | ||
3784 | } | ||
3785 | break; | ||
3786 | case 'I': | ||
3787 | if ((ret = main_atou (my_optarg, & option_iopt_size, 0, | ||
3788 | 0, 'I'))) | ||
3789 | { | ||
3790 | goto exit; | ||
3791 | } | ||
3792 | break; | ||
3793 | case 'P': | ||
3794 | if ((ret = main_atou (my_optarg, & option_sprevsz, 0, | ||
3795 | 0, 'P'))) | ||
3796 | { | ||
3797 | goto exit; | ||
3798 | } | ||
3799 | break; | ||
3800 | case 'W': | ||
3801 | if ((ret = main_atou (my_optarg, & option_winsize, XD3_ALLOCSIZE, | ||
3802 | XD3_HARDMAXWINSIZE, 'W'))) | ||
3803 | { | ||
3804 | goto exit; | ||
3805 | } | ||
3806 | break; | ||
3807 | case 'D': | ||
3808 | #if EXTERNAL_COMPRESSION == 0 | ||
3809 | if (option_verbose > 0) | ||
3810 | { | ||
3811 | XPR(NT "warning: -D option ignored, " | ||
3812 | "external compression support was not compiled\n"); | ||
3813 | } | ||
3814 | #else | ||
3815 | option_decompress_inputs = 0; | ||
3816 | #endif | ||
3817 | break; | ||
3818 | case 'R': | ||
3819 | #if EXTERNAL_COMPRESSION == 0 | ||
3820 | if (option_verbose > 0) | ||
3821 | { | ||
3822 | XPR(NT "warning: -R option ignored, " | ||
3823 | "external compression support was not compiled\n"); | ||
3824 | } | ||
3825 | #else | ||
3826 | option_recompress_outputs = 0; | ||
3827 | #endif | ||
3828 | break; | ||
3829 | case 's': | ||
3830 | if (sfilename != NULL) | ||
3831 | { | ||
3832 | XPR(NT "specify only one source file\n"); | ||
3833 | goto cleanup; | ||
3834 | } | ||
3835 | |||
3836 | sfilename = my_optarg; | ||
3837 | break; | ||
3838 | case 'm': | ||
3839 | if ((merge = (main_merge*) | ||
3840 | main_malloc (sizeof (main_merge))) == NULL) | ||
3841 | { | ||
3842 | goto cleanup; | ||
3843 | } | 592 | } |
3844 | main_merge_list_push_back (& merge_order, merge); | ||
3845 | merge->filename = my_optarg; | ||
3846 | break; | ||
3847 | case 'V': | ||
3848 | ret = main_version (); goto exit; | ||
3849 | default: | ||
3850 | ret = main_help (); goto exit; | ||
3851 | } | ||
3852 | } | ||
3853 | |||
3854 | option_source_filename = sfilename; | ||
3855 | |||
3856 | /* In case there were no arguments, set the default command. */ | ||
3857 | if (cmd == CMD_NONE) { cmd = CMD_DEFAULT; } | ||
3858 | |||
3859 | argc -= my_optind; | ||
3860 | argv += my_optind; | ||
3861 | |||
3862 | /* There may be up to two more arguments. */ | ||
3863 | if (argc > 2) | ||
3864 | { | ||
3865 | XPR(NT "too many filenames: %s ...\n", argv[2]); | ||
3866 | goto cleanup; | ||
3867 | } | ||
3868 | |||
3869 | ifile.flags = RD_FIRST | RD_MAININPUT; | ||
3870 | sfile.flags = RD_FIRST; | ||
3871 | sfile.filename = option_source_filename; | ||
3872 | |||
3873 | /* The infile takes the next argument, if there is one. But if not, infile | ||
3874 | * is set to stdin. */ | ||
3875 | if (argc > 0) | ||
3876 | { | ||
3877 | ifile.filename = argv[0]; | ||
3878 | |||
3879 | if ((ret = main_file_open (& ifile, ifile.filename, XO_READ))) | ||
3880 | { | ||
3881 | goto cleanup; | ||
3882 | } | 593 | } |
3883 | } | 594 | else |
3884 | else | ||
3885 | { | ||
3886 | XSTDIN_XF (& ifile); | ||
3887 | } | ||
3888 | |||
3889 | /* The ofile takes the following argument, if there is one. But if not, it | ||
3890 | * is left NULL until the application header is processed. It will be set | ||
3891 | * in main_open_output. */ | ||
3892 | if (argc > 1) | ||
3893 | { | ||
3894 | /* Check for conflicting arguments. */ | ||
3895 | if (option_stdout && ! option_quiet) | ||
3896 | { | 595 | { |
3897 | XPR(NT "warning: -c option overrides output filename: %s\n", | 596 | XPR(NT "source block %"Q"u read (lru_hits=%u, lru_misses=%u, " |
3898 | argv[1]); | 597 | "lru_filled=%u)\n", blkno, lru_hits, lru_misses, lru_filled); |
3899 | } | 598 | } |
3900 | |||
3901 | if (! option_stdout) { ofile.filename = argv[1]; } | ||
3902 | } | ||
3903 | |||
3904 | #if VCDIFF_TOOLS | ||
3905 | if (cmd == CMD_MERGE && | ||
3906 | (ret = main_merge_arguments (&merge_order))) | ||
3907 | { | ||
3908 | goto cleanup; | ||
3909 | } | ||
3910 | #endif /* VCDIFF_TOOLS */ | ||
3911 | |||
3912 | switch (cmd) | ||
3913 | { | ||
3914 | case CMD_PRINTHDR: | ||
3915 | case CMD_PRINTHDRS: | ||
3916 | case CMD_PRINTDELTA: | ||
3917 | #if XD3_ENCODER | ||
3918 | case CMD_ENCODE: | ||
3919 | case CMD_RECODE: | ||
3920 | case CMD_MERGE: | ||
3921 | #endif | ||
3922 | case CMD_DECODE: | ||
3923 | ret = main_input (cmd, & ifile, & ofile, & sfile); | ||
3924 | break; | ||
3925 | |||
3926 | #if REGRESSION_TEST | ||
3927 | case CMD_TEST: | ||
3928 | main_config (); | ||
3929 | ret = xd3_selftest (); | ||
3930 | break; | ||
3931 | #endif | ||
3932 | |||
3933 | case CMD_CONFIG: | ||
3934 | ret = main_config (); | ||
3935 | break; | ||
3936 | |||
3937 | default: | ||
3938 | ret = main_help (); | ||
3939 | break; | ||
3940 | } | ||
3941 | |||
3942 | if (0) | ||
3943 | { | ||
3944 | cleanup: | ||
3945 | ret = EXIT_FAILURE; | ||
3946 | exit: | ||
3947 | (void)0; | ||
3948 | } | ||
3949 | |||
3950 | #if EXTERNAL_COMPRESSION | ||
3951 | if (ext_tmpfile != NULL) | ||
3952 | { | ||
3953 | unlink (ext_tmpfile); | ||
3954 | } | ||
3955 | #endif | ||
3956 | |||
3957 | main_file_cleanup (& ifile); | ||
3958 | main_file_cleanup (& ofile); | ||
3959 | main_file_cleanup (& sfile); | ||
3960 | |||
3961 | while (! main_merge_list_empty (& merge_order)) | ||
3962 | { | ||
3963 | merge = main_merge_list_pop_front (& merge_order); | ||
3964 | main_free (merge); | ||
3965 | } | 599 | } |
3966 | 600 | ||
3967 | main_free (free_argv); | 601 | source->curblk = blru->blk; |
3968 | main_free (free_value); | 602 | source->curblkno = blkno; |
3969 | 603 | source->onblk = nread; | |
3970 | main_cleanup (); | 604 | blru->size = nread; |
3971 | 605 | ||
3972 | fflush (stdout); | 606 | IF_DEBUG1 (DP(RINT "[main_getblk] blkno %"Q"u onblk %u pos %"Q"u " |
3973 | fflush (stderr); | 607 | "srcpos %"Q"u\n", |
3974 | return ret; | 608 | blkno, nread, pos, sfile->source_position)); |
3975 | } | ||
3976 | 609 | ||
3977 | static int | 610 | return 0; |
3978 | main_help (void) | ||
3979 | { | ||
3980 | main_version(); | ||
3981 | |||
3982 | /* Note: update wiki when command-line features change */ | ||
3983 | DP(RINT "usage: xdelta3 [command/options] [input [output]]\n"); | ||
3984 | DP(RINT "make patch:\n"); | ||
3985 | DP(RINT "\n"); | ||
3986 | DP(RINT " xdelta3.exe -e -s old_file new_file delta_file\n"); | ||
3987 | DP(RINT "\n"); | ||
3988 | DP(RINT "apply patch:\n"); | ||
3989 | DP(RINT "\n"); | ||
3990 | DP(RINT " xdelta3.exe -d -s old_file delta_file decoded_new_file\n"); | ||
3991 | DP(RINT "\n"); | ||
3992 | DP(RINT "special command names:\n"); | ||
3993 | DP(RINT " config prints xdelta3 configuration\n"); | ||
3994 | DP(RINT " decode decompress the input\n"); | ||
3995 | DP(RINT " encode compress the input%s\n", | ||
3996 | XD3_ENCODER ? "" : " [Not compiled]"); | ||
3997 | #if REGRESSION_TEST | ||
3998 | DP(RINT " test run the builtin tests\n"); | ||
3999 | #endif | ||
4000 | #if VCDIFF_TOOLS | ||
4001 | DP(RINT "special commands for VCDIFF inputs:\n"); | ||
4002 | DP(RINT " printdelta print information about the entire delta\n"); | ||
4003 | DP(RINT " printhdr print information about the first window\n"); | ||
4004 | DP(RINT " printhdrs print information about all windows\n"); | ||
4005 | DP(RINT " recode encode with new application/secondary settings\n"); | ||
4006 | DP(RINT " merge merge VCDIFF inputs (see below)\n"); | ||
4007 | #endif | ||
4008 | DP(RINT "merge patches:\n"); | ||
4009 | DP(RINT "\n"); | ||
4010 | DP(RINT " xdelta3 merge -m 1.vcdiff -m 2.vcdiff 3.vcdiff merged.vcdiff\n"); | ||
4011 | DP(RINT "\n"); | ||
4012 | DP(RINT "standard options:\n"); | ||
4013 | DP(RINT " -0 .. -9 compression level\n"); | ||
4014 | DP(RINT " -c use stdout\n"); | ||
4015 | DP(RINT " -d decompress\n"); | ||
4016 | DP(RINT " -e compress%s\n", | ||
4017 | XD3_ENCODER ? "" : " [Not compiled]"); | ||
4018 | DP(RINT " -f force (overwrite, ignore trailing garbage)\n"); | ||
4019 | #if EXTERNAL_COMPRESSION | ||
4020 | DP(RINT " -F force the external-compression subprocess\n"); | ||
4021 | #endif | ||
4022 | DP(RINT " -h show help\n"); | ||
4023 | DP(RINT " -q be quiet\n"); | ||
4024 | DP(RINT " -v be verbose (max 2)\n"); | ||
4025 | DP(RINT " -V show version\n"); | ||
4026 | |||
4027 | DP(RINT "memory options:\n"); | ||
4028 | DP(RINT " -B bytes source window size\n"); | ||
4029 | DP(RINT " -W bytes input window size\n"); | ||
4030 | DP(RINT " -P size compression duplicates window\n"); | ||
4031 | DP(RINT " -I size instruction buffer size (0 = unlimited)\n"); | ||
4032 | |||
4033 | DP(RINT "compression options:\n"); | ||
4034 | DP(RINT " -s source source file to copy from (if any)\n"); | ||
4035 | DP(RINT " -S [djw|fgk] enable/disable secondary compression\n"); | ||
4036 | DP(RINT " -N disable small string-matching compression\n"); | ||
4037 | DP(RINT " -D disable external decompression (encode/decode)\n"); | ||
4038 | DP(RINT " -R disable external recompression (decode)\n"); | ||
4039 | DP(RINT " -n disable checksum (encode/decode)\n"); | ||
4040 | DP(RINT " -C soft config (encode, undocumented)\n"); | ||
4041 | DP(RINT " -A [apphead] disable/provide application header (encode)\n"); | ||
4042 | DP(RINT " -J disable output (check/compute only)\n"); | ||
4043 | DP(RINT " -T use alternate code table (test)\n"); | ||
4044 | DP(RINT " -m arguments for \"merge\"\n"); | ||
4045 | |||
4046 | DP(RINT "the XDELTA environment variable may contain extra args:\n"); | ||
4047 | DP(RINT " XDELTA=\"-s source-x.y.tar.gz\" \\\n"); | ||
4048 | DP(RINT " tar --use-compress-program=xdelta3 \\\n"); | ||
4049 | DP(RINT " -cf target-x.z.tar.gz.vcdiff target-x.y\n"); | ||
4050 | return EXIT_FAILURE; | ||
4051 | } | 611 | } |
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h index ff6ade6..e47d708 100644 --- a/xdelta3/xdelta3-main.h +++ b/xdelta3/xdelta3-main.h | |||
@@ -192,8 +192,6 @@ typedef enum | |||
192 | 192 | ||
193 | typedef struct _main_file main_file; | 193 | typedef struct _main_file main_file; |
194 | typedef struct _main_extcomp main_extcomp; | 194 | typedef struct _main_extcomp main_extcomp; |
195 | typedef struct _main_blklru main_blklru; | ||
196 | typedef struct _main_blklru_list main_blklru_list; | ||
197 | typedef struct _main_merge main_merge; | 195 | typedef struct _main_merge main_merge; |
198 | typedef struct _main_merge_list main_merge_list; | 196 | typedef struct _main_merge_list main_merge_list; |
199 | 197 | ||
@@ -246,29 +244,6 @@ struct _main_extcomp | |||
246 | int flags; | 244 | int flags; |
247 | }; | 245 | }; |
248 | 246 | ||
249 | /* This file implements a small LRU of source blocks. For encoding purposes, | ||
250 | * we prevent paging in blocks we've already scanned in the source (return | ||
251 | * XD3_NOTAVAIL). */ | ||
252 | struct _main_blklru_list | ||
253 | { | ||
254 | main_blklru_list *next; | ||
255 | main_blklru_list *prev; | ||
256 | }; | ||
257 | |||
258 | struct _main_blklru | ||
259 | { | ||
260 | uint8_t *blk; | ||
261 | xoff_t blkno; | ||
262 | usize_t size; | ||
263 | main_blklru_list link; | ||
264 | }; | ||
265 | |||
266 | #define MAX_LRU_SIZE 32U | ||
267 | #define XD3_MINSRCWINSZ XD3_ALLOCSIZE | ||
268 | |||
269 | /* ... represented as a list (no cache index). */ | ||
270 | XD3_MAKELIST(main_blklru_list,main_blklru,link); | ||
271 | |||
272 | /* Merge state: */ | 247 | /* Merge state: */ |
273 | 248 | ||
274 | struct _main_merge_list | 249 | struct _main_merge_list |
@@ -336,17 +311,6 @@ static uint8_t* appheader_used = NULL; | |||
336 | static uint8_t* main_bdata = NULL; | 311 | static uint8_t* main_bdata = NULL; |
337 | static usize_t main_bsize = 0; | 312 | static usize_t main_bsize = 0; |
338 | 313 | ||
339 | /* The LRU: obviously this is shared by all callers. */ | ||
340 | static usize_t lru_size = 0; | ||
341 | static main_blklru *lru = NULL; /* array of lru_size elts */ | ||
342 | static main_blklru_list lru_list; | ||
343 | static main_blklru_list lru_free; | ||
344 | static int do_src_fifo = 0; /* set to avoid lru */ | ||
345 | |||
346 | static int lru_hits = 0; | ||
347 | static int lru_misses = 0; | ||
348 | static int lru_filled = 0; | ||
349 | |||
350 | /* Hacks for VCDIFF tools */ | 314 | /* Hacks for VCDIFF tools */ |
351 | static int allow_fake_source = 0; | 315 | static int allow_fake_source = 0; |
352 | 316 | ||
@@ -379,8 +343,24 @@ static void main_get_appheader (xd3_stream *stream, main_file *ifile, | |||
379 | static int main_getblk_func (xd3_stream *stream, | 343 | static int main_getblk_func (xd3_stream *stream, |
380 | xd3_source *source, | 344 | xd3_source *source, |
381 | xoff_t blkno); | 345 | xoff_t blkno); |
346 | static void main_free (void *ptr); | ||
347 | static void* main_malloc (usize_t size); | ||
348 | |||
349 | static int main_file_open (main_file *xfile, const char* name, int mode); | ||
350 | static int main_file_stat (main_file *xfile, xoff_t *size); | ||
351 | static int main_file_seek (main_file *xfile, xoff_t pos); | ||
352 | static int main_read_primary_input (main_file *file, | ||
353 | uint8_t *buf, | ||
354 | usize_t size, | ||
355 | usize_t *nread); | ||
356 | |||
357 | static const char* main_format_bcnt (xoff_t r, char *buf); | ||
382 | static int main_help (void); | 358 | static int main_help (void); |
383 | 359 | ||
360 | /* The code in xdelta3-blk.h is essentially part of this unit, see | ||
361 | * comments there. */ | ||
362 | #include "xdelta3-blkcache.h" | ||
363 | |||
384 | static int | 364 | static int |
385 | main_version (void) | 365 | main_version (void) |
386 | { | 366 | { |
@@ -418,12 +398,12 @@ main_config (void) | |||
418 | DP(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ); | 398 | DP(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ); |
419 | DP(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE); | 399 | DP(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE); |
420 | DP(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE); | 400 | DP(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE); |
421 | DP(RINT "sizeof(void*)=%d\n", sizeof(void*)); | 401 | DP(RINT "sizeof(void*)=%d\n", (int)sizeof(void*)); |
422 | DP(RINT "sizeof(int)=%d\n", sizeof(int)); | 402 | DP(RINT "sizeof(int)=%d\n", (int)sizeof(int)); |
423 | DP(RINT "sizeof(uint32_t)=%d\n", sizeof(uint32_t)); | 403 | DP(RINT "sizeof(uint32_t)=%d\n", (int)sizeof(uint32_t)); |
424 | DP(RINT "sizeof(uint64_t)=%d\n", sizeof(uint64_t)); | 404 | DP(RINT "sizeof(uint64_t)=%d\n", (int)sizeof(uint64_t)); |
425 | DP(RINT "sizeof(usize_t)=%d\n", sizeof(usize_t)); | 405 | DP(RINT "sizeof(usize_t)=%d\n", (int)sizeof(usize_t)); |
426 | DP(RINT "sizeof(xoff_t)=%d\n", sizeof(xoff_t)); | 406 | DP(RINT "sizeof(xoff_t)=%d\n", (int)sizeof(xoff_t)); |
427 | 407 | ||
428 | return EXIT_SUCCESS; | 408 | return EXIT_SUCCESS; |
429 | } | 409 | } |
@@ -447,15 +427,11 @@ reset_defaults(void) | |||
447 | appheader_used = NULL; | 427 | appheader_used = NULL; |
448 | main_bdata = NULL; | 428 | main_bdata = NULL; |
449 | main_bsize = 0; | 429 | main_bsize = 0; |
450 | lru_size = 0; | ||
451 | lru = NULL; | ||
452 | do_src_fifo = 0; | ||
453 | lru_hits = 0; | ||
454 | lru_misses = 0; | ||
455 | lru_filled = 0; | ||
456 | allow_fake_source = 0; | 430 | allow_fake_source = 0; |
457 | option_smatch_config = NULL; | 431 | option_smatch_config = NULL; |
458 | 432 | ||
433 | main_lru_reset(); | ||
434 | |||
459 | option_use_appheader = 1; | 435 | option_use_appheader = 1; |
460 | option_use_checksum = 1; | 436 | option_use_checksum = 1; |
461 | #if EXTERNAL_COMPRESSION | 437 | #if EXTERNAL_COMPRESSION |
@@ -531,7 +507,7 @@ get_errno (void) | |||
531 | return errno; | 507 | return errno; |
532 | #else | 508 | #else |
533 | DWORD err_num = GetLastError(); | 509 | DWORD err_num = GetLastError(); |
534 | if (err_num == NO_ERROR) | 510 | if (err_num == NO_ERROR) |
535 | { | 511 | { |
536 | err_num = XD3_INTERNAL; | 512 | err_num = XD3_INTERNAL; |
537 | } | 513 | } |
@@ -543,7 +519,7 @@ const char* | |||
543 | xd3_mainerror(int err_num) { | 519 | xd3_mainerror(int err_num) { |
544 | #ifndef _WIN32 | 520 | #ifndef _WIN32 |
545 | const char* x = xd3_strerror (err_num); | 521 | const char* x = xd3_strerror (err_num); |
546 | if (x != NULL) | 522 | if (x != NULL) |
547 | { | 523 | { |
548 | return x; | 524 | return x; |
549 | } | 525 | } |
@@ -894,12 +870,12 @@ main_file_open (main_file *xfile, const char* name, int mode) | |||
894 | (mode == XO_READ) ? GENERIC_READ : GENERIC_WRITE, | 870 | (mode == XO_READ) ? GENERIC_READ : GENERIC_WRITE, |
895 | FILE_SHARE_READ, | 871 | FILE_SHARE_READ, |
896 | NULL, | 872 | NULL, |
897 | (mode == XO_READ) ? | 873 | (mode == XO_READ) ? |
898 | OPEN_EXISTING : | 874 | OPEN_EXISTING : |
899 | (option_force ? CREATE_ALWAYS : CREATE_NEW), | 875 | (option_force ? CREATE_ALWAYS : CREATE_NEW), |
900 | FILE_ATTRIBUTE_NORMAL, | 876 | FILE_ATTRIBUTE_NORMAL, |
901 | NULL); | 877 | NULL); |
902 | if (xfile->file == INVALID_HANDLE_VALUE) | 878 | if (xfile->file == INVALID_HANDLE_VALUE) |
903 | { | 879 | { |
904 | ret = get_errno (); | 880 | ret = get_errno (); |
905 | } | 881 | } |
@@ -1126,7 +1102,7 @@ main_file_seek (main_file *xfile, xoff_t pos) | |||
1126 | # if (_WIN32_WINNT >= 0x0500) | 1102 | # if (_WIN32_WINNT >= 0x0500) |
1127 | LARGE_INTEGER move, out; | 1103 | LARGE_INTEGER move, out; |
1128 | move.QuadPart = pos; | 1104 | move.QuadPart = pos; |
1129 | if (SetFilePointerEx(xfile->file, move, &out, FILE_BEGIN) == 0) | 1105 | if (SetFilePointerEx(xfile->file, move, &out, FILE_BEGIN) == 0) |
1130 | { | 1106 | { |
1131 | ret = get_errno (); | 1107 | ret = get_errno (); |
1132 | } | 1108 | } |
@@ -2500,17 +2476,17 @@ main_secondary_decompress_check (main_file *file, | |||
2500 | { | 2476 | { |
2501 | XPR(NT | 2477 | XPR(NT |
2502 | "WARNING: the encoder is automatically decompressing the input file;\n"); | 2478 | "WARNING: the encoder is automatically decompressing the input file;\n"); |
2503 | XPR(NT | 2479 | XPR(NT |
2504 | "WARNING: the decoder will automatically recompress the output file;\n"); | 2480 | "WARNING: the decoder will automatically recompress the output file;\n"); |
2505 | XPR(NT | 2481 | XPR(NT |
2506 | "WARNING: this may result in different compressed data and checksums\n"); | 2482 | "WARNING: this may result in different compressed data and checksums\n"); |
2507 | XPR(NT | 2483 | XPR(NT |
2508 | "WARNING: despite being identical data; if this is an issue, use -D\n"); | 2484 | "WARNING: despite being identical data; if this is an issue, use -D\n"); |
2509 | XPR(NT | 2485 | XPR(NT |
2510 | "WARNING: to avoid decompression and/or use -R to avoid recompression\n"); | 2486 | "WARNING: to avoid decompression and/or use -R to avoid recompression\n"); |
2511 | XPR(NT | 2487 | XPR(NT |
2512 | "WARNING: and/or manually decompress the input file; if you know the\n"); | 2488 | "WARNING: and/or manually decompress the input file; if you know the\n"); |
2513 | XPR(NT | 2489 | XPR(NT |
2514 | "WARNING: compression settings that will produce identical output\n"); | 2490 | "WARNING: compression settings that will produce identical output\n"); |
2515 | XPR(NT | 2491 | XPR(NT |
2516 | "WARNING: you may set those flags using the environment (e.g., GZIP=-9)\n"); | 2492 | "WARNING: you may set those flags using the environment (e.g., GZIP=-9)\n"); |
@@ -2755,7 +2731,7 @@ main_set_appheader (xd3_stream *stream, main_file *input, main_file *sfile) | |||
2755 | } | 2731 | } |
2756 | } | 2732 | } |
2757 | 2733 | ||
2758 | xd3_set_appheader (stream, appheader_used, | 2734 | xd3_set_appheader (stream, appheader_used, |
2759 | (usize_t) strlen ((char*)appheader_used)); | 2735 | (usize_t) strlen ((char*)appheader_used)); |
2760 | 2736 | ||
2761 | return 0; | 2737 | return 0; |
@@ -2957,234 +2933,6 @@ main_open_output (xd3_stream *stream, main_file *ofile) | |||
2957 | return 0; | 2933 | return 0; |
2958 | } | 2934 | } |
2959 | 2935 | ||
2960 | /* This is called at different times for encoding and decoding. The | ||
2961 | * encoder calls it immediately, the decoder delays until the | ||
2962 | * application header is received. */ | ||
2963 | static int | ||
2964 | main_set_source (xd3_stream *stream, xd3_cmd cmd, | ||
2965 | main_file *sfile, xd3_source *source) | ||
2966 | { | ||
2967 | int ret = 0; | ||
2968 | usize_t i; | ||
2969 | usize_t blksize; | ||
2970 | xoff_t source_size = 0; | ||
2971 | main_blklru block0; | ||
2972 | |||
2973 | XD3_ASSERT (lru == NULL); | ||
2974 | XD3_ASSERT (stream->src == NULL); | ||
2975 | XD3_ASSERT (option_srcwinsz >= XD3_MINSRCWINSZ); | ||
2976 | |||
2977 | main_blklru_list_init (& lru_list); | ||
2978 | main_blklru_list_init (& lru_free); | ||
2979 | |||
2980 | if (allow_fake_source) | ||
2981 | { | ||
2982 | sfile->mode = XO_READ; | ||
2983 | sfile->realname = sfile->filename; | ||
2984 | sfile->nread = 0; | ||
2985 | } | ||
2986 | else | ||
2987 | { | ||
2988 | if ((ret = main_file_open (sfile, sfile->filename, XO_READ))) | ||
2989 | { | ||
2990 | return ret; | ||
2991 | } | ||
2992 | |||
2993 | /* Allow non-seekable sources from the start. If the file | ||
2994 | * turns out to be externally compressed, size_known may change. */ | ||
2995 | sfile->size_known = (main_file_stat (sfile, &source_size) == 0); | ||
2996 | } | ||
2997 | |||
2998 | /* The API requires power-of-two blocksize, */ | ||
2999 | blksize = xd3_pow2_roundup (max (option_srcwinsz / MAX_LRU_SIZE, | ||
3000 | XD3_MINSRCWINSZ)); | ||
3001 | |||
3002 | /* TODO(jmacd): The organization of this code and the implementation | ||
3003 | * of the LRU cache could be improved. This is too convoluted, the | ||
3004 | * code uses main_getblk_func() to read the first block, which may | ||
3005 | * trigger external-decompression, in large part so that the | ||
3006 | * verbose-printing and counters maintained by getblk_func are | ||
3007 | * consistent. There used to be additional optimizations going on | ||
3008 | * here: (1) if the source size is known we would like to lower | ||
3009 | * option_srcwinsz, if possible, (2) avoid allocating more memory | ||
3010 | * than needed, and (3) also want to use a single block when | ||
3011 | * source_size < option_srcwinsz, this because compression is better | ||
3012 | * for larger blocks (especially due to the blockwise-reverse | ||
3013 | * insertion of checksums). | ||
3014 | * | ||
3015 | * (3) is no longer implemented. (2) is only implemented for files | ||
3016 | * that are shorter than the default blocksize, and (1) is not | ||
3017 | * implemented. These optimizations are not taken because the code | ||
3018 | * is already too complicated. | ||
3019 | * | ||
3020 | * The ideal solution may be to allocate a block of memory equal to | ||
3021 | * half of the option_srcwinsz. Read as much data as possible into | ||
3022 | * that block. If the entire file fits, pass one block to the | ||
3023 | * library for best compression. If not, copy the 50% block into | ||
3024 | * smaller (option_srcwinsz / MAX_LRU_SIZE) blocks and proceed. Too | ||
3025 | * much effort for too little payback. */ | ||
3026 | |||
3027 | memset (&block0, 0, sizeof (block0)); | ||
3028 | block0.blkno = (xoff_t) -1; | ||
3029 | |||
3030 | /* Allocate the first block. Even if the size is known at this | ||
3031 | * point, we do not lower it because decompression may happen. */ | ||
3032 | if ((block0.blk = (uint8_t*) main_malloc (blksize)) == NULL) | ||
3033 | { | ||
3034 | ret = ENOMEM; | ||
3035 | return ret; | ||
3036 | } | ||
3037 | |||
3038 | source->blksize = blksize; | ||
3039 | source->name = sfile->filename; | ||
3040 | source->ioh = sfile; | ||
3041 | source->curblkno = (xoff_t) -1; | ||
3042 | source->curblk = NULL; | ||
3043 | |||
3044 | /* We have to read the first block into the cache now, because | ||
3045 | * size_known can still change (due to secondary decompression). | ||
3046 | * Calls main_secondary_decompress_check() via | ||
3047 | * main_read_primary_input(). */ | ||
3048 | lru_size = 1; | ||
3049 | lru = &block0; | ||
3050 | XD3_ASSERT (main_blklru_list_empty (& lru_free)); | ||
3051 | XD3_ASSERT (main_blklru_list_empty (& lru_list)); | ||
3052 | main_blklru_list_push_back (& lru_free, & lru[0]); | ||
3053 | /* This call is partly so that the diagnostics printed by | ||
3054 | * this function appear to happen normally for the first read, | ||
3055 | * which is special. */ | ||
3056 | ret = main_getblk_func (stream, source, 0); | ||
3057 | main_blklru_list_remove (& lru[0]); | ||
3058 | XD3_ASSERT (main_blklru_list_empty (& lru_free)); | ||
3059 | XD3_ASSERT (main_blklru_list_empty (& lru_list)); | ||
3060 | lru = NULL; | ||
3061 | |||
3062 | if (ret != 0) | ||
3063 | { | ||
3064 | main_free (block0.blk); | ||
3065 | |||
3066 | XPR(NT "error reading source: %s: %s\n", | ||
3067 | sfile->filename, | ||
3068 | xd3_mainerror (ret)); | ||
3069 | |||
3070 | return ret; | ||
3071 | } | ||
3072 | |||
3073 | source->onblk = block0.size; | ||
3074 | |||
3075 | /* If the file is smaller than a block, size is known. */ | ||
3076 | if (!sfile->size_known && source->onblk < blksize) | ||
3077 | { | ||
3078 | source_size = source->onblk; | ||
3079 | sfile->size_known = 1; | ||
3080 | } | ||
3081 | |||
3082 | /* We update lru_size accordingly, and modify option_srcwinsz, which | ||
3083 | * will be passed via xd3_config. */ | ||
3084 | if (sfile->size_known && source_size <= option_srcwinsz) | ||
3085 | { | ||
3086 | lru_size = (usize_t) ((source_size + blksize - 1) / blksize); | ||
3087 | } | ||
3088 | else | ||
3089 | { | ||
3090 | lru_size = (option_srcwinsz + blksize - 1) / blksize; | ||
3091 | } | ||
3092 | |||
3093 | XD3_ASSERT (lru_size >= 1); | ||
3094 | option_srcwinsz = lru_size * blksize; | ||
3095 | |||
3096 | if ((lru = (main_blklru*) main_malloc (lru_size * sizeof (main_blklru))) | ||
3097 | == NULL) | ||
3098 | { | ||
3099 | ret = ENOMEM; | ||
3100 | return ret; | ||
3101 | } | ||
3102 | |||
3103 | lru[0].blk = block0.blk; | ||
3104 | lru[0].blkno = 0; | ||
3105 | lru[0].size = source->onblk; | ||
3106 | |||
3107 | if (! sfile->size_known) | ||
3108 | { | ||
3109 | do_src_fifo = 1; | ||
3110 | } | ||
3111 | else if (! do_src_fifo) | ||
3112 | { | ||
3113 | main_blklru_list_push_back (& lru_list, & lru[0]); | ||
3114 | } | ||
3115 | |||
3116 | for (i = 1; i < lru_size; i += 1) | ||
3117 | { | ||
3118 | lru[i].blkno = (xoff_t) -1; | ||
3119 | |||
3120 | if ((lru[i].blk = (uint8_t*) main_malloc (source->blksize)) == NULL) | ||
3121 | { | ||
3122 | ret = ENOMEM; | ||
3123 | return ret; | ||
3124 | } | ||
3125 | |||
3126 | if (! do_src_fifo) | ||
3127 | { | ||
3128 | main_blklru_list_push_back (& lru_free, & lru[i]); | ||
3129 | } | ||
3130 | } | ||
3131 | |||
3132 | if (sfile->size_known) | ||
3133 | { | ||
3134 | ret = xd3_set_source_and_size (stream, source, source_size); | ||
3135 | } | ||
3136 | else | ||
3137 | { | ||
3138 | ret = xd3_set_source (stream, source); | ||
3139 | } | ||
3140 | |||
3141 | if (ret) | ||
3142 | { | ||
3143 | XPR(NT XD3_LIB_ERRMSG (stream, ret)); | ||
3144 | return ret; | ||
3145 | } | ||
3146 | |||
3147 | XD3_ASSERT (stream->src == source); | ||
3148 | XD3_ASSERT (source->blksize == blksize); | ||
3149 | |||
3150 | if (option_verbose) | ||
3151 | { | ||
3152 | static char srcszbuf[32]; | ||
3153 | static char srccntbuf[32]; | ||
3154 | static char winszbuf[32]; | ||
3155 | static char blkszbuf[32]; | ||
3156 | static char nbufs[32]; | ||
3157 | |||
3158 | if (sfile->size_known) | ||
3159 | { | ||
3160 | sprintf (srcszbuf, "source size %s [%"Q"u]", | ||
3161 | main_format_bcnt (source_size, srccntbuf), | ||
3162 | source_size); | ||
3163 | } | ||
3164 | else | ||
3165 | { | ||
3166 | strcpy(srcszbuf, "source size unknown"); | ||
3167 | } | ||
3168 | |||
3169 | nbufs[0] = 0; | ||
3170 | |||
3171 | if (option_verbose > 1) | ||
3172 | { | ||
3173 | sprintf(nbufs, " #bufs %u", lru_size); | ||
3174 | } | ||
3175 | |||
3176 | XPR(NT "source %s %s blksize %s window %s%s%s\n", | ||
3177 | sfile->filename, | ||
3178 | srcszbuf, | ||
3179 | main_format_bcnt (blksize, blkszbuf), | ||
3180 | main_format_bcnt (option_srcwinsz, winszbuf), | ||
3181 | nbufs, | ||
3182 | do_src_fifo ? " (FIFO)" : ""); | ||
3183 | } | ||
3184 | |||
3185 | return 0; | ||
3186 | } | ||
3187 | |||
3188 | static usize_t | 2936 | static usize_t |
3189 | main_get_winsize (main_file *ifile) { | 2937 | main_get_winsize (main_file *ifile) { |
3190 | xoff_t file_size = 0; | 2938 | xoff_t file_size = 0; |
@@ -3208,310 +2956,6 @@ main_get_winsize (main_file *ifile) { | |||
3208 | return size; | 2956 | return size; |
3209 | } | 2957 | } |
3210 | 2958 | ||
3211 | /******************************************************************* | ||
3212 | Source routines | ||
3213 | *******************************************************************/ | ||
3214 | |||
3215 | static int | ||
3216 | main_getblk_lru (xd3_source *source, xoff_t blkno, | ||
3217 | main_blklru** blrup, int *is_new) | ||
3218 | { | ||
3219 | main_blklru *blru = NULL; | ||
3220 | usize_t i; | ||
3221 | |||
3222 | (*is_new) = 0; | ||
3223 | |||
3224 | if (do_src_fifo) | ||
3225 | { | ||
3226 | /* Direct lookup assumes sequential scan w/o skipping blocks. */ | ||
3227 | int idx = blkno % lru_size; | ||
3228 | blru = & lru[idx]; | ||
3229 | if (blru->blkno == blkno) | ||
3230 | { | ||
3231 | (*blrup) = blru; | ||
3232 | return 0; | ||
3233 | } | ||
3234 | |||
3235 | if (blru->blkno != (xoff_t)-1 && | ||
3236 | blru->blkno != (xoff_t)(blkno - lru_size)) | ||
3237 | { | ||
3238 | return XD3_TOOFARBACK; | ||
3239 | } | ||
3240 | } | ||
3241 | else | ||
3242 | { | ||
3243 | /* Sequential search through LRU. */ | ||
3244 | for (i = 0; i < lru_size; i += 1) | ||
3245 | { | ||
3246 | blru = & lru[i]; | ||
3247 | if (blru->blkno == blkno) | ||
3248 | { | ||
3249 | main_blklru_list_remove (blru); | ||
3250 | main_blklru_list_push_back (& lru_list, blru); | ||
3251 | (*blrup) = blru; | ||
3252 | return 0; | ||
3253 | } | ||
3254 | } | ||
3255 | } | ||
3256 | |||
3257 | if (do_src_fifo) | ||
3258 | { | ||
3259 | int idx = blkno % lru_size; | ||
3260 | blru = & lru[idx]; | ||
3261 | } | ||
3262 | else | ||
3263 | { | ||
3264 | if (! main_blklru_list_empty (& lru_free)) | ||
3265 | { | ||
3266 | blru = main_blklru_list_pop_front (& lru_free); | ||
3267 | main_blklru_list_push_back (& lru_list, blru); | ||
3268 | } | ||
3269 | else | ||
3270 | { | ||
3271 | XD3_ASSERT (! main_blklru_list_empty (& lru_list)); | ||
3272 | blru = main_blklru_list_pop_front (& lru_list); | ||
3273 | main_blklru_list_push_back (& lru_list, blru); | ||
3274 | } | ||
3275 | } | ||
3276 | |||
3277 | lru_filled += 1; | ||
3278 | (*is_new) = 1; | ||
3279 | (*blrup) = blru; | ||
3280 | blru->blkno = blkno; | ||
3281 | return 0; | ||
3282 | } | ||
3283 | |||
3284 | static int | ||
3285 | main_read_seek_source (xd3_stream *stream, | ||
3286 | xd3_source *source, | ||
3287 | xoff_t blkno) { | ||
3288 | xoff_t pos = blkno * source->blksize; | ||
3289 | main_file *sfile = (main_file*) source->ioh; | ||
3290 | main_blklru *blru; | ||
3291 | int is_new; | ||
3292 | usize_t nread = 0; | ||
3293 | int ret = 0; | ||
3294 | |||
3295 | if (!sfile->seek_failed) | ||
3296 | { | ||
3297 | ret = main_file_seek (sfile, pos); | ||
3298 | |||
3299 | if (ret == 0) | ||
3300 | { | ||
3301 | sfile->source_position = pos; | ||
3302 | } | ||
3303 | } | ||
3304 | |||
3305 | if (sfile->seek_failed || ret != 0) | ||
3306 | { | ||
3307 | /* For an unseekable file (or other seek error, does it | ||
3308 | * matter?) */ | ||
3309 | if (sfile->source_position > pos) | ||
3310 | { | ||
3311 | /* Could assert !IS_ENCODE(), this shouldn't happen | ||
3312 | * because of do_src_fifo during encode. */ | ||
3313 | if (!option_quiet) | ||
3314 | { | ||
3315 | XPR(NT "source can't seek backwards; requested block offset " | ||
3316 | "%"Q"u source position is %"Q"u\n", | ||
3317 | pos, sfile->source_position); | ||
3318 | } | ||
3319 | |||
3320 | sfile->seek_failed = 1; | ||
3321 | stream->msg = "non-seekable source: " | ||
3322 | "copy is too far back (try raising -B)"; | ||
3323 | return XD3_TOOFARBACK; | ||
3324 | } | ||
3325 | |||
3326 | /* There's a chance here, that an genuine lseek error will cause | ||
3327 | * xdelta3 to shift into non-seekable mode, entering a degraded | ||
3328 | * condition. */ | ||
3329 | if (!sfile->seek_failed && option_verbose) | ||
3330 | { | ||
3331 | XPR(NT "source can't seek, will use FIFO for %s\n", | ||
3332 | sfile->filename); | ||
3333 | |||
3334 | if (option_verbose > 1) | ||
3335 | { | ||
3336 | XPR(NT "seek error at offset %"Q"u: %s\n", | ||
3337 | pos, xd3_mainerror (ret)); | ||
3338 | } | ||
3339 | } | ||
3340 | |||
3341 | sfile->seek_failed = 1; | ||
3342 | |||
3343 | while (sfile->source_position < pos) | ||
3344 | { | ||
3345 | xoff_t skip_blkno; | ||
3346 | usize_t skip_offset; | ||
3347 | |||
3348 | xd3_blksize_div (sfile->source_position, source, | ||
3349 | &skip_blkno, &skip_offset); | ||
3350 | |||
3351 | /* Read past unused data */ | ||
3352 | XD3_ASSERT (pos - sfile->source_position >= source->blksize); | ||
3353 | XD3_ASSERT (skip_offset == 0); | ||
3354 | |||
3355 | if ((ret = main_getblk_lru (source, skip_blkno, | ||
3356 | & blru, & is_new))) | ||
3357 | { | ||
3358 | return ret; | ||
3359 | } | ||
3360 | |||
3361 | XD3_ASSERT (is_new); | ||
3362 | |||
3363 | if (option_verbose > 1) | ||
3364 | { | ||
3365 | XPR(NT "non-seekable source skipping %"Q"u bytes @ %"Q"u\n", | ||
3366 | pos - sfile->source_position, | ||
3367 | sfile->source_position); | ||
3368 | } | ||
3369 | |||
3370 | if ((ret = main_read_primary_input (sfile, | ||
3371 | (uint8_t*) blru->blk, | ||
3372 | source->blksize, | ||
3373 | & nread))) | ||
3374 | { | ||
3375 | return ret; | ||
3376 | } | ||
3377 | |||
3378 | if (nread != source->blksize) | ||
3379 | { | ||
3380 | IF_DEBUG1 (DP(RINT "[getblk] short skip block nread = %u\n", | ||
3381 | nread)); | ||
3382 | stream->msg = "non-seekable input is short"; | ||
3383 | return XD3_INVALID_INPUT; | ||
3384 | } | ||
3385 | |||
3386 | sfile->source_position += nread; | ||
3387 | blru->size = nread; | ||
3388 | |||
3389 | IF_DEBUG1 (DP(RINT "[getblk] skip blkno %"Q"u size %u\n", | ||
3390 | skip_blkno, blru->size)); | ||
3391 | |||
3392 | XD3_ASSERT (sfile->source_position <= pos); | ||
3393 | } | ||
3394 | } | ||
3395 | |||
3396 | return 0; | ||
3397 | } | ||
3398 | |||
3399 | /* This is the callback for reading a block of source. This function | ||
3400 | * is blocking and it implements a small LRU. | ||
3401 | * | ||
3402 | * Note that it is possible for main_input() to handle getblk requests | ||
3403 | * in a non-blocking manner. If the callback is NULL then the caller | ||
3404 | * of xd3_*_input() must handle the XD3_GETSRCBLK return value and | ||
3405 | * fill the source in the same way. See xd3_getblk for details. To | ||
3406 | * see an example of non-blocking getblk, see xdelta-test.h. */ | ||
3407 | static int | ||
3408 | main_getblk_func (xd3_stream *stream, | ||
3409 | xd3_source *source, | ||
3410 | xoff_t blkno) | ||
3411 | { | ||
3412 | int ret = 0; | ||
3413 | xoff_t pos = blkno * source->blksize; | ||
3414 | main_file *sfile = (main_file*) source->ioh; | ||
3415 | main_blklru *blru; | ||
3416 | int is_new; | ||
3417 | int did_seek = 0; | ||
3418 | usize_t nread = 0; | ||
3419 | |||
3420 | if (allow_fake_source) | ||
3421 | { | ||
3422 | source->curblkno = blkno; | ||
3423 | source->onblk = 0; | ||
3424 | source->curblk = lru[0].blk; | ||
3425 | lru[0].size = 0; | ||
3426 | return 0; | ||
3427 | } | ||
3428 | |||
3429 | if ((ret = main_getblk_lru (source, blkno, & blru, & is_new))) | ||
3430 | { | ||
3431 | return ret; | ||
3432 | } | ||
3433 | |||
3434 | if (!is_new) | ||
3435 | { | ||
3436 | source->curblkno = blkno; | ||
3437 | source->onblk = blru->size; | ||
3438 | source->curblk = blru->blk; | ||
3439 | lru_hits++; | ||
3440 | return 0; | ||
3441 | } | ||
3442 | |||
3443 | lru_misses += 1; | ||
3444 | |||
3445 | if (pos != sfile->source_position) | ||
3446 | { | ||
3447 | /* Only try to seek when the position is wrong. This means the | ||
3448 | * decoder will fail when the source buffer is too small, but | ||
3449 | * only when the input is non-seekable. */ | ||
3450 | if ((ret = main_read_seek_source (stream, source, blkno))) | ||
3451 | { | ||
3452 | return ret; | ||
3453 | } | ||
3454 | |||
3455 | /* Indicates that another call to main_getblk_lru() may be | ||
3456 | * needed */ | ||
3457 | did_seek = 1; | ||
3458 | } | ||
3459 | |||
3460 | XD3_ASSERT (sfile->source_position == pos); | ||
3461 | |||
3462 | if (did_seek && | ||
3463 | (ret = main_getblk_lru (source, blkno, & blru, & is_new))) | ||
3464 | { | ||
3465 | return ret; | ||
3466 | } | ||
3467 | |||
3468 | if ((ret = main_read_primary_input (sfile, | ||
3469 | (uint8_t*) blru->blk, | ||
3470 | source->blksize, | ||
3471 | & nread))) | ||
3472 | { | ||
3473 | return ret; | ||
3474 | } | ||
3475 | |||
3476 | /* Save the last block read, used to handle non-seekable files. */ | ||
3477 | sfile->source_position = pos + nread; | ||
3478 | |||
3479 | if (option_verbose > 3) | ||
3480 | { | ||
3481 | if (blru->blkno != (xoff_t)-1) | ||
3482 | { | ||
3483 | if (blru->blkno != blkno) | ||
3484 | { | ||
3485 | XPR(NT "source block %"Q"u ejects %"Q"u (lru_hits=%u, " | ||
3486 | "lru_misses=%u, lru_filled=%u)\n", | ||
3487 | blkno, blru->blkno, lru_hits, lru_misses, lru_filled); | ||
3488 | } | ||
3489 | else | ||
3490 | { | ||
3491 | XPR(NT "source block %"Q"u read (lru_hits=%u, " | ||
3492 | "lru_misses=%u, lru_filled=%u)\n", | ||
3493 | blkno, lru_hits, lru_misses, lru_filled); | ||
3494 | } | ||
3495 | } | ||
3496 | else | ||
3497 | { | ||
3498 | XPR(NT "source block %"Q"u read (lru_hits=%u, lru_misses=%u, " | ||
3499 | "lru_filled=%u)\n", blkno, lru_hits, lru_misses, lru_filled); | ||
3500 | } | ||
3501 | } | ||
3502 | |||
3503 | source->curblk = blru->blk; | ||
3504 | source->curblkno = blkno; | ||
3505 | source->onblk = nread; | ||
3506 | blru->size = nread; | ||
3507 | |||
3508 | IF_DEBUG1 (DP(RINT "[main_getblk] blkno %"Q"u onblk %u pos %"Q"u " | ||
3509 | "srcpos %"Q"u\n", | ||
3510 | blkno, nread, pos, sfile->source_position)); | ||
3511 | |||
3512 | return 0; | ||
3513 | } | ||
3514 | |||
3515 | /********************************************************************* | 2959 | /********************************************************************* |
3516 | Main routines | 2960 | Main routines |
3517 | ********************************************************************/ | 2961 | ********************************************************************/ |
@@ -4015,8 +3459,6 @@ done: | |||
4015 | static void | 3459 | static void |
4016 | main_cleanup (void) | 3460 | main_cleanup (void) |
4017 | { | 3461 | { |
4018 | usize_t i; | ||
4019 | |||
4020 | if (appheader_used != NULL && | 3462 | if (appheader_used != NULL && |
4021 | appheader_used != option_appheader) | 3463 | appheader_used != option_appheader) |
4022 | { | 3464 | { |
@@ -4033,17 +3475,7 @@ main_cleanup (void) | |||
4033 | ext_tmpfile = NULL; | 3475 | ext_tmpfile = NULL; |
4034 | #endif | 3476 | #endif |
4035 | 3477 | ||
4036 | for (i = 0; lru && i < lru_size; i += 1) | 3478 | main_lru_cleanup(); |
4037 | { | ||
4038 | main_free (lru[i].blk); | ||
4039 | } | ||
4040 | |||
4041 | main_free (lru); | ||
4042 | lru = NULL; | ||
4043 | |||
4044 | lru_hits = 0; | ||
4045 | lru_misses = 0; | ||
4046 | lru_filled = 0; | ||
4047 | 3479 | ||
4048 | if (recode_stream != NULL) | 3480 | if (recode_stream != NULL) |
4049 | { | 3481 | { |
@@ -4301,9 +3733,9 @@ main (int argc, char **argv) | |||
4301 | option_level = ret - '0'; | 3733 | option_level = ret - '0'; |
4302 | break; | 3734 | break; |
4303 | case 'f': option_force = 1; break; | 3735 | case 'f': option_force = 1; break; |
4304 | case 'F': | 3736 | case 'F': |
4305 | #if EXTERNAL_COMPRESSION | 3737 | #if EXTERNAL_COMPRESSION |
4306 | option_force2 = 1; | 3738 | option_force2 = 1; |
4307 | #else | 3739 | #else |
4308 | XPR(NT "warning: -F option ignored, " | 3740 | XPR(NT "warning: -F option ignored, " |
4309 | "external compression support was not compiled\n"); | 3741 | "external compression support was not compiled\n"); |