diff options
Diffstat (limited to 'xdelta3/xdelta3-main.h')
-rw-r--r-- | xdelta3/xdelta3-main.h | 646 |
1 files changed, 39 insertions, 607 deletions
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h index ff6ade6..e47d708 100644 --- a/xdelta3/xdelta3-main.h +++ b/xdelta3/xdelta3-main.h | |||
@@ -192,8 +192,6 @@ typedef enum | |||
192 | 192 | ||
193 | typedef struct _main_file main_file; | 193 | typedef struct _main_file main_file; |
194 | typedef struct _main_extcomp main_extcomp; | 194 | typedef struct _main_extcomp main_extcomp; |
195 | typedef struct _main_blklru main_blklru; | ||
196 | typedef struct _main_blklru_list main_blklru_list; | ||
197 | typedef struct _main_merge main_merge; | 195 | typedef struct _main_merge main_merge; |
198 | typedef struct _main_merge_list main_merge_list; | 196 | typedef struct _main_merge_list main_merge_list; |
199 | 197 | ||
@@ -246,29 +244,6 @@ struct _main_extcomp | |||
246 | int flags; | 244 | int flags; |
247 | }; | 245 | }; |
248 | 246 | ||
249 | /* This file implements a small LRU of source blocks. For encoding purposes, | ||
250 | * we prevent paging in blocks we've already scanned in the source (return | ||
251 | * XD3_NOTAVAIL). */ | ||
252 | struct _main_blklru_list | ||
253 | { | ||
254 | main_blklru_list *next; | ||
255 | main_blklru_list *prev; | ||
256 | }; | ||
257 | |||
258 | struct _main_blklru | ||
259 | { | ||
260 | uint8_t *blk; | ||
261 | xoff_t blkno; | ||
262 | usize_t size; | ||
263 | main_blklru_list link; | ||
264 | }; | ||
265 | |||
266 | #define MAX_LRU_SIZE 32U | ||
267 | #define XD3_MINSRCWINSZ XD3_ALLOCSIZE | ||
268 | |||
269 | /* ... represented as a list (no cache index). */ | ||
270 | XD3_MAKELIST(main_blklru_list,main_blklru,link); | ||
271 | |||
272 | /* Merge state: */ | 247 | /* Merge state: */ |
273 | 248 | ||
274 | struct _main_merge_list | 249 | struct _main_merge_list |
@@ -336,17 +311,6 @@ static uint8_t* appheader_used = NULL; | |||
336 | static uint8_t* main_bdata = NULL; | 311 | static uint8_t* main_bdata = NULL; |
337 | static usize_t main_bsize = 0; | 312 | static usize_t main_bsize = 0; |
338 | 313 | ||
339 | /* The LRU: obviously this is shared by all callers. */ | ||
340 | static usize_t lru_size = 0; | ||
341 | static main_blklru *lru = NULL; /* array of lru_size elts */ | ||
342 | static main_blklru_list lru_list; | ||
343 | static main_blklru_list lru_free; | ||
344 | static int do_src_fifo = 0; /* set to avoid lru */ | ||
345 | |||
346 | static int lru_hits = 0; | ||
347 | static int lru_misses = 0; | ||
348 | static int lru_filled = 0; | ||
349 | |||
350 | /* Hacks for VCDIFF tools */ | 314 | /* Hacks for VCDIFF tools */ |
351 | static int allow_fake_source = 0; | 315 | static int allow_fake_source = 0; |
352 | 316 | ||
@@ -379,8 +343,24 @@ static void main_get_appheader (xd3_stream *stream, main_file *ifile, | |||
379 | static int main_getblk_func (xd3_stream *stream, | 343 | static int main_getblk_func (xd3_stream *stream, |
380 | xd3_source *source, | 344 | xd3_source *source, |
381 | xoff_t blkno); | 345 | xoff_t blkno); |
346 | static void main_free (void *ptr); | ||
347 | static void* main_malloc (usize_t size); | ||
348 | |||
349 | static int main_file_open (main_file *xfile, const char* name, int mode); | ||
350 | static int main_file_stat (main_file *xfile, xoff_t *size); | ||
351 | static int main_file_seek (main_file *xfile, xoff_t pos); | ||
352 | static int main_read_primary_input (main_file *file, | ||
353 | uint8_t *buf, | ||
354 | usize_t size, | ||
355 | usize_t *nread); | ||
356 | |||
357 | static const char* main_format_bcnt (xoff_t r, char *buf); | ||
382 | static int main_help (void); | 358 | static int main_help (void); |
383 | 359 | ||
360 | /* The code in xdelta3-blk.h is essentially part of this unit, see | ||
361 | * comments there. */ | ||
362 | #include "xdelta3-blkcache.h" | ||
363 | |||
384 | static int | 364 | static int |
385 | main_version (void) | 365 | main_version (void) |
386 | { | 366 | { |
@@ -418,12 +398,12 @@ main_config (void) | |||
418 | DP(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ); | 398 | DP(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ); |
419 | DP(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE); | 399 | DP(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE); |
420 | DP(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE); | 400 | DP(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE); |
421 | DP(RINT "sizeof(void*)=%d\n", sizeof(void*)); | 401 | DP(RINT "sizeof(void*)=%d\n", (int)sizeof(void*)); |
422 | DP(RINT "sizeof(int)=%d\n", sizeof(int)); | 402 | DP(RINT "sizeof(int)=%d\n", (int)sizeof(int)); |
423 | DP(RINT "sizeof(uint32_t)=%d\n", sizeof(uint32_t)); | 403 | DP(RINT "sizeof(uint32_t)=%d\n", (int)sizeof(uint32_t)); |
424 | DP(RINT "sizeof(uint64_t)=%d\n", sizeof(uint64_t)); | 404 | DP(RINT "sizeof(uint64_t)=%d\n", (int)sizeof(uint64_t)); |
425 | DP(RINT "sizeof(usize_t)=%d\n", sizeof(usize_t)); | 405 | DP(RINT "sizeof(usize_t)=%d\n", (int)sizeof(usize_t)); |
426 | DP(RINT "sizeof(xoff_t)=%d\n", sizeof(xoff_t)); | 406 | DP(RINT "sizeof(xoff_t)=%d\n", (int)sizeof(xoff_t)); |
427 | 407 | ||
428 | return EXIT_SUCCESS; | 408 | return EXIT_SUCCESS; |
429 | } | 409 | } |
@@ -447,15 +427,11 @@ reset_defaults(void) | |||
447 | appheader_used = NULL; | 427 | appheader_used = NULL; |
448 | main_bdata = NULL; | 428 | main_bdata = NULL; |
449 | main_bsize = 0; | 429 | main_bsize = 0; |
450 | lru_size = 0; | ||
451 | lru = NULL; | ||
452 | do_src_fifo = 0; | ||
453 | lru_hits = 0; | ||
454 | lru_misses = 0; | ||
455 | lru_filled = 0; | ||
456 | allow_fake_source = 0; | 430 | allow_fake_source = 0; |
457 | option_smatch_config = NULL; | 431 | option_smatch_config = NULL; |
458 | 432 | ||
433 | main_lru_reset(); | ||
434 | |||
459 | option_use_appheader = 1; | 435 | option_use_appheader = 1; |
460 | option_use_checksum = 1; | 436 | option_use_checksum = 1; |
461 | #if EXTERNAL_COMPRESSION | 437 | #if EXTERNAL_COMPRESSION |
@@ -531,7 +507,7 @@ get_errno (void) | |||
531 | return errno; | 507 | return errno; |
532 | #else | 508 | #else |
533 | DWORD err_num = GetLastError(); | 509 | DWORD err_num = GetLastError(); |
534 | if (err_num == NO_ERROR) | 510 | if (err_num == NO_ERROR) |
535 | { | 511 | { |
536 | err_num = XD3_INTERNAL; | 512 | err_num = XD3_INTERNAL; |
537 | } | 513 | } |
@@ -543,7 +519,7 @@ const char* | |||
543 | xd3_mainerror(int err_num) { | 519 | xd3_mainerror(int err_num) { |
544 | #ifndef _WIN32 | 520 | #ifndef _WIN32 |
545 | const char* x = xd3_strerror (err_num); | 521 | const char* x = xd3_strerror (err_num); |
546 | if (x != NULL) | 522 | if (x != NULL) |
547 | { | 523 | { |
548 | return x; | 524 | return x; |
549 | } | 525 | } |
@@ -894,12 +870,12 @@ main_file_open (main_file *xfile, const char* name, int mode) | |||
894 | (mode == XO_READ) ? GENERIC_READ : GENERIC_WRITE, | 870 | (mode == XO_READ) ? GENERIC_READ : GENERIC_WRITE, |
895 | FILE_SHARE_READ, | 871 | FILE_SHARE_READ, |
896 | NULL, | 872 | NULL, |
897 | (mode == XO_READ) ? | 873 | (mode == XO_READ) ? |
898 | OPEN_EXISTING : | 874 | OPEN_EXISTING : |
899 | (option_force ? CREATE_ALWAYS : CREATE_NEW), | 875 | (option_force ? CREATE_ALWAYS : CREATE_NEW), |
900 | FILE_ATTRIBUTE_NORMAL, | 876 | FILE_ATTRIBUTE_NORMAL, |
901 | NULL); | 877 | NULL); |
902 | if (xfile->file == INVALID_HANDLE_VALUE) | 878 | if (xfile->file == INVALID_HANDLE_VALUE) |
903 | { | 879 | { |
904 | ret = get_errno (); | 880 | ret = get_errno (); |
905 | } | 881 | } |
@@ -1126,7 +1102,7 @@ main_file_seek (main_file *xfile, xoff_t pos) | |||
1126 | # if (_WIN32_WINNT >= 0x0500) | 1102 | # if (_WIN32_WINNT >= 0x0500) |
1127 | LARGE_INTEGER move, out; | 1103 | LARGE_INTEGER move, out; |
1128 | move.QuadPart = pos; | 1104 | move.QuadPart = pos; |
1129 | if (SetFilePointerEx(xfile->file, move, &out, FILE_BEGIN) == 0) | 1105 | if (SetFilePointerEx(xfile->file, move, &out, FILE_BEGIN) == 0) |
1130 | { | 1106 | { |
1131 | ret = get_errno (); | 1107 | ret = get_errno (); |
1132 | } | 1108 | } |
@@ -2500,17 +2476,17 @@ main_secondary_decompress_check (main_file *file, | |||
2500 | { | 2476 | { |
2501 | XPR(NT | 2477 | XPR(NT |
2502 | "WARNING: the encoder is automatically decompressing the input file;\n"); | 2478 | "WARNING: the encoder is automatically decompressing the input file;\n"); |
2503 | XPR(NT | 2479 | XPR(NT |
2504 | "WARNING: the decoder will automatically recompress the output file;\n"); | 2480 | "WARNING: the decoder will automatically recompress the output file;\n"); |
2505 | XPR(NT | 2481 | XPR(NT |
2506 | "WARNING: this may result in different compressed data and checksums\n"); | 2482 | "WARNING: this may result in different compressed data and checksums\n"); |
2507 | XPR(NT | 2483 | XPR(NT |
2508 | "WARNING: despite being identical data; if this is an issue, use -D\n"); | 2484 | "WARNING: despite being identical data; if this is an issue, use -D\n"); |
2509 | XPR(NT | 2485 | XPR(NT |
2510 | "WARNING: to avoid decompression and/or use -R to avoid recompression\n"); | 2486 | "WARNING: to avoid decompression and/or use -R to avoid recompression\n"); |
2511 | XPR(NT | 2487 | XPR(NT |
2512 | "WARNING: and/or manually decompress the input file; if you know the\n"); | 2488 | "WARNING: and/or manually decompress the input file; if you know the\n"); |
2513 | XPR(NT | 2489 | XPR(NT |
2514 | "WARNING: compression settings that will produce identical output\n"); | 2490 | "WARNING: compression settings that will produce identical output\n"); |
2515 | XPR(NT | 2491 | XPR(NT |
2516 | "WARNING: you may set those flags using the environment (e.g., GZIP=-9)\n"); | 2492 | "WARNING: you may set those flags using the environment (e.g., GZIP=-9)\n"); |
@@ -2755,7 +2731,7 @@ main_set_appheader (xd3_stream *stream, main_file *input, main_file *sfile) | |||
2755 | } | 2731 | } |
2756 | } | 2732 | } |
2757 | 2733 | ||
2758 | xd3_set_appheader (stream, appheader_used, | 2734 | xd3_set_appheader (stream, appheader_used, |
2759 | (usize_t) strlen ((char*)appheader_used)); | 2735 | (usize_t) strlen ((char*)appheader_used)); |
2760 | 2736 | ||
2761 | return 0; | 2737 | return 0; |
@@ -2957,234 +2933,6 @@ main_open_output (xd3_stream *stream, main_file *ofile) | |||
2957 | return 0; | 2933 | return 0; |
2958 | } | 2934 | } |
2959 | 2935 | ||
2960 | /* This is called at different times for encoding and decoding. The | ||
2961 | * encoder calls it immediately, the decoder delays until the | ||
2962 | * application header is received. */ | ||
2963 | static int | ||
2964 | main_set_source (xd3_stream *stream, xd3_cmd cmd, | ||
2965 | main_file *sfile, xd3_source *source) | ||
2966 | { | ||
2967 | int ret = 0; | ||
2968 | usize_t i; | ||
2969 | usize_t blksize; | ||
2970 | xoff_t source_size = 0; | ||
2971 | main_blklru block0; | ||
2972 | |||
2973 | XD3_ASSERT (lru == NULL); | ||
2974 | XD3_ASSERT (stream->src == NULL); | ||
2975 | XD3_ASSERT (option_srcwinsz >= XD3_MINSRCWINSZ); | ||
2976 | |||
2977 | main_blklru_list_init (& lru_list); | ||
2978 | main_blklru_list_init (& lru_free); | ||
2979 | |||
2980 | if (allow_fake_source) | ||
2981 | { | ||
2982 | sfile->mode = XO_READ; | ||
2983 | sfile->realname = sfile->filename; | ||
2984 | sfile->nread = 0; | ||
2985 | } | ||
2986 | else | ||
2987 | { | ||
2988 | if ((ret = main_file_open (sfile, sfile->filename, XO_READ))) | ||
2989 | { | ||
2990 | return ret; | ||
2991 | } | ||
2992 | |||
2993 | /* Allow non-seekable sources from the start. If the file | ||
2994 | * turns out to be externally compressed, size_known may change. */ | ||
2995 | sfile->size_known = (main_file_stat (sfile, &source_size) == 0); | ||
2996 | } | ||
2997 | |||
2998 | /* The API requires power-of-two blocksize, */ | ||
2999 | blksize = xd3_pow2_roundup (max (option_srcwinsz / MAX_LRU_SIZE, | ||
3000 | XD3_MINSRCWINSZ)); | ||
3001 | |||
3002 | /* TODO(jmacd): The organization of this code and the implementation | ||
3003 | * of the LRU cache could be improved. This is too convoluted, the | ||
3004 | * code uses main_getblk_func() to read the first block, which may | ||
3005 | * trigger external-decompression, in large part so that the | ||
3006 | * verbose-printing and counters maintained by getblk_func are | ||
3007 | * consistent. There used to be additional optimizations going on | ||
3008 | * here: (1) if the source size is known we would like to lower | ||
3009 | * option_srcwinsz, if possible, (2) avoid allocating more memory | ||
3010 | * than needed, and (3) also want to use a single block when | ||
3011 | * source_size < option_srcwinsz, this because compression is better | ||
3012 | * for larger blocks (especially due to the blockwise-reverse | ||
3013 | * insertion of checksums). | ||
3014 | * | ||
3015 | * (3) is no longer implemented. (2) is only implemented for files | ||
3016 | * that are shorter than the default blocksize, and (1) is not | ||
3017 | * implemented. These optimizations are not taken because the code | ||
3018 | * is already too complicated. | ||
3019 | * | ||
3020 | * The ideal solution may be to allocate a block of memory equal to | ||
3021 | * half of the option_srcwinsz. Read as much data as possible into | ||
3022 | * that block. If the entire file fits, pass one block to the | ||
3023 | * library for best compression. If not, copy the 50% block into | ||
3024 | * smaller (option_srcwinsz / MAX_LRU_SIZE) blocks and proceed. Too | ||
3025 | * much effort for too little payback. */ | ||
3026 | |||
3027 | memset (&block0, 0, sizeof (block0)); | ||
3028 | block0.blkno = (xoff_t) -1; | ||
3029 | |||
3030 | /* Allocate the first block. Even if the size is known at this | ||
3031 | * point, we do not lower it because decompression may happen. */ | ||
3032 | if ((block0.blk = (uint8_t*) main_malloc (blksize)) == NULL) | ||
3033 | { | ||
3034 | ret = ENOMEM; | ||
3035 | return ret; | ||
3036 | } | ||
3037 | |||
3038 | source->blksize = blksize; | ||
3039 | source->name = sfile->filename; | ||
3040 | source->ioh = sfile; | ||
3041 | source->curblkno = (xoff_t) -1; | ||
3042 | source->curblk = NULL; | ||
3043 | |||
3044 | /* We have to read the first block into the cache now, because | ||
3045 | * size_known can still change (due to secondary decompression). | ||
3046 | * Calls main_secondary_decompress_check() via | ||
3047 | * main_read_primary_input(). */ | ||
3048 | lru_size = 1; | ||
3049 | lru = &block0; | ||
3050 | XD3_ASSERT (main_blklru_list_empty (& lru_free)); | ||
3051 | XD3_ASSERT (main_blklru_list_empty (& lru_list)); | ||
3052 | main_blklru_list_push_back (& lru_free, & lru[0]); | ||
3053 | /* This call is partly so that the diagnostics printed by | ||
3054 | * this function appear to happen normally for the first read, | ||
3055 | * which is special. */ | ||
3056 | ret = main_getblk_func (stream, source, 0); | ||
3057 | main_blklru_list_remove (& lru[0]); | ||
3058 | XD3_ASSERT (main_blklru_list_empty (& lru_free)); | ||
3059 | XD3_ASSERT (main_blklru_list_empty (& lru_list)); | ||
3060 | lru = NULL; | ||
3061 | |||
3062 | if (ret != 0) | ||
3063 | { | ||
3064 | main_free (block0.blk); | ||
3065 | |||
3066 | XPR(NT "error reading source: %s: %s\n", | ||
3067 | sfile->filename, | ||
3068 | xd3_mainerror (ret)); | ||
3069 | |||
3070 | return ret; | ||
3071 | } | ||
3072 | |||
3073 | source->onblk = block0.size; | ||
3074 | |||
3075 | /* If the file is smaller than a block, size is known. */ | ||
3076 | if (!sfile->size_known && source->onblk < blksize) | ||
3077 | { | ||
3078 | source_size = source->onblk; | ||
3079 | sfile->size_known = 1; | ||
3080 | } | ||
3081 | |||
3082 | /* We update lru_size accordingly, and modify option_srcwinsz, which | ||
3083 | * will be passed via xd3_config. */ | ||
3084 | if (sfile->size_known && source_size <= option_srcwinsz) | ||
3085 | { | ||
3086 | lru_size = (usize_t) ((source_size + blksize - 1) / blksize); | ||
3087 | } | ||
3088 | else | ||
3089 | { | ||
3090 | lru_size = (option_srcwinsz + blksize - 1) / blksize; | ||
3091 | } | ||
3092 | |||
3093 | XD3_ASSERT (lru_size >= 1); | ||
3094 | option_srcwinsz = lru_size * blksize; | ||
3095 | |||
3096 | if ((lru = (main_blklru*) main_malloc (lru_size * sizeof (main_blklru))) | ||
3097 | == NULL) | ||
3098 | { | ||
3099 | ret = ENOMEM; | ||
3100 | return ret; | ||
3101 | } | ||
3102 | |||
3103 | lru[0].blk = block0.blk; | ||
3104 | lru[0].blkno = 0; | ||
3105 | lru[0].size = source->onblk; | ||
3106 | |||
3107 | if (! sfile->size_known) | ||
3108 | { | ||
3109 | do_src_fifo = 1; | ||
3110 | } | ||
3111 | else if (! do_src_fifo) | ||
3112 | { | ||
3113 | main_blklru_list_push_back (& lru_list, & lru[0]); | ||
3114 | } | ||
3115 | |||
3116 | for (i = 1; i < lru_size; i += 1) | ||
3117 | { | ||
3118 | lru[i].blkno = (xoff_t) -1; | ||
3119 | |||
3120 | if ((lru[i].blk = (uint8_t*) main_malloc (source->blksize)) == NULL) | ||
3121 | { | ||
3122 | ret = ENOMEM; | ||
3123 | return ret; | ||
3124 | } | ||
3125 | |||
3126 | if (! do_src_fifo) | ||
3127 | { | ||
3128 | main_blklru_list_push_back (& lru_free, & lru[i]); | ||
3129 | } | ||
3130 | } | ||
3131 | |||
3132 | if (sfile->size_known) | ||
3133 | { | ||
3134 | ret = xd3_set_source_and_size (stream, source, source_size); | ||
3135 | } | ||
3136 | else | ||
3137 | { | ||
3138 | ret = xd3_set_source (stream, source); | ||
3139 | } | ||
3140 | |||
3141 | if (ret) | ||
3142 | { | ||
3143 | XPR(NT XD3_LIB_ERRMSG (stream, ret)); | ||
3144 | return ret; | ||
3145 | } | ||
3146 | |||
3147 | XD3_ASSERT (stream->src == source); | ||
3148 | XD3_ASSERT (source->blksize == blksize); | ||
3149 | |||
3150 | if (option_verbose) | ||
3151 | { | ||
3152 | static char srcszbuf[32]; | ||
3153 | static char srccntbuf[32]; | ||
3154 | static char winszbuf[32]; | ||
3155 | static char blkszbuf[32]; | ||
3156 | static char nbufs[32]; | ||
3157 | |||
3158 | if (sfile->size_known) | ||
3159 | { | ||
3160 | sprintf (srcszbuf, "source size %s [%"Q"u]", | ||
3161 | main_format_bcnt (source_size, srccntbuf), | ||
3162 | source_size); | ||
3163 | } | ||
3164 | else | ||
3165 | { | ||
3166 | strcpy(srcszbuf, "source size unknown"); | ||
3167 | } | ||
3168 | |||
3169 | nbufs[0] = 0; | ||
3170 | |||
3171 | if (option_verbose > 1) | ||
3172 | { | ||
3173 | sprintf(nbufs, " #bufs %u", lru_size); | ||
3174 | } | ||
3175 | |||
3176 | XPR(NT "source %s %s blksize %s window %s%s%s\n", | ||
3177 | sfile->filename, | ||
3178 | srcszbuf, | ||
3179 | main_format_bcnt (blksize, blkszbuf), | ||
3180 | main_format_bcnt (option_srcwinsz, winszbuf), | ||
3181 | nbufs, | ||
3182 | do_src_fifo ? " (FIFO)" : ""); | ||
3183 | } | ||
3184 | |||
3185 | return 0; | ||
3186 | } | ||
3187 | |||
3188 | static usize_t | 2936 | static usize_t |
3189 | main_get_winsize (main_file *ifile) { | 2937 | main_get_winsize (main_file *ifile) { |
3190 | xoff_t file_size = 0; | 2938 | xoff_t file_size = 0; |
@@ -3208,310 +2956,6 @@ main_get_winsize (main_file *ifile) { | |||
3208 | return size; | 2956 | return size; |
3209 | } | 2957 | } |
3210 | 2958 | ||
3211 | /******************************************************************* | ||
3212 | Source routines | ||
3213 | *******************************************************************/ | ||
3214 | |||
3215 | static int | ||
3216 | main_getblk_lru (xd3_source *source, xoff_t blkno, | ||
3217 | main_blklru** blrup, int *is_new) | ||
3218 | { | ||
3219 | main_blklru *blru = NULL; | ||
3220 | usize_t i; | ||
3221 | |||
3222 | (*is_new) = 0; | ||
3223 | |||
3224 | if (do_src_fifo) | ||
3225 | { | ||
3226 | /* Direct lookup assumes sequential scan w/o skipping blocks. */ | ||
3227 | int idx = blkno % lru_size; | ||
3228 | blru = & lru[idx]; | ||
3229 | if (blru->blkno == blkno) | ||
3230 | { | ||
3231 | (*blrup) = blru; | ||
3232 | return 0; | ||
3233 | } | ||
3234 | |||
3235 | if (blru->blkno != (xoff_t)-1 && | ||
3236 | blru->blkno != (xoff_t)(blkno - lru_size)) | ||
3237 | { | ||
3238 | return XD3_TOOFARBACK; | ||
3239 | } | ||
3240 | } | ||
3241 | else | ||
3242 | { | ||
3243 | /* Sequential search through LRU. */ | ||
3244 | for (i = 0; i < lru_size; i += 1) | ||
3245 | { | ||
3246 | blru = & lru[i]; | ||
3247 | if (blru->blkno == blkno) | ||
3248 | { | ||
3249 | main_blklru_list_remove (blru); | ||
3250 | main_blklru_list_push_back (& lru_list, blru); | ||
3251 | (*blrup) = blru; | ||
3252 | return 0; | ||
3253 | } | ||
3254 | } | ||
3255 | } | ||
3256 | |||
3257 | if (do_src_fifo) | ||
3258 | { | ||
3259 | int idx = blkno % lru_size; | ||
3260 | blru = & lru[idx]; | ||
3261 | } | ||
3262 | else | ||
3263 | { | ||
3264 | if (! main_blklru_list_empty (& lru_free)) | ||
3265 | { | ||
3266 | blru = main_blklru_list_pop_front (& lru_free); | ||
3267 | main_blklru_list_push_back (& lru_list, blru); | ||
3268 | } | ||
3269 | else | ||
3270 | { | ||
3271 | XD3_ASSERT (! main_blklru_list_empty (& lru_list)); | ||
3272 | blru = main_blklru_list_pop_front (& lru_list); | ||
3273 | main_blklru_list_push_back (& lru_list, blru); | ||
3274 | } | ||
3275 | } | ||
3276 | |||
3277 | lru_filled += 1; | ||
3278 | (*is_new) = 1; | ||
3279 | (*blrup) = blru; | ||
3280 | blru->blkno = blkno; | ||
3281 | return 0; | ||
3282 | } | ||
3283 | |||
3284 | static int | ||
3285 | main_read_seek_source (xd3_stream *stream, | ||
3286 | xd3_source *source, | ||
3287 | xoff_t blkno) { | ||
3288 | xoff_t pos = blkno * source->blksize; | ||
3289 | main_file *sfile = (main_file*) source->ioh; | ||
3290 | main_blklru *blru; | ||
3291 | int is_new; | ||
3292 | usize_t nread = 0; | ||
3293 | int ret = 0; | ||
3294 | |||
3295 | if (!sfile->seek_failed) | ||
3296 | { | ||
3297 | ret = main_file_seek (sfile, pos); | ||
3298 | |||
3299 | if (ret == 0) | ||
3300 | { | ||
3301 | sfile->source_position = pos; | ||
3302 | } | ||
3303 | } | ||
3304 | |||
3305 | if (sfile->seek_failed || ret != 0) | ||
3306 | { | ||
3307 | /* For an unseekable file (or other seek error, does it | ||
3308 | * matter?) */ | ||
3309 | if (sfile->source_position > pos) | ||
3310 | { | ||
3311 | /* Could assert !IS_ENCODE(), this shouldn't happen | ||
3312 | * because of do_src_fifo during encode. */ | ||
3313 | if (!option_quiet) | ||
3314 | { | ||
3315 | XPR(NT "source can't seek backwards; requested block offset " | ||
3316 | "%"Q"u source position is %"Q"u\n", | ||
3317 | pos, sfile->source_position); | ||
3318 | } | ||
3319 | |||
3320 | sfile->seek_failed = 1; | ||
3321 | stream->msg = "non-seekable source: " | ||
3322 | "copy is too far back (try raising -B)"; | ||
3323 | return XD3_TOOFARBACK; | ||
3324 | } | ||
3325 | |||
3326 | /* There's a chance here, that an genuine lseek error will cause | ||
3327 | * xdelta3 to shift into non-seekable mode, entering a degraded | ||
3328 | * condition. */ | ||
3329 | if (!sfile->seek_failed && option_verbose) | ||
3330 | { | ||
3331 | XPR(NT "source can't seek, will use FIFO for %s\n", | ||
3332 | sfile->filename); | ||
3333 | |||
3334 | if (option_verbose > 1) | ||
3335 | { | ||
3336 | XPR(NT "seek error at offset %"Q"u: %s\n", | ||
3337 | pos, xd3_mainerror (ret)); | ||
3338 | } | ||
3339 | } | ||
3340 | |||
3341 | sfile->seek_failed = 1; | ||
3342 | |||
3343 | while (sfile->source_position < pos) | ||
3344 | { | ||
3345 | xoff_t skip_blkno; | ||
3346 | usize_t skip_offset; | ||
3347 | |||
3348 | xd3_blksize_div (sfile->source_position, source, | ||
3349 | &skip_blkno, &skip_offset); | ||
3350 | |||
3351 | /* Read past unused data */ | ||
3352 | XD3_ASSERT (pos - sfile->source_position >= source->blksize); | ||
3353 | XD3_ASSERT (skip_offset == 0); | ||
3354 | |||
3355 | if ((ret = main_getblk_lru (source, skip_blkno, | ||
3356 | & blru, & is_new))) | ||
3357 | { | ||
3358 | return ret; | ||
3359 | } | ||
3360 | |||
3361 | XD3_ASSERT (is_new); | ||
3362 | |||
3363 | if (option_verbose > 1) | ||
3364 | { | ||
3365 | XPR(NT "non-seekable source skipping %"Q"u bytes @ %"Q"u\n", | ||
3366 | pos - sfile->source_position, | ||
3367 | sfile->source_position); | ||
3368 | } | ||
3369 | |||
3370 | if ((ret = main_read_primary_input (sfile, | ||
3371 | (uint8_t*) blru->blk, | ||
3372 | source->blksize, | ||
3373 | & nread))) | ||
3374 | { | ||
3375 | return ret; | ||
3376 | } | ||
3377 | |||
3378 | if (nread != source->blksize) | ||
3379 | { | ||
3380 | IF_DEBUG1 (DP(RINT "[getblk] short skip block nread = %u\n", | ||
3381 | nread)); | ||
3382 | stream->msg = "non-seekable input is short"; | ||
3383 | return XD3_INVALID_INPUT; | ||
3384 | } | ||
3385 | |||
3386 | sfile->source_position += nread; | ||
3387 | blru->size = nread; | ||
3388 | |||
3389 | IF_DEBUG1 (DP(RINT "[getblk] skip blkno %"Q"u size %u\n", | ||
3390 | skip_blkno, blru->size)); | ||
3391 | |||
3392 | XD3_ASSERT (sfile->source_position <= pos); | ||
3393 | } | ||
3394 | } | ||
3395 | |||
3396 | return 0; | ||
3397 | } | ||
3398 | |||
3399 | /* This is the callback for reading a block of source. This function | ||
3400 | * is blocking and it implements a small LRU. | ||
3401 | * | ||
3402 | * Note that it is possible for main_input() to handle getblk requests | ||
3403 | * in a non-blocking manner. If the callback is NULL then the caller | ||
3404 | * of xd3_*_input() must handle the XD3_GETSRCBLK return value and | ||
3405 | * fill the source in the same way. See xd3_getblk for details. To | ||
3406 | * see an example of non-blocking getblk, see xdelta-test.h. */ | ||
3407 | static int | ||
3408 | main_getblk_func (xd3_stream *stream, | ||
3409 | xd3_source *source, | ||
3410 | xoff_t blkno) | ||
3411 | { | ||
3412 | int ret = 0; | ||
3413 | xoff_t pos = blkno * source->blksize; | ||
3414 | main_file *sfile = (main_file*) source->ioh; | ||
3415 | main_blklru *blru; | ||
3416 | int is_new; | ||
3417 | int did_seek = 0; | ||
3418 | usize_t nread = 0; | ||
3419 | |||
3420 | if (allow_fake_source) | ||
3421 | { | ||
3422 | source->curblkno = blkno; | ||
3423 | source->onblk = 0; | ||
3424 | source->curblk = lru[0].blk; | ||
3425 | lru[0].size = 0; | ||
3426 | return 0; | ||
3427 | } | ||
3428 | |||
3429 | if ((ret = main_getblk_lru (source, blkno, & blru, & is_new))) | ||
3430 | { | ||
3431 | return ret; | ||
3432 | } | ||
3433 | |||
3434 | if (!is_new) | ||
3435 | { | ||
3436 | source->curblkno = blkno; | ||
3437 | source->onblk = blru->size; | ||
3438 | source->curblk = blru->blk; | ||
3439 | lru_hits++; | ||
3440 | return 0; | ||
3441 | } | ||
3442 | |||
3443 | lru_misses += 1; | ||
3444 | |||
3445 | if (pos != sfile->source_position) | ||
3446 | { | ||
3447 | /* Only try to seek when the position is wrong. This means the | ||
3448 | * decoder will fail when the source buffer is too small, but | ||
3449 | * only when the input is non-seekable. */ | ||
3450 | if ((ret = main_read_seek_source (stream, source, blkno))) | ||
3451 | { | ||
3452 | return ret; | ||
3453 | } | ||
3454 | |||
3455 | /* Indicates that another call to main_getblk_lru() may be | ||
3456 | * needed */ | ||
3457 | did_seek = 1; | ||
3458 | } | ||
3459 | |||
3460 | XD3_ASSERT (sfile->source_position == pos); | ||
3461 | |||
3462 | if (did_seek && | ||
3463 | (ret = main_getblk_lru (source, blkno, & blru, & is_new))) | ||
3464 | { | ||
3465 | return ret; | ||
3466 | } | ||
3467 | |||
3468 | if ((ret = main_read_primary_input (sfile, | ||
3469 | (uint8_t*) blru->blk, | ||
3470 | source->blksize, | ||
3471 | & nread))) | ||
3472 | { | ||
3473 | return ret; | ||
3474 | } | ||
3475 | |||
3476 | /* Save the last block read, used to handle non-seekable files. */ | ||
3477 | sfile->source_position = pos + nread; | ||
3478 | |||
3479 | if (option_verbose > 3) | ||
3480 | { | ||
3481 | if (blru->blkno != (xoff_t)-1) | ||
3482 | { | ||
3483 | if (blru->blkno != blkno) | ||
3484 | { | ||
3485 | XPR(NT "source block %"Q"u ejects %"Q"u (lru_hits=%u, " | ||
3486 | "lru_misses=%u, lru_filled=%u)\n", | ||
3487 | blkno, blru->blkno, lru_hits, lru_misses, lru_filled); | ||
3488 | } | ||
3489 | else | ||
3490 | { | ||
3491 | XPR(NT "source block %"Q"u read (lru_hits=%u, " | ||
3492 | "lru_misses=%u, lru_filled=%u)\n", | ||
3493 | blkno, lru_hits, lru_misses, lru_filled); | ||
3494 | } | ||
3495 | } | ||
3496 | else | ||
3497 | { | ||
3498 | XPR(NT "source block %"Q"u read (lru_hits=%u, lru_misses=%u, " | ||
3499 | "lru_filled=%u)\n", blkno, lru_hits, lru_misses, lru_filled); | ||
3500 | } | ||
3501 | } | ||
3502 | |||
3503 | source->curblk = blru->blk; | ||
3504 | source->curblkno = blkno; | ||
3505 | source->onblk = nread; | ||
3506 | blru->size = nread; | ||
3507 | |||
3508 | IF_DEBUG1 (DP(RINT "[main_getblk] blkno %"Q"u onblk %u pos %"Q"u " | ||
3509 | "srcpos %"Q"u\n", | ||
3510 | blkno, nread, pos, sfile->source_position)); | ||
3511 | |||
3512 | return 0; | ||
3513 | } | ||
3514 | |||
3515 | /********************************************************************* | 2959 | /********************************************************************* |
3516 | Main routines | 2960 | Main routines |
3517 | ********************************************************************/ | 2961 | ********************************************************************/ |
@@ -4015,8 +3459,6 @@ done: | |||
4015 | static void | 3459 | static void |
4016 | main_cleanup (void) | 3460 | main_cleanup (void) |
4017 | { | 3461 | { |
4018 | usize_t i; | ||
4019 | |||
4020 | if (appheader_used != NULL && | 3462 | if (appheader_used != NULL && |
4021 | appheader_used != option_appheader) | 3463 | appheader_used != option_appheader) |
4022 | { | 3464 | { |
@@ -4033,17 +3475,7 @@ main_cleanup (void) | |||
4033 | ext_tmpfile = NULL; | 3475 | ext_tmpfile = NULL; |
4034 | #endif | 3476 | #endif |
4035 | 3477 | ||
4036 | for (i = 0; lru && i < lru_size; i += 1) | 3478 | main_lru_cleanup(); |
4037 | { | ||
4038 | main_free (lru[i].blk); | ||
4039 | } | ||
4040 | |||
4041 | main_free (lru); | ||
4042 | lru = NULL; | ||
4043 | |||
4044 | lru_hits = 0; | ||
4045 | lru_misses = 0; | ||
4046 | lru_filled = 0; | ||
4047 | 3479 | ||
4048 | if (recode_stream != NULL) | 3480 | if (recode_stream != NULL) |
4049 | { | 3481 | { |
@@ -4301,9 +3733,9 @@ main (int argc, char **argv) | |||
4301 | option_level = ret - '0'; | 3733 | option_level = ret - '0'; |
4302 | break; | 3734 | break; |
4303 | case 'f': option_force = 1; break; | 3735 | case 'f': option_force = 1; break; |
4304 | case 'F': | 3736 | case 'F': |
4305 | #if EXTERNAL_COMPRESSION | 3737 | #if EXTERNAL_COMPRESSION |
4306 | option_force2 = 1; | 3738 | option_force2 = 1; |
4307 | #else | 3739 | #else |
4308 | XPR(NT "warning: -F option ignored, " | 3740 | XPR(NT "warning: -F option ignored, " |
4309 | "external compression support was not compiled\n"); | 3741 | "external compression support was not compiled\n"); |