summaryrefslogtreecommitdiff
path: root/xdelta3
diff options
context:
space:
mode:
authorjosh.macdonald <jmacd@users.noreply.github.com>2009-11-03 05:08:19 +0000
committerjosh.macdonald <jmacd@users.noreply.github.com>2009-11-03 05:08:19 +0000
commit97eb8c8b6f2b9e9a1d124e972e86765a92c2b37e (patch)
tree6d1291f8590702c216fc20a4d88386869034fd60 /xdelta3
parent7f7ac45b750fe90bc0572c1a53981910d24b878a (diff)
Implement (and manually test) main_getblk_func() using FIFO instead of
LRU for non-seekable sources (always, was not previously for the decoder, which caused problems).
Diffstat (limited to 'xdelta3')
-rw-r--r--xdelta3/Makefile12
-rw-r--r--xdelta3/xdelta3-decode.h1
-rw-r--r--xdelta3/xdelta3-main.h159
-rw-r--r--xdelta3/xdelta3.c6
-rw-r--r--xdelta3/xdelta3.h6
5 files changed, 134 insertions, 50 deletions
diff --git a/xdelta3/Makefile b/xdelta3/Makefile
index 3309c49..6826253 100644
--- a/xdelta3/Makefile
+++ b/xdelta3/Makefile
@@ -31,7 +31,7 @@ SOURCES = xdelta3-cfgs.h \
31TARGETS = xdelta3-debug \ 31TARGETS = xdelta3-debug \
32 xdelta3 \ 32 xdelta3 \
33 xdelta3-debug2 \ 33 xdelta3-debug2 \
34 xdelta3-debug3 \ 34 xdelta3-debugstdio \
35 xdelta3.o \ 35 xdelta3.o \
36 xdelta3_wrap.o \ 36 xdelta3_wrap.o \
37 xdelta3-32 \ 37 xdelta3-32 \
@@ -143,7 +143,7 @@ xdelta3-debug: $(SOURCES)
143 -DSECONDARY_FGK=1 \ 143 -DSECONDARY_FGK=1 \
144 -DXD3_DEBUG=1 \ 144 -DXD3_DEBUG=1 \
145 -DXD3_MAIN=1 \ 145 -DXD3_MAIN=1 \
146 -DXD3_STDIO=1 \ 146 -DXD3_POSIX=1 \
147 -DXD3_USE_LARGEFILE64=1 147 -DXD3_USE_LARGEFILE64=1
148 148
149xdelta3-warnings: $(SOURCES) 149xdelta3-warnings: $(SOURCES)
@@ -172,7 +172,7 @@ xdelta3-debug2: $(SOURCES)
172 xdelta3.c -o xdelta3-debug2 \ 172 xdelta3.c -o xdelta3-debug2 \
173 -DXD3_DEBUG=2 \ 173 -DXD3_DEBUG=2 \
174 -DXD3_MAIN=1 \ 174 -DXD3_MAIN=1 \
175 -DXD3_STDIO=1 \ 175 -DXD3_POSIX=1 \
176 -DXD3_USE_LARGEFILE64=1 \ 176 -DXD3_USE_LARGEFILE64=1 \
177 -DGENERIC_ENCODE_TABLES=1 \ 177 -DGENERIC_ENCODE_TABLES=1 \
178 -DREGRESSION_TEST=1 \ 178 -DREGRESSION_TEST=1 \
@@ -180,14 +180,14 @@ xdelta3-debug2: $(SOURCES)
180 -DSECONDARY_FGK=1 \ 180 -DSECONDARY_FGK=1 \
181 -lm 181 -lm
182 182
183xdelta3-debug3: $(SOURCES) 183xdelta3-debugstdio: $(SOURCES)
184 $(CC) -g $(CFLAGS) xdelta3.c -o xdelta3-debug3 \ 184 $(CC) -g $(CFLAGS) xdelta3.c -o xdelta3-debugstdio \
185 -DXD3_MAIN=1 \ 185 -DXD3_MAIN=1 \
186 -DGENERIC_ENCODE_TABLES=1 \ 186 -DGENERIC_ENCODE_TABLES=1 \
187 -DXD3_USE_LARGEFILE64=1 \ 187 -DXD3_USE_LARGEFILE64=1 \
188 -DXD3_STDIO=1 \ 188 -DXD3_STDIO=1 \
189 -DREGRESSION_TEST=1 \ 189 -DREGRESSION_TEST=1 \
190 -DXD3_DEBUG=3 \ 190 -DXD3_DEBUG=1 \
191 -DSECONDARY_DJW=1 \ 191 -DSECONDARY_DJW=1 \
192 -DSECONDARY_FGK=1 \ 192 -DSECONDARY_FGK=1 \
193 -lm 193 -lm
diff --git a/xdelta3/xdelta3-decode.h b/xdelta3/xdelta3-decode.h
index 542b6aa..0835222 100644
--- a/xdelta3/xdelta3-decode.h
+++ b/xdelta3/xdelta3-decode.h
@@ -435,6 +435,7 @@ xd3_decode_output_halfinst (xd3_stream *stream, xd3_hinst *inst)
435 /* could be a XD3_GETSRCBLK failure. */ 435 /* could be a XD3_GETSRCBLK failure. */
436 if (ret == XD3_TOOFARBACK) 436 if (ret == XD3_TOOFARBACK)
437 { 437 {
438 stream->msg = "non-seekable source in decode";
438 ret = XD3_INTERNAL; 439 ret = XD3_INTERNAL;
439 } 440 }
440 return ret; 441 return ret;
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h
index 0b8740b..6ea6c68 100644
--- a/xdelta3/xdelta3-main.h
+++ b/xdelta3/xdelta3-main.h
@@ -594,19 +594,55 @@ get_millisecs_since (void)
594static char* 594static char*
595main_format_bcnt (xoff_t r, char *buf) 595main_format_bcnt (xoff_t r, char *buf)
596{ 596{
597 static const char* fmts[] = { "B", "KB", "MB", "GB" }; 597 static const char* fmts[] = { "B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB" };
598 usize_t i; 598 usize_t i;
599 599
600 for (i = 0; i < SIZEOF_ARRAY(fmts); i += 1) 600 for (i = 0; i < SIZEOF_ARRAY(fmts) - 1; i += 1)
601 { 601 {
602 if (r <= (10 * 1024) || i == (-1 + (int)SIZEOF_ARRAY(fmts))) 602 xoff_t new_r;
603
604 if (r == 0)
605 {
606 sprintf (buf, "0 %s", fmts[i]);
607 return buf;
608 }
609
610 if (r >= 1 && r < 10)
611 {
612 sprintf (buf, "%.2f %s", (double) r, fmts[i]);
613 return buf;
614 }
615
616 if (r >= 10 && r < 100)
617 {
618 sprintf (buf, "%.1f %s", (double) r, fmts[i]);
619 return buf;
620 }
621
622 if (r >= 100 && r < 1000)
603 { 623 {
604 sprintf (buf, "%"Q"u %s", r, fmts[i]); 624 sprintf (buf, "%"Q"u %s", r, fmts[i]);
605 break; 625 return buf;
626 }
627
628 new_r = r / 1024;
629
630 if (new_r < 10)
631 {
632 sprintf (buf, "%.2f %s", (double) r / 1024.0, fmts[i + 1]);
633 return buf;
634 }
635
636 if (new_r < 100)
637 {
638 sprintf (buf, "%.1f %s", (double) r / 1024.0, fmts[i + 1]);
639 return buf;
606 } 640 }
607 r /= 1024; 641
642 r = new_r;
608 } 643 }
609 return buf; 644 XD3_ASSERT (0);
645 return "";
610} 646}
611 647
612static char* 648static char*
@@ -616,7 +652,7 @@ main_format_rate (xoff_t bytes, long millis, char *buf)
616 static char lbuf[32]; 652 static char lbuf[32];
617 653
618 main_format_bcnt (r, lbuf); 654 main_format_bcnt (r, lbuf);
619 sprintf (buf, "%s/sec", lbuf); 655 sprintf (buf, "%s/s", lbuf);
620 return buf; 656 return buf;
621} 657}
622 658
@@ -954,11 +990,11 @@ xd3_posix_io (int fd, uint8_t *buf, usize_t size,
954/* POSIX is unbuffered, while STDIO is buffered. main_file_read() 990/* POSIX is unbuffered, while STDIO is buffered. main_file_read()
955 * should always be called on blocks. */ 991 * should always be called on blocks. */
956static int 992static int
957main_file_read (main_file *ifile, 993main_file_read (main_file *ifile,
958 uint8_t *buf, 994 uint8_t *buf,
959 usize_t size, 995 usize_t size,
960 usize_t *nread, 996 usize_t *nread,
961 const char *msg) 997 const char *msg)
962{ 998{
963 int ret = 0; 999 int ret = 0;
964 1000
@@ -1103,6 +1139,32 @@ main_write_output (xd3_stream* stream, main_file *ofile)
1103 return 0; 1139 return 0;
1104} 1140}
1105 1141
1142/* Called when the stream transitions from unknown status to
1143 * possibly/definitely non-seekable. */
1144static void
1145main_internal_do_not_lru ()
1146{
1147 usize_t i;
1148
1149 if (do_not_lru)
1150 {
1151 /* Already set */
1152 return;
1153 }
1154
1155 do_not_lru = 1;
1156
1157 /* It's possible that we've already ejected in a non-LRU fashion,
1158 * report on that. */
1159 for (i = 0; i < lru_size; i += 1)
1160 {
1161 if (lru[i].blkno == (xoff_t) -1)
1162 {
1163 continue;
1164 }
1165 }
1166}
1167
1106static int 1168static int
1107main_set_secondary_flags (xd3_config *config) 1169main_set_secondary_flags (xd3_config *config)
1108{ 1170{
@@ -2285,7 +2347,7 @@ main_input_decompress_setup (const main_extcomp *decomp,
2285 * is passed to the pipe copier. This avoids using the same size 2347 * is passed to the pipe copier. This avoids using the same size
2286 * buffer in both cases. */ 2348 * buffer in both cases. */
2287static int 2349static int
2288main_decompress_input_check (main_file *ifile, 2350main_secondary_decompress_check (main_file *ifile,
2289 uint8_t *input_buf, 2351 uint8_t *input_buf,
2290 usize_t input_size, 2352 usize_t input_size,
2291 usize_t *nread) 2353 usize_t *nread)
@@ -2322,6 +2384,8 @@ main_decompress_input_check (main_file *ifile,
2322 decomp->decomp_options); 2384 decomp->decomp_options);
2323 } 2385 }
2324 2386
2387 main_internal_do_not_lru ();
2388
2325 return main_input_decompress_setup (decomp, ifile, 2389 return main_input_decompress_setup (decomp, ifile,
2326 input_buf, input_size, 2390 input_buf, input_size,
2327 check_buf, XD3_ALLOCSIZE, 2391 check_buf, XD3_ALLOCSIZE,
@@ -2670,29 +2734,29 @@ main_get_appheader (xd3_stream *stream, main_file *ifile,
2670 * buffer of data is read. The EXTERNAL_COMPRESSION code is called to 2734 * buffer of data is read. The EXTERNAL_COMPRESSION code is called to
2671 * search for magic numbers. */ 2735 * search for magic numbers. */
2672static int 2736static int
2673main_read_primary_input (main_file *ifile, 2737main_read_primary_input (main_file *file,
2674 uint8_t *buf, 2738 uint8_t *buf,
2675 usize_t size, 2739 usize_t size,
2676 usize_t *nread, 2740 usize_t *nread,
2677 int is_source) 2741 int is_source)
2678{ 2742{
2679#if EXTERNAL_COMPRESSION 2743#if EXTERNAL_COMPRESSION
2680 if (option_decompress_inputs && ifile->flags & RD_FIRST) 2744 if (option_decompress_inputs && file->flags & RD_FIRST)
2681 { 2745 {
2682 ifile->flags &= ~RD_FIRST; 2746 file->flags &= ~RD_FIRST;
2683 2747
2684 if (is_source && ifile->compressor == NULL) 2748 if (is_source && file->compressor == NULL)
2685 { 2749 {
2686 /* Application header overrides magic number. */ 2750 /* Application header overrides magic number. */
2687 } 2751 }
2688 else 2752 else
2689 { 2753 {
2690 return main_decompress_input_check (ifile, buf, size, nread); 2754 return main_secondary_decompress_check (file, buf, size, nread);
2691 } 2755 }
2692 } 2756 }
2693#endif 2757#endif
2694 2758
2695 return main_file_read (ifile, buf, size, nread, "input read failed"); 2759 return main_file_read (file, buf, size, nread, "input read failed");
2696} 2760}
2697 2761
2698/* Open the main output file, sets a default file name, initiate 2762/* Open the main output file, sets a default file name, initiate
@@ -2825,6 +2889,7 @@ main_set_source (xd3_stream *stream, xd3_cmd cmd,
2825 { 2889 {
2826 IF_DEBUG2 (DP(RINT "[main_set_source] %s size not known\n", 2890 IF_DEBUG2 (DP(RINT "[main_set_source] %s size not known\n",
2827 sfile->filename, source_size)); 2891 sfile->filename, source_size));
2892 main_internal_do_not_lru ();
2828 } 2893 }
2829 2894
2830 source->name = sfile->filename; 2895 source->name = sfile->filename;
@@ -2849,22 +2914,23 @@ main_set_source (xd3_stream *stream, xd3_cmd cmd,
2849 2914
2850 if (option_verbose) 2915 if (option_verbose)
2851 { 2916 {
2852 static char buf[32]; 2917 static char winszbuf[32];
2853 static char sizebuf[32]; 2918 static char srcszbuf[32];
2919 static char blkszbuf[32];
2854 if (source_size_known) 2920 if (source_size_known)
2855 { 2921 {
2856 sprintf(sizebuf, " size %"Q"u", source_size); 2922 sprintf(srcszbuf, " size %"Q"u", source_size);
2857 } 2923 }
2858 else 2924 else
2859 { 2925 {
2860 sizebuf[0] = 0; 2926 strcpy(srcszbuf, " size not known");
2861 } 2927 }
2862 2928
2863 XPR(NT "source %s winsize %s blksize %u%s\n", 2929 XPR(NT "source %s winsize %s blksize %s%s\n",
2864 sfile->filename, 2930 sfile->filename,
2865 main_format_bcnt(option_srcwinsz, buf), 2931 main_format_bcnt(option_srcwinsz, winszbuf),
2866 source->blksize, 2932 main_format_bcnt(source->blksize, blkszbuf),
2867 sizebuf); 2933 srcszbuf);
2868 } 2934 }
2869 2935
2870 XD3_ASSERT (lru == NULL); 2936 XD3_ASSERT (lru == NULL);
@@ -3054,12 +3120,18 @@ main_getblk_func (xd3_stream *stream,
3054 if (!sfile->seek_failed) 3120 if (!sfile->seek_failed)
3055 { 3121 {
3056 ret = main_file_seek (sfile, pos); 3122 ret = main_file_seek (sfile, pos);
3123
3124 if (ret == 0)
3125 {
3126 sfile->source_position = pos;
3127 }
3057 } 3128 }
3058 3129
3130 /* There's a chance here, that an genuine lseek error will cause
3131 * xdelta3 to shift into non-seekable mode, entering a degraded
3132 * condition. */
3059 if (sfile->seek_failed || ret != 0) 3133 if (sfile->seek_failed || ret != 0)
3060 { 3134 {
3061 sfile->seek_failed = 1;
3062
3063 /* For an unseekable file (or other seek error, does it 3135 /* For an unseekable file (or other seek error, does it
3064 * matter?) */ 3136 * matter?) */
3065 if (sfile->source_position > pos) 3137 if (sfile->source_position > pos)
@@ -3068,17 +3140,24 @@ main_getblk_func (xd3_stream *stream,
3068 * because of do_not_lru during encode. */ 3140 * because of do_not_lru during encode. */
3069 if (option_verbose) 3141 if (option_verbose)
3070 { 3142 {
3071 XPR(NT "copy lags source position by %"Q"u bytes\n", 3143 XPR(NT "source can't seek backwards; requested block offset "
3072 sfile->source_position - pos); 3144 "%"Q"u source position is %"Q"u\n",
3145 pos, sfile->source_position);
3073 } 3146 }
3074 3147
3148 sfile->seek_failed = 1;
3075 stream->msg = "non-seekable source: copy is too far back (try raising -B)"; 3149 stream->msg = "non-seekable source: copy is too far back (try raising -B)";
3076 return XD3_TOOFARBACK; 3150 return XD3_TOOFARBACK;
3077 } 3151 }
3078 3152
3079 IF_DEBUG1 (DP(RINT "[getblk] skip %"Q"u starting at %"Q"u\n", 3153 if (option_verbose > 2 || (option_verbose > 1 && !sfile->seek_failed))
3080 pos - sfile->source_position, 3154 {
3081 sfile->source_position)); 3155 XPR(NT "non-seekable source skipping %"Q"u bytes @ %"Q"u\n",
3156 pos - sfile->source_position,
3157 sfile->source_position);
3158 }
3159
3160 sfile->seek_failed = 1;
3082 3161
3083 while (sfile->source_position < pos) 3162 while (sfile->source_position < pos)
3084 { 3163 {
@@ -3166,7 +3245,7 @@ main_getblk_func (xd3_stream *stream,
3166 source->onblk = nread; 3245 source->onblk = nread;
3167 blru->size = nread; 3246 blru->size = nread;
3168 3247
3169 IF_DEBUG1 (DP(RINT "[main_getblk] blkno %"Q"u onblk %u pos %"Q"u sfile->source_position %"Q"u\n", 3248 IF_DEBUG1 (DP(RINT "[main_getblk] blkno %"Q"u onblk %u pos %"Q"u srcpos %"Q"u\n",
3170 blkno, nread, pos, sfile->source_position)); 3249 blkno, nread, pos, sfile->source_position));
3171 3250
3172 return 0; 3251 return 0;
@@ -3528,6 +3607,7 @@ main_input (xd3_cmd cmd,
3528 char rrateavg[32], wrateavg[32], tm[32]; 3607 char rrateavg[32], wrateavg[32], tm[32];
3529 char rdb[32], wdb[32]; 3608 char rdb[32], wdb[32];
3530 char trdb[32], twdb[32]; 3609 char trdb[32], twdb[32];
3610 char srcpos[32];
3531 long millis = get_millisecs_since (); 3611 long millis = get_millisecs_since ();
3532 usize_t this_read = (usize_t)(stream.total_in - 3612 usize_t this_read = (usize_t)(stream.total_in -
3533 last_total_in); 3613 last_total_in);
@@ -3539,7 +3619,7 @@ main_input (xd3_cmd cmd,
3539 if (option_verbose > 1) 3619 if (option_verbose > 1)
3540 { 3620 {
3541 XPR(NT "%"Q"u: in %s (%s): out %s (%s): " 3621 XPR(NT "%"Q"u: in %s (%s): out %s (%s): "
3542 "total in %s: out %s: %s\n", 3622 "total in %s: out %s: %s: srcpos %s\n",
3543 stream.current_window, 3623 stream.current_window,
3544 main_format_bcnt (this_read, rdb), 3624 main_format_bcnt (this_read, rdb),
3545 main_format_rate (this_read, millis, rrateavg), 3625 main_format_rate (this_read, millis, rrateavg),
@@ -3547,7 +3627,8 @@ main_input (xd3_cmd cmd,
3547 main_format_rate (this_write, millis, wrateavg), 3627 main_format_rate (this_write, millis, wrateavg),
3548 main_format_bcnt (stream.total_in, trdb), 3628 main_format_bcnt (stream.total_in, trdb),
3549 main_format_bcnt (stream.total_out, twdb), 3629 main_format_bcnt (stream.total_out, twdb),
3550 main_format_millis (millis, tm)); 3630 main_format_millis (millis, tm),
3631 main_format_bcnt (sfile->source_position, srcpos));
3551 } 3632 }
3552 else 3633 else
3553 { 3634 {
@@ -3659,7 +3740,7 @@ done:
3659 long end_time = get_millisecs_now (); 3740 long end_time = get_millisecs_now ();
3660 xoff_t nwrite = ofile != NULL ? ofile->nwrite : 0; 3741 xoff_t nwrite = ofile != NULL ? ofile->nwrite : 0;
3661 3742
3662 XPR(NT "finished in %s; input %"Q"u output %"Q"u bytes (%0.2f%%)\n", 3743 XPR(NT "finished in %s; input %"Q"u output %"Q"u bytes (%0.2f%%)\n",
3663 main_format_millis (end_time - start_time, tm), 3744 main_format_millis (end_time - start_time, tm),
3664 ifile->nread, nwrite, 100.0 * nwrite / ifile->nread); 3745 ifile->nread, nwrite, 100.0 * nwrite / ifile->nread);
3665 } 3746 }
diff --git a/xdelta3/xdelta3.c b/xdelta3/xdelta3.c
index 435e8d8..01172ea 100644
--- a/xdelta3/xdelta3.c
+++ b/xdelta3/xdelta3.c
@@ -2518,6 +2518,7 @@ xd3_getblk (xd3_stream *stream, xoff_t blkno)
2518 ret = stream->getblk (stream, source, blkno); 2518 ret = stream->getblk (stream, source, blkno);
2519 if (ret != 0) 2519 if (ret != 0)
2520 { 2520 {
2521 IF_DEBUG1 (DP(RINT "[getblk] app error: %s\n", xd3_strerror (ret)));
2521 return ret; 2522 return ret;
2522 } 2523 }
2523 } 2524 }
@@ -2581,14 +2582,15 @@ xd3_set_source (xd3_stream *stream,
2581 src->srclen = 0; 2582 src->srclen = 0;
2582 src->srcbase = 0; 2583 src->srcbase = 0;
2583 2584
2584 // If src->blksize is a power-of-two, xd3_blksize_div() will use 2585 /* Enforce power-of-two blocksize so that source-block number
2585 // shift and mask rather than divide. Check that here. 2586 * calculations are cheap. */
2586 if (!xd3_check_pow2 (src->blksize, &shiftby) == 0) 2587 if (!xd3_check_pow2 (src->blksize, &shiftby) == 0)
2587 { 2588 {
2588 int check; 2589 int check;
2589 src->blksize = xd3_pow2_roundup(src->blksize); 2590 src->blksize = xd3_pow2_roundup(src->blksize);
2590 check = xd3_check_pow2 (src->blksize, &shiftby); 2591 check = xd3_check_pow2 (src->blksize, &shiftby);
2591 XD3_ASSERT (check == 0); 2592 XD3_ASSERT (check == 0);
2593 IF_DEBUG1 (DP(RINT "raising srcblksz to %u\n", src->blksize));
2592 } 2594 }
2593 2595
2594 src->shiftby = shiftby; 2596 src->shiftby = shiftby;
diff --git a/xdelta3/xdelta3.h b/xdelta3/xdelta3.h
index c2741dd..bb7736e 100644
--- a/xdelta3/xdelta3.h
+++ b/xdelta3/xdelta3.h
@@ -1297,9 +1297,9 @@ const char* xd3_errstring (xd3_stream *stream)
1297} 1297}
1298 1298
1299 1299
1300/* 64-bit divisions are expensive, which is why we require a power-of-2 1300/* 64-bit divisions are expensive, which is why we require a
1301 * block size. To relax this restriction is relatively easy, see the history 1301 * power-of-two source->blksize. To relax this restriction is
1302 * for this method. gcc users should not see __udivdi3() in xd3_ profiles. */ 1302 * relatively easy, see the history for xd3_blksize_div(). */
1303static inline 1303static inline
1304void xd3_blksize_div (const xoff_t offset, 1304void xd3_blksize_div (const xoff_t offset,
1305 const xd3_source *source, 1305 const xd3_source *source,