diff options
author | josh.macdonald <jmacd@users.noreply.github.com> | 2010-11-07 07:44:13 +0000 |
---|---|---|
committer | josh.macdonald <jmacd@users.noreply.github.com> | 2010-11-07 07:44:13 +0000 |
commit | 176e2509eb0264d14b473c6e036daa5428b0bac7 (patch) | |
tree | 93bdb6a711a368aa98cc916675d9634adccdbce0 | |
parent | 46593c36a012edda887a0072cf93949086f33b75 (diff) |
Clean up and simplify main_set_source()
-rw-r--r-- | xdelta3/xdelta3-blkcache.h | 216 | ||||
-rw-r--r-- | xdelta3/xdelta3-main.h | 2 |
2 files changed, 101 insertions, 117 deletions
diff --git a/xdelta3/xdelta3-blkcache.h b/xdelta3/xdelta3-blkcache.h index d01322e..6fbb8dd 100644 --- a/xdelta3/xdelta3-blkcache.h +++ b/xdelta3/xdelta3-blkcache.h | |||
@@ -18,6 +18,9 @@ | |||
18 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | 18 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
19 | */ | 19 | */ |
20 | 20 | ||
21 | /* TODO: This code is heavily revised from 3.0z but still needs major | ||
22 | * refactoring. */ | ||
23 | |||
21 | typedef struct _main_blklru main_blklru; | 24 | typedef struct _main_blklru main_blklru; |
22 | typedef struct _main_blklru_list main_blklru_list; | 25 | typedef struct _main_blklru_list main_blklru_list; |
23 | 26 | ||
@@ -35,8 +38,9 @@ struct _main_blklru | |||
35 | main_blklru_list link; | 38 | main_blklru_list link; |
36 | }; | 39 | }; |
37 | 40 | ||
38 | #define MAX_LRU_SIZE 32U | 41 | /* TODO: The value for MAX_LRU_SIZE has what significance? */ |
39 | #define XD3_MINSRCWINSZ XD3_ALLOCSIZE | 42 | #define MAX_LRU_SIZE 32U /* must be a power of 2 */ |
43 | #define XD3_MINSRCWINSZ (XD3_ALLOCSIZE * MAX_LRU_SIZE) | ||
40 | 44 | ||
41 | XD3_MAKELIST(main_blklru_list,main_blklru,link); | 45 | XD3_MAKELIST(main_blklru_list,main_blklru,link); |
42 | 46 | ||
@@ -62,13 +66,9 @@ static void main_lru_reset() | |||
62 | 66 | ||
63 | static void main_lru_cleanup() | 67 | static void main_lru_cleanup() |
64 | { | 68 | { |
65 | int i; | 69 | if (lru != NULL) |
66 | /* TODO(jmacd): HERE YOU ARE | ||
67 | * Remove this loop, free only lru[0].blk. | ||
68 | */ | ||
69 | for (i = 0; lru && i < lru_size; i += 1) | ||
70 | { | 70 | { |
71 | main_free (lru[i].blk); | 71 | main_free (lru[0].blk); |
72 | } | 72 | } |
73 | 73 | ||
74 | main_free (lru); | 74 | main_free (lru); |
@@ -90,112 +90,108 @@ main_set_source (xd3_stream *stream, xd3_cmd cmd, | |||
90 | usize_t i; | 90 | usize_t i; |
91 | usize_t blksize; | 91 | usize_t blksize; |
92 | xoff_t source_size = 0; | 92 | xoff_t source_size = 0; |
93 | main_blklru block0; | ||
94 | 93 | ||
95 | XD3_ASSERT (lru == NULL); | 94 | XD3_ASSERT (lru == NULL); |
96 | XD3_ASSERT (stream->src == NULL); | 95 | XD3_ASSERT (stream->src == NULL); |
97 | XD3_ASSERT (option_srcwinsz >= XD3_MINSRCWINSZ); | 96 | XD3_ASSERT (option_srcwinsz >= XD3_MINSRCWINSZ); |
98 | 97 | ||
98 | /* TODO: this code needs refactoring into FIFO, LRU, FAKE. Yuck! | ||
99 | * This is simplified from 3.0z which had issues with sizing the | ||
100 | * source buffer memory allocation and the source blocksize. */ | ||
101 | |||
102 | /* LRU-specific */ | ||
99 | main_blklru_list_init (& lru_list); | 103 | main_blklru_list_init (& lru_list); |
100 | main_blklru_list_init (& lru_free); | 104 | main_blklru_list_init (& lru_free); |
101 | 105 | ||
102 | if (allow_fake_source) | 106 | if (allow_fake_source) |
103 | { | 107 | { |
108 | /* TOOLS-specific (e.g., "printdelta"). Note: Check | ||
109 | * "allow_fake_source" mode looks broken now. */ | ||
104 | sfile->mode = XO_READ; | 110 | sfile->mode = XO_READ; |
105 | sfile->realname = sfile->filename; | 111 | sfile->realname = sfile->filename; |
106 | sfile->nread = 0; | 112 | sfile->nread = 0; |
107 | } | 113 | } |
108 | else | 114 | else |
109 | { | 115 | { |
116 | /* Either a regular file (possibly compressed) or a FIFO | ||
117 | * (possibly compressed). */ | ||
110 | if ((ret = main_file_open (sfile, sfile->filename, XO_READ))) | 118 | if ((ret = main_file_open (sfile, sfile->filename, XO_READ))) |
111 | { | 119 | { |
112 | return ret; | 120 | return ret; |
113 | } | 121 | } |
114 | 122 | ||
115 | /* Allow non-seekable sources from the start. If the file | 123 | /* If the file is regular we know it's size. If the file turns |
116 | * turns out to be externally compressed, size_known may change. */ | 124 | * out to be externally compressed, size_known may change. */ |
117 | sfile->size_known = (main_file_stat (sfile, &source_size) == 0); | 125 | sfile->size_known = (main_file_stat (sfile, &source_size) == 0); |
118 | } | 126 | } |
119 | 127 | ||
120 | /* The API requires power-of-two blocksize, */ | 128 | /* Note: The API requires a power-of-two blocksize and srcwinsz |
121 | blksize = xd3_pow2_roundup (max (option_srcwinsz / MAX_LRU_SIZE, | 129 | * (-B). The logic here will use a single block if the entire file |
122 | XD3_MINSRCWINSZ)); | 130 | * is known to fit into srcwinsz. */ |
123 | 131 | option_srcwinsz = xd3_pow2_roundup (option_srcwinsz); | |
124 | /* TODO(jmacd): The organization of this code and the implementation | 132 | |
125 | * of the LRU cache could be improved. This is too convoluted, the | 133 | /* Though called "lru", it is not LRU-specific. We always allocate |
126 | * code uses main_getblk_func() to read the first block, which may | 134 | * a maximum number of source block buffers. If the entire file |
127 | * trigger external-decompression, in large part so that the | 135 | * fits into srcwinsz, this buffer will stay as the only |
128 | * verbose-printing and counters maintained by getblk_func are | 136 | * (lru_size==1) source block. Otherwise, we know that at least |
129 | * consistent. There used to be additional optimizations going on | 137 | * option_srcwinsz bytes are available. Split the source window |
130 | * here: (1) if the source size is known we would like to lower | 138 | * into buffers. */ |
131 | * option_srcwinsz, if possible, (2) avoid allocating more memory | 139 | if ((lru = (main_blklru*) main_malloc (MAX_LRU_SIZE * |
132 | * than needed, and (3) also want to use a single block when | 140 | sizeof (main_blklru))) == NULL) |
133 | * source_size < option_srcwinsz, this because compression is better | ||
134 | * for larger blocks (especially due to the blockwise-reverse | ||
135 | * insertion of checksums). | ||
136 | * | ||
137 | * (3) is no longer implemented. (2) is only implemented for files | ||
138 | * that are shorter than the default blocksize, and (1) is not | ||
139 | * implemented. These optimizations are not taken because the code | ||
140 | * is already too complicated. | ||
141 | * | ||
142 | * The ideal solution may be to allocate a block of memory equal to | ||
143 | * half of the option_srcwinsz. Read as much data as possible into | ||
144 | * that block. If the entire file fits, pass one block to the | ||
145 | * library for best compression. If not, copy the 50% block into | ||
146 | * smaller (option_srcwinsz / MAX_LRU_SIZE) blocks and proceed. Too | ||
147 | * much effort for too little payback. */ | ||
148 | |||
149 | memset (&block0, 0, sizeof (block0)); | ||
150 | block0.blkno = (xoff_t) -1; | ||
151 | |||
152 | /* TODO(jmacd): HERE YOU ARE | ||
153 | * Allocate only one block of option_srcwinsz. | ||
154 | */ | ||
155 | /* Allocate the first block. Even if the size is known at this | ||
156 | * point, we do not lower it because decompression may happen. */ | ||
157 | if ((block0.blk = (uint8_t*) main_malloc (blksize)) == NULL) | ||
158 | { | 141 | { |
159 | ret = ENOMEM; | 142 | ret = ENOMEM; |
160 | return ret; | 143 | return ret; |
161 | } | 144 | } |
162 | 145 | ||
163 | source->blksize = blksize; | 146 | memset (lru, 0, sizeof(lru[0]) * MAX_LRU_SIZE); |
164 | source->name = sfile->filename; | 147 | |
165 | source->ioh = sfile; | 148 | /* Allocate the entire buffer. */ |
166 | source->curblkno = (xoff_t) -1; | 149 | if ((lru[0].blk = (uint8_t*) main_malloc (option_srcwinsz)) == NULL) |
167 | source->curblk = NULL; | 150 | { |
151 | ret = ENOMEM; | ||
152 | return ret; | ||
153 | } | ||
168 | 154 | ||
169 | /* We have to read the first block into the cache now, because | 155 | /* Main calls main_getblk_func() once before xd3_set_source(). This |
170 | * size_known can still change (due to secondary decompression). | 156 | * is the point at which external decompression may begin. Set the |
171 | * Calls main_secondary_decompress_check() via | 157 | * system for a single block. */ |
172 | * main_read_primary_input(). */ | ||
173 | lru_size = 1; | 158 | lru_size = 1; |
174 | lru = &block0; | 159 | lru[0].blkno = -1; |
160 | blksize = option_srcwinsz; | ||
175 | XD3_ASSERT (main_blklru_list_empty (& lru_free)); | 161 | XD3_ASSERT (main_blklru_list_empty (& lru_free)); |
176 | XD3_ASSERT (main_blklru_list_empty (& lru_list)); | 162 | XD3_ASSERT (main_blklru_list_empty (& lru_list)); |
177 | main_blklru_list_push_back (& lru_free, & lru[0]); | ||
178 | /* This call is partly so that the diagnostics printed by | ||
179 | * this function appear to happen normally for the first read, | ||
180 | * which is special. */ | ||
181 | ret = main_getblk_func (stream, source, 0); | ||
182 | main_blklru_list_remove (& lru[0]); | ||
183 | XD3_ASSERT (main_blklru_list_empty (& lru_free)); | ||
184 | XD3_ASSERT (main_blklru_list_empty (& lru_list)); | ||
185 | lru = NULL; | ||
186 | 163 | ||
187 | if (ret != 0) | 164 | /* TODO: Refactor. */ |
165 | if (! do_src_fifo) | ||
188 | { | 166 | { |
189 | main_free (block0.blk); | 167 | main_blklru_list_push_back (& lru_free, & lru[0]); |
168 | } | ||
169 | |||
170 | /* Initialize xd3_source. */ | ||
171 | source->blksize = blksize; | ||
172 | source->name = sfile->filename; | ||
173 | source->ioh = sfile; | ||
174 | source->curblkno = (xoff_t) -1; | ||
175 | source->curblk = NULL; | ||
190 | 176 | ||
177 | if ((ret = main_getblk_func (stream, source, 0)) != 0) | ||
178 | { | ||
191 | XPR(NT "error reading source: %s: %s\n", | 179 | XPR(NT "error reading source: %s: %s\n", |
192 | sfile->filename, | 180 | sfile->filename, |
193 | xd3_mainerror (ret)); | 181 | xd3_mainerror (ret)); |
194 | |||
195 | return ret; | 182 | return ret; |
196 | } | 183 | } |
197 | 184 | ||
198 | source->onblk = block0.size; | 185 | /* TODO: Refactor */ |
186 | if (!do_src_fifo) | ||
187 | { | ||
188 | XD3_ASSERT (!main_blklru_list_empty (& lru_list)); | ||
189 | main_blklru_list_remove (& lru[0]); | ||
190 | } | ||
191 | XD3_ASSERT (main_blklru_list_empty (& lru_free)); | ||
192 | XD3_ASSERT (main_blklru_list_empty (& lru_list)); | ||
193 | |||
194 | source->onblk = lru[0].size; /* xd3 sets onblk */ | ||
199 | 195 | ||
200 | /* If the file is smaller than a block, size is known. */ | 196 | /* If the file is smaller than a block, size is known. */ |
201 | if (!sfile->size_known && source->onblk < blksize) | 197 | if (!sfile->size_known && source->onblk < blksize) |
@@ -204,56 +200,44 @@ main_set_source (xd3_stream *stream, xd3_cmd cmd, | |||
204 | sfile->size_known = 1; | 200 | sfile->size_known = 1; |
205 | } | 201 | } |
206 | 202 | ||
207 | /* We update lru_size accordingly, and modify option_srcwinsz, which | 203 | /* If the size is not known or is greater than the buffer size, we |
208 | * will be passed via xd3_config. */ | 204 | * split the buffer across MAX_LRU_SIZE blocks (already allocated in |
209 | if (sfile->size_known && source_size <= option_srcwinsz) | 205 | * "lru"). */ |
206 | if (!sfile->size_known || source_size > option_srcwinsz) | ||
210 | { | 207 | { |
211 | lru_size = (usize_t) ((source_size + blksize - 1) / blksize); | 208 | /* Modify block 0, change blocksize. */ |
212 | } | 209 | lru_size = MAX_LRU_SIZE; |
213 | else | 210 | blksize = option_srcwinsz / MAX_LRU_SIZE; /* Power of 2 */ |
214 | { | 211 | source->blksize = blksize; |
215 | lru_size = (option_srcwinsz + blksize - 1) / blksize; | 212 | source->onblk = blksize; /* xd3 sets onblk */ |
216 | } | 213 | lru[0].size = blksize; |
217 | 214 | ||
218 | XD3_ASSERT (lru_size >= 1); | 215 | /* Setup rest of blocks. */ |
219 | option_srcwinsz = lru_size * blksize; | 216 | for (i = 1; i < lru_size; i += 1) |
220 | 217 | { | |
221 | if ((lru = (main_blklru*) main_malloc (lru_size * sizeof (main_blklru))) | 218 | lru[i].blk = lru[0].blk + (blksize * i); |
222 | == NULL) | 219 | lru[i].blkno = i; |
223 | { | 220 | lru[i].size = blksize; |
224 | ret = ENOMEM; | 221 | } |
225 | return ret; | ||
226 | } | 222 | } |
227 | 223 | ||
228 | lru[0].blk = block0.blk; | ||
229 | lru[0].blkno = 0; | ||
230 | lru[0].size = source->onblk; | ||
231 | |||
232 | if (! sfile->size_known) | 224 | if (! sfile->size_known) |
233 | { | 225 | { |
226 | /* If the size is not know, we must use FIFO discipline. */ | ||
234 | do_src_fifo = 1; | 227 | do_src_fifo = 1; |
235 | } | 228 | } |
236 | else if (! do_src_fifo) | 229 | else if (! do_src_fifo) |
237 | { | 230 | { |
238 | main_blklru_list_push_back (& lru_list, & lru[0]); | 231 | /* TODO: Refactor. |
239 | } | 232 | * The encoder forces FIFO, otherwise initialize the LRU data structures */ |
240 | 233 | for (i = 0; i < lru_size; ++i) | |
241 | for (i = 1; i < lru_size; i += 1) | ||
242 | { | ||
243 | lru[i].blkno = (xoff_t) -1; | ||
244 | |||
245 | if ((lru[i].blk = (uint8_t*) main_malloc (source->blksize)) == NULL) | ||
246 | { | ||
247 | ret = ENOMEM; | ||
248 | return ret; | ||
249 | } | ||
250 | |||
251 | if (! do_src_fifo) | ||
252 | { | 234 | { |
253 | main_blklru_list_push_back (& lru_free, & lru[i]); | 235 | main_blklru_list_push_back (& lru_list, & lru[i]); |
254 | } | 236 | } |
255 | } | 237 | } |
256 | 238 | ||
239 | /* Call the appropriate set_source method, handle errors, print | ||
240 | * verbose message, etc. */ | ||
257 | if (sfile->size_known) | 241 | if (sfile->size_known) |
258 | { | 242 | { |
259 | ret = xd3_set_source_and_size (stream, source, source_size); | 243 | ret = xd3_set_source_and_size (stream, source, source_size); |
@@ -580,21 +564,21 @@ main_getblk_func (xd3_stream *stream, | |||
580 | { | 564 | { |
581 | if (blru->blkno != blkno) | 565 | if (blru->blkno != blkno) |
582 | { | 566 | { |
583 | XPR(NT "source block %"Q"u ejects %"Q"u (lru_hits=%u, " | 567 | XPR(NT "source block %"Q"u read %u ejects %"Q"u (lru_hits=%u, " |
584 | "lru_misses=%u, lru_filled=%u)\n", | 568 | "lru_misses=%u, lru_filled=%u)\n", |
585 | blkno, blru->blkno, lru_hits, lru_misses, lru_filled); | 569 | blkno, nread, blru->blkno, lru_hits, lru_misses, lru_filled); |
586 | } | 570 | } |
587 | else | 571 | else |
588 | { | 572 | { |
589 | XPR(NT "source block %"Q"u read (lru_hits=%u, " | 573 | XPR(NT "source block %"Q"u read %u (lru_hits=%u, " |
590 | "lru_misses=%u, lru_filled=%u)\n", | 574 | "lru_misses=%u, lru_filled=%u)\n", |
591 | blkno, lru_hits, lru_misses, lru_filled); | 575 | blkno, nread, lru_hits, lru_misses, lru_filled); |
592 | } | 576 | } |
593 | } | 577 | } |
594 | else | 578 | else |
595 | { | 579 | { |
596 | XPR(NT "source block %"Q"u read (lru_hits=%u, lru_misses=%u, " | 580 | XPR(NT "source block %"Q"u read %u (lru_hits=%u, lru_misses=%u, " |
597 | "lru_filled=%u)\n", blkno, lru_hits, lru_misses, lru_filled); | 581 | "lru_filled=%u)\n", blkno, nread, lru_hits, lru_misses, lru_filled); |
598 | } | 582 | } |
599 | } | 583 | } |
600 | 584 | ||
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h index e47d708..62018e0 100644 --- a/xdelta3/xdelta3-main.h +++ b/xdelta3/xdelta3-main.h | |||
@@ -3034,7 +3034,7 @@ main_input (xd3_cmd cmd, | |||
3034 | 3034 | ||
3035 | #if XD3_ENCODER | 3035 | #if XD3_ENCODER |
3036 | case CMD_ENCODE: | 3036 | case CMD_ENCODE: |
3037 | do_src_fifo = 1; | 3037 | do_src_fifo = 1; |
3038 | input_func = xd3_encode_input; | 3038 | input_func = xd3_encode_input; |
3039 | output_func = main_write_output; | 3039 | output_func = main_write_output; |
3040 | 3040 | ||