diff options
author | josh.macdonald <jmacd@users.noreply.github.com> | 2011-01-09 07:42:42 +0000 |
---|---|---|
committer | josh.macdonald <jmacd@users.noreply.github.com> | 2011-01-09 07:42:42 +0000 |
commit | cc0cbbd791870eada49da8480979ddeabf45437f (patch) | |
tree | d384a45aa10e4a1161df545b203001b667b74185 | |
parent | d26de3993a0ff4927f9c1f380097f8b2ca9a511d (diff) |
Tests a directory of sample data. Using this now to verify behavior of
secondary compression for the new release.
-rw-r--r-- | xdelta3/testing/regtest.cc | 12 | ||||
-rwxr-xr-x | xdelta3/testing/xdelta3-regtest.py | 104 |
2 files changed, 80 insertions, 36 deletions
diff --git a/xdelta3/testing/regtest.cc b/xdelta3/testing/regtest.cc index ba2e42c..45d99d0 100644 --- a/xdelta3/testing/regtest.cc +++ b/xdelta3/testing/regtest.cc | |||
@@ -731,8 +731,8 @@ void TestMergeCommand1() { | |||
731 | continue; | 731 | continue; |
732 | } | 732 | } |
733 | 733 | ||
734 | DP(RINT "S0 = %lu\n", size0); | 734 | // DP(RINT "S0 = %lu\n", size0); |
735 | DP(RINT "C1 = %lu\n", change1); | 735 | // DP(RINT "C1 = %lu\n", change1); |
736 | 736 | ||
737 | size_t add1_pos = size0 ? rand.Rand32() % size0 : 0; | 737 | size_t add1_pos = size0 ? rand.Rand32() % size0 : 0; |
738 | size_t del2_pos = size0 ? rand.Rand32() % size0 : 0; | 738 | size_t del2_pos = size0 ? rand.Rand32() % size0 : 0; |
@@ -796,10 +796,10 @@ void TestMergeCommand2() { | |||
796 | continue; | 796 | continue; |
797 | } | 797 | } |
798 | 798 | ||
799 | DP(RINT "S0 = %lu\n", size0); | 799 | // DP(RINT "S0 = %lu\n", size0); |
800 | DP(RINT "S1 = %lu\n", size1); | 800 | // DP(RINT "S1 = %lu\n", size1); |
801 | DP(RINT "S2 = %lu\n", size2); | 801 | // DP(RINT "S2 = %lu\n", size2); |
802 | DP(RINT "S3 = %lu\n", size3); | 802 | // DP(RINT "S3 = %lu\n", size3); |
803 | 803 | ||
804 | spec0.GenerateFixedSize(size0); | 804 | spec0.GenerateFixedSize(size0); |
805 | 805 | ||
diff --git a/xdelta3/testing/xdelta3-regtest.py b/xdelta3/testing/xdelta3-regtest.py index d0a602f..f41660f 100755 --- a/xdelta3/testing/xdelta3-regtest.py +++ b/xdelta3/testing/xdelta3-regtest.py | |||
@@ -29,9 +29,8 @@ import xdelta3 | |||
29 | #RCSDIR = 'G:/jmacd/PRCS_copy' | 29 | #RCSDIR = 'G:/jmacd/PRCS_copy' |
30 | #SAMPLEDIR = "C:/sample_data/Wesnoth/tar" | 30 | #SAMPLEDIR = "C:/sample_data/Wesnoth/tar" |
31 | 31 | ||
32 | #RCSDIR = '/Users/jmacd/src/ftp.kernel.org/pub/scm/linux/kernel/bkcvs/linux-2.4/net/x25' | ||
33 | #RCSDIR = '/Users/jmacd/src/ftp.kernel.org/pub/scm/linux/kernel/bkcvs/linux-2.4/fs' | ||
34 | RCSDIR = '/Users/jmacd/src/ftp.kernel.org' | 32 | RCSDIR = '/Users/jmacd/src/ftp.kernel.org' |
33 | SAMPLEDIR = '/Users/jmacd/src/xdelta3/linux' | ||
35 | 34 | ||
36 | # | 35 | # |
37 | MIN_SIZE = 0 | 36 | MIN_SIZE = 0 |
@@ -47,7 +46,7 @@ MIN_STDDEV_PCT = 1.5 | |||
47 | 46 | ||
48 | # How many results per round | 47 | # How many results per round |
49 | MAX_RESULTS = 500 | 48 | MAX_RESULTS = 500 |
50 | TEST_ROUNDS = 500 | 49 | TEST_ROUNDS = 10 |
51 | KEEP_P = (0.5) | 50 | KEEP_P = (0.5) |
52 | 51 | ||
53 | # For RCS testing, what percent to select | 52 | # For RCS testing, what percent to select |
@@ -59,7 +58,7 @@ MAX_RUN = 1000 * 1000 * 10 | |||
59 | 58 | ||
60 | # Testwide defaults | 59 | # Testwide defaults |
61 | ALL_ARGS = [ | 60 | ALL_ARGS = [ |
62 | '-vv' | 61 | '-q' # '-vv' |
63 | ] | 62 | ] |
64 | 63 | ||
65 | # The first 7 args go to -C | 64 | # The first 7 args go to -C |
@@ -141,6 +140,8 @@ TMPDIR = '/tmp/xd3regtest.%d' % os.getpid() | |||
141 | RUNFILE = os.path.join(TMPDIR, 'run') | 140 | RUNFILE = os.path.join(TMPDIR, 'run') |
142 | DFILE = os.path.join(TMPDIR, 'output') | 141 | DFILE = os.path.join(TMPDIR, 'output') |
143 | RFILE = os.path.join(TMPDIR, 'recon') | 142 | RFILE = os.path.join(TMPDIR, 'recon') |
143 | CMPTMP1 = os.path.join(TMPDIR, 'cmptmp1') | ||
144 | CMPTMP2 = os.path.join(TMPDIR, 'cmptmp2') | ||
144 | 145 | ||
145 | HEAD_STATE = 0 | 146 | HEAD_STATE = 0 |
146 | BAR_STATE = 1 | 147 | BAR_STATE = 1 |
@@ -178,16 +179,21 @@ class StatList: | |||
178 | self.l = l | 179 | self.l = l |
179 | self.total = SumList(l) | 180 | self.total = SumList(l) |
180 | self.mean = self.total / float(self.cnt) | 181 | self.mean = self.total / float(self.cnt) |
181 | self.s = math.sqrt(SumList([(x-self.mean) * (x - self.mean) for x in l]) / float(self.cnt-1)) | 182 | self.s = math.sqrt(SumList([(x-self.mean) * |
183 | (x - self.mean) for x in l]) / | ||
184 | float(self.cnt-1)) | ||
182 | self.q0 = l[0] | 185 | self.q0 = l[0] |
183 | self.q1 = l[int(self.cnt/4.0+0.5)] | 186 | self.q1 = l[int(self.cnt/4.0+0.5)] |
184 | self.q2 = l[int(self.cnt/2.0+0.5)] | 187 | self.q2 = l[int(self.cnt/2.0+0.5)] |
185 | self.q3 = l[min(self.cnt-1,int((3.0*self.cnt)/4.0+0.5))] | 188 | self.q3 = l[min(self.cnt-1,int((3.0*self.cnt)/4.0+0.5))] |
186 | self.q4 = l[self.cnt-1]+1 | 189 | self.q4 = l[self.cnt-1] |
187 | self.siqr = (self.q3-self.q1)/2.0; | 190 | self.siqr = (self.q3-self.q1)/2.0; |
188 | self.spread = (self.q4-self.q0) | 191 | self.spread = (self.q4-self.q0) |
189 | self.str = '%s %d; mean %d; sdev %d; q2 %d; .5(q3-q1) %.1f; spread %d' % \ | 192 | if len(l) == 1: |
190 | (desc, self.total, self.mean, self.s, self.q2, self.siqr, self.spread) | 193 | self.str = '%s %s' % (desc, l[0]) |
194 | else: | ||
195 | self.str = '%s mean %.1f: 25%-ile %d %d %d %d %d' % \ | ||
196 | (desc, self.mean, self.q0, self.q1, self.q2, self.q3, self.q4) | ||
191 | #end | 197 | #end |
192 | #end | 198 | #end |
193 | 199 | ||
@@ -230,14 +236,14 @@ class TimedTest: | |||
230 | self.min_stddev_pct = min_stddev_pct | 236 | self.min_stddev_pct = min_stddev_pct |
231 | 237 | ||
232 | self.encode_time = self.DoTest(DFILE, | 238 | self.encode_time = self.DoTest(DFILE, |
233 | lambda x: x.Encode(self.target, self.source, DFILE)) | 239 | lambda x: x.Encode(self.target, |
240 | self.source, DFILE)) | ||
234 | self.encode_size = runnable.EncodeSize(DFILE) | 241 | self.encode_size = runnable.EncodeSize(DFILE) |
235 | 242 | ||
236 | self.decode_time = self.DoTest(RFILE, | 243 | self.decode_time = self.DoTest(RFILE, |
237 | lambda x: x.Decode(DFILE, self.source, RFILE), | 244 | lambda x: x.Decode(DFILE, |
245 | self.source, RFILE), | ||
238 | ) | 246 | ) |
239 | |||
240 | # verify | ||
241 | runnable.Verify(self.target, RFILE) | 247 | runnable.Verify(self.target, RFILE) |
242 | #end | 248 | #end |
243 | 249 | ||
@@ -364,7 +370,10 @@ class Xdelta3RunClass: | |||
364 | #end | 370 | #end |
365 | 371 | ||
366 | class Xdelta3Runner: | 372 | class Xdelta3Runner: |
367 | def __init__(self, extra): | 373 | # Use "forkexec" to get special command-line only features like |
374 | # external compression support. | ||
375 | def __init__(self, extra, forkexec=False): | ||
376 | self.forkexec = forkexec | ||
368 | self.extra = extra | 377 | self.extra = extra |
369 | #end | 378 | #end |
370 | 379 | ||
@@ -392,7 +401,12 @@ class Xdelta3Runner: | |||
392 | #end | 401 | #end |
393 | 402 | ||
394 | def Verify(self, target, recon): | 403 | def Verify(self, target, recon): |
395 | RunCommand(('cmp', target, recon)) | 404 | if target[-3:] == ".gz": |
405 | RunCommandIO(('gzip', '-dc'), target, CMPTMP1) | ||
406 | RunCommandIO(('gzip', '-dc'), recon, CMPTMP2) | ||
407 | RunCommand(('cmp', CMPTMP1, CMPTMP2)) | ||
408 | else: | ||
409 | RunCommand(('cmp', target, recon)) | ||
396 | #end | 410 | #end |
397 | 411 | ||
398 | def EncodeSize(self, output): | 412 | def EncodeSize(self, output): |
@@ -401,8 +415,10 @@ class Xdelta3Runner: | |||
401 | 415 | ||
402 | def Main(self, args): | 416 | def Main(self, args): |
403 | try: | 417 | try: |
404 | #print 'Run %s' % (' '.join(args)) | 418 | if self.forkexec: |
405 | xdelta3.xd3_main_cmdline(args) | 419 | RunCommand(['../xdelta3'] + args) |
420 | else: | ||
421 | xdelta3.xd3_main_cmdline(args) | ||
406 | except Exception, e: | 422 | except Exception, e: |
407 | raise CommandError(args, "xdelta3.main exception: %s" % e) | 423 | raise CommandError(args, "xdelta3.main exception: %s" % e) |
408 | #end | 424 | #end |
@@ -816,10 +832,11 @@ def GetTestRcsFiles(): | |||
816 | raise CommandError('', 'no RCS files') | 832 | raise CommandError('', 'no RCS files') |
817 | #end | 833 | #end |
818 | rcsf.Summarize() | 834 | rcsf.Summarize() |
819 | print "rcsfiles: rcsfiles %d; subdirs %d; others %d; skipped %d" % (len(rcsf.rcsfiles), | 835 | print "rcsfiles: rcsfiles %d; subdirs %d; others %d; skipped %d" % ( |
820 | len(rcsf.subdirs), | 836 | len(rcsf.rcsfiles), |
821 | len(rcsf.others), | 837 | len(rcsf.subdirs), |
822 | len(rcsf.skipped)) | 838 | len(rcsf.others), |
839 | len(rcsf.skipped)) | ||
823 | print StatList([x.rcssize for x in rcsf.rcsfiles], "rcssize").str | 840 | print StatList([x.rcssize for x in rcsf.rcsfiles], "rcssize").str |
824 | print StatList([x.totrev for x in rcsf.rcsfiles], "totrev").str | 841 | print StatList([x.totrev for x in rcsf.rcsfiles], "totrev").str |
825 | return rcsf | 842 | return rcsf |
@@ -827,6 +844,7 @@ def GetTestRcsFiles(): | |||
827 | 844 | ||
828 | class SampleDataTest: | 845 | class SampleDataTest: |
829 | def __init__(self, dirs): | 846 | def __init__(self, dirs): |
847 | dirs_in = dirs | ||
830 | self.pairs = [] | 848 | self.pairs = [] |
831 | while dirs: | 849 | while dirs: |
832 | d = dirs[0] | 850 | d = dirs[0] |
@@ -843,12 +861,16 @@ class SampleDataTest: | |||
843 | #end | 861 | #end |
844 | if len(files) > 1: | 862 | if len(files) > 1: |
845 | files.sort() | 863 | files.sort() |
846 | for x in xrange(len(files) - 1): | 864 | for x in xrange(len(files)): |
847 | self.pairs.append((files[x], files[x+1], | 865 | for y in xrange(len(files)): |
848 | '%s-%s' % (files[x], files[x+1]))) | 866 | self.pairs.append((files[x], files[y], |
867 | '%s-%s' % (files[x], files[y]))) | ||
868 | #end | ||
849 | #end | 869 | #end |
850 | #end | 870 | #end |
851 | #end | 871 | #end |
872 | print "Sample data test using %d file pairs in %s" % ( | ||
873 | len(self.pairs), dirs_in) | ||
852 | #end | 874 | #end |
853 | 875 | ||
854 | def Generator(self): | 876 | def Generator(self): |
@@ -978,7 +1000,7 @@ def RandomTestConfigs(rand, input_configs, num_results): | |||
978 | return outputs | 1000 | return outputs |
979 | #end | 1001 | #end |
980 | 1002 | ||
981 | def RunTestLoop(rand, generator, rounds): | 1003 | def RunOptimizationLoop(rand, generator, rounds): |
982 | configs = [] | 1004 | configs = [] |
983 | for rnum in xrange(rounds): | 1005 | for rnum in xrange(rounds): |
984 | configs = RandomTestConfigs(rand, configs, MAX_RESULTS) | 1006 | configs = RandomTestConfigs(rand, configs, MAX_RESULTS) |
@@ -1192,28 +1214,50 @@ def GraphSummary(desc, results_ignore): | |||
1192 | #end | 1214 | #end |
1193 | #end | 1215 | #end |
1194 | 1216 | ||
1217 | def RunRegressionTest(pairs, rounds): | ||
1218 | for args in [ | ||
1219 | [], | ||
1220 | ['-S djw'], | ||
1221 | |||
1222 | ]: | ||
1223 | print "Args %s" % (args) | ||
1224 | for (file1, file2, testkey) in pairs: | ||
1225 | ttest = TimedTest(file1, file2, Xdelta3Runner(args, forkexec=True), | ||
1226 | skip_trials = 0, | ||
1227 | min_trials = 1, | ||
1228 | max_trials = 1) | ||
1229 | print "Source %s\nTarget %s\nEncode %s\nDecode %s\nSize %s\n\n" % ( | ||
1230 | file1, file2, | ||
1231 | ttest.encode_time.str, | ||
1232 | ttest.decode_time.str, | ||
1233 | ttest.encode_size) | ||
1234 | #end | ||
1235 | #end | ||
1236 | |||
1195 | if __name__ == "__main__": | 1237 | if __name__ == "__main__": |
1196 | try: | 1238 | try: |
1197 | RunCommand(['rm', '-rf', TMPDIR]) | 1239 | RunCommand(['rm', '-rf', TMPDIR]) |
1198 | os.mkdir(TMPDIR) | 1240 | os.mkdir(TMPDIR) |
1199 | 1241 | ||
1200 | rcsf = GetTestRcsFiles() | 1242 | #rcsf = GetTestRcsFiles() |
1201 | generator = rcsf.Generator() | 1243 | #generator = rcsf.Generator() |
1202 | 1244 | ||
1203 | #sample = SampleDataTest([SAMPLEDIR]) | 1245 | sample = SampleDataTest([SAMPLEDIR]) |
1204 | #generator = sample.Generator() | 1246 | generator = sample.Generator() |
1205 | 1247 | ||
1206 | rand = random.Random(135135135135135) | 1248 | rand = random.Random(135135135135135) |
1207 | RunTestLoop(rand, generator, TEST_ROUNDS) | 1249 | |
1250 | RunRegressionTest(sample.pairs, TEST_ROUNDS) | ||
1208 | 1251 | ||
1209 | #RunSpeedTest() | 1252 | #RunSpeedTest() |
1210 | 1253 | ||
1254 | # the idea below is to add the default configurations and | ||
1255 | # xdelta1 to the optimization loop: | ||
1211 | #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-1', '-3', '-6'])) | 1256 | #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-1', '-3', '-6'])) |
1212 | #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9'])) | 1257 | #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9'])) |
1213 | #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-S', 'djw'])) | 1258 | #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-S', 'djw'])) |
1214 | #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-1', '-S', 'djw'])) | 1259 | #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-1', '-S', 'djw'])) |
1215 | #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-T'])) | 1260 | #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-T'])) |
1216 | |||
1217 | #x1r = rcsf.AllPairsByDate(Xdelta1RunClass()) | 1261 | #x1r = rcsf.AllPairsByDate(Xdelta1RunClass()) |
1218 | 1262 | ||
1219 | except CommandError: | 1263 | except CommandError: |