summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--xdelta3/testing/regtest.cc12
-rwxr-xr-xxdelta3/testing/xdelta3-regtest.py104
2 files changed, 80 insertions, 36 deletions
diff --git a/xdelta3/testing/regtest.cc b/xdelta3/testing/regtest.cc
index ba2e42c..45d99d0 100644
--- a/xdelta3/testing/regtest.cc
+++ b/xdelta3/testing/regtest.cc
@@ -731,8 +731,8 @@ void TestMergeCommand1() {
731 continue; 731 continue;
732 } 732 }
733 733
734 DP(RINT "S0 = %lu\n", size0); 734 // DP(RINT "S0 = %lu\n", size0);
735 DP(RINT "C1 = %lu\n", change1); 735 // DP(RINT "C1 = %lu\n", change1);
736 736
737 size_t add1_pos = size0 ? rand.Rand32() % size0 : 0; 737 size_t add1_pos = size0 ? rand.Rand32() % size0 : 0;
738 size_t del2_pos = size0 ? rand.Rand32() % size0 : 0; 738 size_t del2_pos = size0 ? rand.Rand32() % size0 : 0;
@@ -796,10 +796,10 @@ void TestMergeCommand2() {
796 continue; 796 continue;
797 } 797 }
798 798
799 DP(RINT "S0 = %lu\n", size0); 799 // DP(RINT "S0 = %lu\n", size0);
800 DP(RINT "S1 = %lu\n", size1); 800 // DP(RINT "S1 = %lu\n", size1);
801 DP(RINT "S2 = %lu\n", size2); 801 // DP(RINT "S2 = %lu\n", size2);
802 DP(RINT "S3 = %lu\n", size3); 802 // DP(RINT "S3 = %lu\n", size3);
803 803
804 spec0.GenerateFixedSize(size0); 804 spec0.GenerateFixedSize(size0);
805 805
diff --git a/xdelta3/testing/xdelta3-regtest.py b/xdelta3/testing/xdelta3-regtest.py
index d0a602f..f41660f 100755
--- a/xdelta3/testing/xdelta3-regtest.py
+++ b/xdelta3/testing/xdelta3-regtest.py
@@ -29,9 +29,8 @@ import xdelta3
29#RCSDIR = 'G:/jmacd/PRCS_copy' 29#RCSDIR = 'G:/jmacd/PRCS_copy'
30#SAMPLEDIR = "C:/sample_data/Wesnoth/tar" 30#SAMPLEDIR = "C:/sample_data/Wesnoth/tar"
31 31
32#RCSDIR = '/Users/jmacd/src/ftp.kernel.org/pub/scm/linux/kernel/bkcvs/linux-2.4/net/x25'
33#RCSDIR = '/Users/jmacd/src/ftp.kernel.org/pub/scm/linux/kernel/bkcvs/linux-2.4/fs'
34RCSDIR = '/Users/jmacd/src/ftp.kernel.org' 32RCSDIR = '/Users/jmacd/src/ftp.kernel.org'
33SAMPLEDIR = '/Users/jmacd/src/xdelta3/linux'
35 34
36# 35#
37MIN_SIZE = 0 36MIN_SIZE = 0
@@ -47,7 +46,7 @@ MIN_STDDEV_PCT = 1.5
47 46
48# How many results per round 47# How many results per round
49MAX_RESULTS = 500 48MAX_RESULTS = 500
50TEST_ROUNDS = 500 49TEST_ROUNDS = 10
51KEEP_P = (0.5) 50KEEP_P = (0.5)
52 51
53# For RCS testing, what percent to select 52# For RCS testing, what percent to select
@@ -59,7 +58,7 @@ MAX_RUN = 1000 * 1000 * 10
59 58
60# Testwide defaults 59# Testwide defaults
61ALL_ARGS = [ 60ALL_ARGS = [
62 '-vv' 61 '-q' # '-vv'
63 ] 62 ]
64 63
65# The first 7 args go to -C 64# The first 7 args go to -C
@@ -141,6 +140,8 @@ TMPDIR = '/tmp/xd3regtest.%d' % os.getpid()
141RUNFILE = os.path.join(TMPDIR, 'run') 140RUNFILE = os.path.join(TMPDIR, 'run')
142DFILE = os.path.join(TMPDIR, 'output') 141DFILE = os.path.join(TMPDIR, 'output')
143RFILE = os.path.join(TMPDIR, 'recon') 142RFILE = os.path.join(TMPDIR, 'recon')
143CMPTMP1 = os.path.join(TMPDIR, 'cmptmp1')
144CMPTMP2 = os.path.join(TMPDIR, 'cmptmp2')
144 145
145HEAD_STATE = 0 146HEAD_STATE = 0
146BAR_STATE = 1 147BAR_STATE = 1
@@ -178,16 +179,21 @@ class StatList:
178 self.l = l 179 self.l = l
179 self.total = SumList(l) 180 self.total = SumList(l)
180 self.mean = self.total / float(self.cnt) 181 self.mean = self.total / float(self.cnt)
181 self.s = math.sqrt(SumList([(x-self.mean) * (x - self.mean) for x in l]) / float(self.cnt-1)) 182 self.s = math.sqrt(SumList([(x-self.mean) *
183 (x - self.mean) for x in l]) /
184 float(self.cnt-1))
182 self.q0 = l[0] 185 self.q0 = l[0]
183 self.q1 = l[int(self.cnt/4.0+0.5)] 186 self.q1 = l[int(self.cnt/4.0+0.5)]
184 self.q2 = l[int(self.cnt/2.0+0.5)] 187 self.q2 = l[int(self.cnt/2.0+0.5)]
185 self.q3 = l[min(self.cnt-1,int((3.0*self.cnt)/4.0+0.5))] 188 self.q3 = l[min(self.cnt-1,int((3.0*self.cnt)/4.0+0.5))]
186 self.q4 = l[self.cnt-1]+1 189 self.q4 = l[self.cnt-1]
187 self.siqr = (self.q3-self.q1)/2.0; 190 self.siqr = (self.q3-self.q1)/2.0;
188 self.spread = (self.q4-self.q0) 191 self.spread = (self.q4-self.q0)
189 self.str = '%s %d; mean %d; sdev %d; q2 %d; .5(q3-q1) %.1f; spread %d' % \ 192 if len(l) == 1:
190 (desc, self.total, self.mean, self.s, self.q2, self.siqr, self.spread) 193 self.str = '%s %s' % (desc, l[0])
194 else:
195 self.str = '%s mean %.1f: 25%-ile %d %d %d %d %d' % \
196 (desc, self.mean, self.q0, self.q1, self.q2, self.q3, self.q4)
191 #end 197 #end
192#end 198#end
193 199
@@ -230,14 +236,14 @@ class TimedTest:
230 self.min_stddev_pct = min_stddev_pct 236 self.min_stddev_pct = min_stddev_pct
231 237
232 self.encode_time = self.DoTest(DFILE, 238 self.encode_time = self.DoTest(DFILE,
233 lambda x: x.Encode(self.target, self.source, DFILE)) 239 lambda x: x.Encode(self.target,
240 self.source, DFILE))
234 self.encode_size = runnable.EncodeSize(DFILE) 241 self.encode_size = runnable.EncodeSize(DFILE)
235 242
236 self.decode_time = self.DoTest(RFILE, 243 self.decode_time = self.DoTest(RFILE,
237 lambda x: x.Decode(DFILE, self.source, RFILE), 244 lambda x: x.Decode(DFILE,
245 self.source, RFILE),
238 ) 246 )
239
240 # verify
241 runnable.Verify(self.target, RFILE) 247 runnable.Verify(self.target, RFILE)
242 #end 248 #end
243 249
@@ -364,7 +370,10 @@ class Xdelta3RunClass:
364#end 370#end
365 371
366class Xdelta3Runner: 372class Xdelta3Runner:
367 def __init__(self, extra): 373 # Use "forkexec" to get special command-line only features like
374 # external compression support.
375 def __init__(self, extra, forkexec=False):
376 self.forkexec = forkexec
368 self.extra = extra 377 self.extra = extra
369 #end 378 #end
370 379
@@ -392,7 +401,12 @@ class Xdelta3Runner:
392 #end 401 #end
393 402
394 def Verify(self, target, recon): 403 def Verify(self, target, recon):
395 RunCommand(('cmp', target, recon)) 404 if target[-3:] == ".gz":
405 RunCommandIO(('gzip', '-dc'), target, CMPTMP1)
406 RunCommandIO(('gzip', '-dc'), recon, CMPTMP2)
407 RunCommand(('cmp', CMPTMP1, CMPTMP2))
408 else:
409 RunCommand(('cmp', target, recon))
396 #end 410 #end
397 411
398 def EncodeSize(self, output): 412 def EncodeSize(self, output):
@@ -401,8 +415,10 @@ class Xdelta3Runner:
401 415
402 def Main(self, args): 416 def Main(self, args):
403 try: 417 try:
404 #print 'Run %s' % (' '.join(args)) 418 if self.forkexec:
405 xdelta3.xd3_main_cmdline(args) 419 RunCommand(['../xdelta3'] + args)
420 else:
421 xdelta3.xd3_main_cmdline(args)
406 except Exception, e: 422 except Exception, e:
407 raise CommandError(args, "xdelta3.main exception: %s" % e) 423 raise CommandError(args, "xdelta3.main exception: %s" % e)
408 #end 424 #end
@@ -816,10 +832,11 @@ def GetTestRcsFiles():
816 raise CommandError('', 'no RCS files') 832 raise CommandError('', 'no RCS files')
817 #end 833 #end
818 rcsf.Summarize() 834 rcsf.Summarize()
819 print "rcsfiles: rcsfiles %d; subdirs %d; others %d; skipped %d" % (len(rcsf.rcsfiles), 835 print "rcsfiles: rcsfiles %d; subdirs %d; others %d; skipped %d" % (
820 len(rcsf.subdirs), 836 len(rcsf.rcsfiles),
821 len(rcsf.others), 837 len(rcsf.subdirs),
822 len(rcsf.skipped)) 838 len(rcsf.others),
839 len(rcsf.skipped))
823 print StatList([x.rcssize for x in rcsf.rcsfiles], "rcssize").str 840 print StatList([x.rcssize for x in rcsf.rcsfiles], "rcssize").str
824 print StatList([x.totrev for x in rcsf.rcsfiles], "totrev").str 841 print StatList([x.totrev for x in rcsf.rcsfiles], "totrev").str
825 return rcsf 842 return rcsf
@@ -827,6 +844,7 @@ def GetTestRcsFiles():
827 844
828class SampleDataTest: 845class SampleDataTest:
829 def __init__(self, dirs): 846 def __init__(self, dirs):
847 dirs_in = dirs
830 self.pairs = [] 848 self.pairs = []
831 while dirs: 849 while dirs:
832 d = dirs[0] 850 d = dirs[0]
@@ -843,12 +861,16 @@ class SampleDataTest:
843 #end 861 #end
844 if len(files) > 1: 862 if len(files) > 1:
845 files.sort() 863 files.sort()
846 for x in xrange(len(files) - 1): 864 for x in xrange(len(files)):
847 self.pairs.append((files[x], files[x+1], 865 for y in xrange(len(files)):
848 '%s-%s' % (files[x], files[x+1]))) 866 self.pairs.append((files[x], files[y],
867 '%s-%s' % (files[x], files[y])))
868 #end
849 #end 869 #end
850 #end 870 #end
851 #end 871 #end
872 print "Sample data test using %d file pairs in %s" % (
873 len(self.pairs), dirs_in)
852 #end 874 #end
853 875
854 def Generator(self): 876 def Generator(self):
@@ -978,7 +1000,7 @@ def RandomTestConfigs(rand, input_configs, num_results):
978 return outputs 1000 return outputs
979#end 1001#end
980 1002
981def RunTestLoop(rand, generator, rounds): 1003def RunOptimizationLoop(rand, generator, rounds):
982 configs = [] 1004 configs = []
983 for rnum in xrange(rounds): 1005 for rnum in xrange(rounds):
984 configs = RandomTestConfigs(rand, configs, MAX_RESULTS) 1006 configs = RandomTestConfigs(rand, configs, MAX_RESULTS)
@@ -1192,28 +1214,50 @@ def GraphSummary(desc, results_ignore):
1192 #end 1214 #end
1193#end 1215#end
1194 1216
1217def RunRegressionTest(pairs, rounds):
1218 for args in [
1219 [],
1220 ['-S djw'],
1221
1222 ]:
1223 print "Args %s" % (args)
1224 for (file1, file2, testkey) in pairs:
1225 ttest = TimedTest(file1, file2, Xdelta3Runner(args, forkexec=True),
1226 skip_trials = 0,
1227 min_trials = 1,
1228 max_trials = 1)
1229 print "Source %s\nTarget %s\nEncode %s\nDecode %s\nSize %s\n\n" % (
1230 file1, file2,
1231 ttest.encode_time.str,
1232 ttest.decode_time.str,
1233 ttest.encode_size)
1234 #end
1235#end
1236
1195if __name__ == "__main__": 1237if __name__ == "__main__":
1196 try: 1238 try:
1197 RunCommand(['rm', '-rf', TMPDIR]) 1239 RunCommand(['rm', '-rf', TMPDIR])
1198 os.mkdir(TMPDIR) 1240 os.mkdir(TMPDIR)
1199 1241
1200 rcsf = GetTestRcsFiles() 1242 #rcsf = GetTestRcsFiles()
1201 generator = rcsf.Generator() 1243 #generator = rcsf.Generator()
1202 1244
1203 #sample = SampleDataTest([SAMPLEDIR]) 1245 sample = SampleDataTest([SAMPLEDIR])
1204 #generator = sample.Generator() 1246 generator = sample.Generator()
1205 1247
1206 rand = random.Random(135135135135135) 1248 rand = random.Random(135135135135135)
1207 RunTestLoop(rand, generator, TEST_ROUNDS) 1249
1250 RunRegressionTest(sample.pairs, TEST_ROUNDS)
1208 1251
1209 #RunSpeedTest() 1252 #RunSpeedTest()
1210 1253
1254 # the idea below is to add the default configurations and
1255 # xdelta1 to the optimization loop:
1211 #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-1', '-3', '-6'])) 1256 #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-1', '-3', '-6']))
1212 #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9'])) 1257 #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9']))
1213 #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-S', 'djw'])) 1258 #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-S', 'djw']))
1214 #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-1', '-S', 'djw'])) 1259 #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-1', '-S', 'djw']))
1215 #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-T'])) 1260 #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-T']))
1216
1217 #x1r = rcsf.AllPairsByDate(Xdelta1RunClass()) 1261 #x1r = rcsf.AllPairsByDate(Xdelta1RunClass())
1218 1262
1219 except CommandError: 1263 except CommandError: