diff options
Diffstat (limited to 'xdelta3/xdelta3-regtest.py')
-rwxr-xr-x | xdelta3/xdelta3-regtest.py | 121 |
1 files changed, 66 insertions, 55 deletions
diff --git a/xdelta3/xdelta3-regtest.py b/xdelta3/xdelta3-regtest.py index 1248584..5c7b6bf 100755 --- a/xdelta3/xdelta3-regtest.py +++ b/xdelta3/xdelta3-regtest.py | |||
@@ -16,13 +16,21 @@ | |||
16 | # along with this program; if not, write to the Free Software | 16 | # along with this program; if not, write to the Free Software |
17 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | 17 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
18 | 18 | ||
19 | # TODO: Test 1.5 vs. greedy | 19 | # TODO: test 1.5 vs. greedy |
20 | # TODO: Compare w/ bsdiff and generate more summary | 20 | # TODO: generate a graph |
21 | 21 | ||
22 | import os, sys, math, re, time, types, array, random | 22 | import os, sys, math, re, time, types, array, random |
23 | import xdelta3main | 23 | import xdelta3main |
24 | import xdelta3 | 24 | import xdelta3 |
25 | 25 | ||
26 | #RCSDIR = '/mnt/polaroid/Polaroid/orbit_linux/home/jmacd/PRCS' | ||
27 | RCSDIR = '/tmp/PRCS_read_copy/prcs' | ||
28 | SAMPLEDIR = "/tmp/WESNOTH_tmp/diff" | ||
29 | |||
30 | #RCSDIR = 'G:/jmacd/PRCS/prcs/b' | ||
31 | #SAMPLEDIR = "C:/sample_data/Wesnoth/tar" | ||
32 | |||
33 | # | ||
26 | MIN_SIZE = 0 | 34 | MIN_SIZE = 0 |
27 | 35 | ||
28 | TIME_TOO_SHORT = 0.050 | 36 | TIME_TOO_SHORT = 0.050 |
@@ -32,7 +40,7 @@ MIN_TRIALS = 3 | |||
32 | MAX_TRIALS = 15 | 40 | MAX_TRIALS = 15 |
33 | 41 | ||
34 | # 10 = fast 1.5 = slow | 42 | # 10 = fast 1.5 = slow |
35 | MIN_STDDEV_PCT = 10 | 43 | MIN_STDDEV_PCT = 1.5 |
36 | 44 | ||
37 | # How many results per round | 45 | # How many results per round |
38 | MAX_RESULTS = 4 | 46 | MAX_RESULTS = 4 |
@@ -48,73 +56,76 @@ MIN_RUN = 1000 * 1000 * 1 | |||
48 | MAX_RUN = 1000 * 1000 * 10 | 56 | MAX_RUN = 1000 * 1000 * 10 |
49 | 57 | ||
50 | # Testwide defaults | 58 | # Testwide defaults |
51 | ALL_ARGS = [ '-f' ] | 59 | ALL_ARGS = [ |
60 | # -v | ||
61 | ] | ||
52 | 62 | ||
53 | # The first 10 args go to -C | 63 | # The first 7 args go to -C |
54 | SOFT_CONFIG_CNT = 10 | 64 | SOFT_CONFIG_CNT = 7 |
55 | 65 | ||
56 | CONFIG_ORDER = [ 'large_look', | 66 | CONFIG_ORDER = [ 'large_look', |
57 | 'large_step', | 67 | 'large_step', |
58 | 'small_look', | 68 | 'small_look', |
59 | 'small_chain', | 69 | 'small_chain', |
60 | 'small_lchain', | 70 | 'small_lchain', |
61 | 'ssmatch', | ||
62 | 'trylazy', | ||
63 | 'max_lazy', | 71 | 'max_lazy', |
64 | 'long_enough', | 72 | 'long_enough', |
65 | 'promote', | 73 | |
74 | # > SOFT_CONFIG_CNT | ||
75 | 'nocompress', | ||
66 | 'winsize', | 76 | 'winsize', |
67 | 'srcwinsize', | 77 | 'srcwinsize', |
68 | 'sprevsz', | 78 | 'sprevsz', |
69 | 'iopt', | 79 | 'iopt', |
70 | 80 | 'djw', | |
71 | # TODO: nocompress, djw (boolean flags) | ||
72 | ] | 81 | ] |
73 | 82 | ||
74 | CONFIG_ARGMAP = { | 83 | CONFIG_ARGMAP = { |
75 | 'winsize' : '-W', | 84 | 'winsize' : '-W', |
76 | 'srcwinsize' : '-B', | 85 | 'srcwinsize' : '-B', |
77 | 'sprevsz' : '-P', | 86 | 'sprevsz' : '-P', |
78 | 'iopt' : '-I', | 87 | 'iopt' : '-I', |
88 | 'nocompress' : '-N', | ||
89 | 'djw' : '-Sdjw' | ||
79 | } | 90 | } |
80 | 91 | ||
81 | def INPUT_SPEC(rand): | 92 | def INPUT_SPEC(rand): |
82 | return { | 93 | return { |
83 | # computational costs | ||
84 | 'large_look' : lambda d: rand.choice([9]), | ||
85 | 'large_step' : lambda d: rand.choice([15]), | ||
86 | 94 | ||
87 | 'small_chain' : lambda d: rand.choice([1]), | 95 | # Time/space costs. |
88 | 'small_lchain' : lambda d: rand.choice([1]), | ||
89 | 96 | ||
97 | # -C 1,2,3,4,5,6,7 | ||
98 | 'large_look' : lambda d: rand.choice([9]), | ||
99 | 'large_step' : lambda d: rand.choice([3, 15]), | ||
100 | 'small_chain' : lambda d: rand.choice([1, 2]), | ||
101 | 'small_lchain' : lambda d: rand.choice([1]), | ||
90 | 'max_lazy' : lambda d: rand.choice([18]), | 102 | 'max_lazy' : lambda d: rand.choice([18]), |
91 | 'long_enough' : lambda d: rand.choice([18]), | 103 | 'long_enough' : lambda d: rand.choice([18]), |
92 | |||
93 | 'small_look' : lambda d: rand.choice([4]), | 104 | 'small_look' : lambda d: rand.choice([4]), |
94 | 'ssmatch' : lambda d: rand.choice([0]), | ||
95 | 105 | ||
96 | 'promote' : lambda d: rand.choice([0]), | 106 | # -N |
107 | 'nocompress' : lambda d: rand.choice(['false']), | ||
97 | 108 | ||
98 | 'trylazy' : lambda d: rand.choice([1]), | 109 | # -S djw |
110 | 'djw' : lambda d: rand.choice(['false']), | ||
99 | 111 | ||
100 | # memory costs | 112 | # Mmemory costs. |
101 | 'iopt' : lambda d: 0, # unlimited | ||
102 | 113 | ||
114 | # -W | ||
103 | 'winsize' : lambda d: 8 * (1<<20), | 115 | 'winsize' : lambda d: 8 * (1<<20), |
116 | |||
117 | # -B | ||
104 | 'srcwinsize' : lambda d: 64 * (1<<20), | 118 | 'srcwinsize' : lambda d: 64 * (1<<20), |
105 | 119 | ||
106 | 'sprevsz' : lambda d: 1 * (1<<18), # only powers of two | 120 | # -I 0 is unlimited |
107 | } | 121 | 'iopt' : lambda d: 0, |
108 | 122 | ||
109 | # | 123 | # -P only powers of two |
110 | # | 124 | 'sprevsz' : lambda d: rand.choice([x * (1<<16) for x in [4]]), |
111 | #RCSDIR = '/mnt/polaroid/Polaroid/orbit_linux/home/jmacd/PRCS' | 125 | } |
112 | RCSDIR = '/tmp/PRCS_read_copy/prcs' | 126 | #end |
113 | SAMPLEDIR = "/tmp/WESNOTH_tmp/tar" | ||
114 | |||
115 | #RCSDIR = 'G:/jmacd/PRCS/prcs/b' | ||
116 | #SAMPLEDIR = "C:/sample_data/Wesnoth/tar" | ||
117 | 127 | ||
128 | # | ||
118 | TMPDIR = '/tmp/xd3regtest.%d' % os.getpid() | 129 | TMPDIR = '/tmp/xd3regtest.%d' % os.getpid() |
119 | 130 | ||
120 | RUNFILE = os.path.join(TMPDIR, 'run') | 131 | RUNFILE = os.path.join(TMPDIR, 'run') |
@@ -139,7 +150,7 @@ RE_HDRSZ = re.compile('VCDIFF header size: +(\\d+)') | |||
139 | RE_EXTCOMP = re.compile('XDELTA ext comp.*') | 150 | RE_EXTCOMP = re.compile('XDELTA ext comp.*') |
140 | 151 | ||
141 | def c2s(c): | 152 | def c2s(c): |
142 | return ' '.join(['%02d' % x for x in c]) | 153 | return ' '.join(['%s' % x for x in c]) |
143 | #end | 154 | #end |
144 | 155 | ||
145 | def SumList(l): | 156 | def SumList(l): |
@@ -470,8 +481,6 @@ class Xdelta1Runner: | |||
470 | #end | 481 | #end |
471 | #end | 482 | #end |
472 | 483 | ||
473 | # TODO: cleanup below this line | ||
474 | |||
475 | # exceptions | 484 | # exceptions |
476 | class SkipRcsException: | 485 | class SkipRcsException: |
477 | def __init__(self,reason): | 486 | def __init__(self,reason): |
@@ -771,14 +780,13 @@ def GetTestRcsFiles(): | |||
771 | return rcsf | 780 | return rcsf |
772 | #end | 781 | #end |
773 | 782 | ||
774 | # TODO: cleanup below this line | ||
775 | |||
776 | # | 783 | # |
777 | class RandomTestResult: | 784 | class RandomTestResult: |
778 | def __init__(self, round, config, runtime, compsize): | 785 | def __init__(self, round, config, runtime, compsize, decodetime): |
779 | self.round = round | 786 | self.round = round |
780 | self.myconfig = config | 787 | self.myconfig = config |
781 | self.runtime = runtime | 788 | self.runtime = runtime |
789 | self.decodetime = decodetime | ||
782 | self.compsize = compsize | 790 | self.compsize = compsize |
783 | self.score = None | 791 | self.score = None |
784 | self.time_pos = None | 792 | self.time_pos = None |
@@ -787,10 +795,11 @@ class RandomTestResult: | |||
787 | #end | 795 | #end |
788 | 796 | ||
789 | def __str__(self): | 797 | def __str__(self): |
790 | return 'time %.6f%s size %d%s << %s >>' % ( | 798 | return 'time %.6f%s size %d%s << %s >> %.6f' % ( |
791 | self.time(), ((self.time_pos != None) and (" (%s)" % self.time_pos) or ""), | 799 | self.time(), ((self.time_pos != None) and (" (%s)" % self.time_pos) or ""), |
792 | self.size(), ((self.size_pos != None) and (" (%s)" % self.size_pos) or ""), | 800 | self.size(), ((self.size_pos != None) and (" (%s)" % self.size_pos) or ""), |
793 | c2s(self.config())) | 801 | c2s(self.config()), |
802 | self.decodetime) | ||
794 | #end | 803 | #end |
795 | 804 | ||
796 | def time(self): | 805 | def time(self): |
@@ -853,12 +862,6 @@ class RandomTester: | |||
853 | config.append(val) | 862 | config.append(val) |
854 | #end | 863 | #end |
855 | 864 | ||
856 | if map['small_chain'] < map['small_lchain']: | ||
857 | return None | ||
858 | |||
859 | if map['large_look'] < map['small_look']: | ||
860 | return None | ||
861 | |||
862 | for r in self.results: | 865 | for r in self.results: |
863 | if c2s(r.config()) == c2s(config): | 866 | if c2s(r.config()) == c2s(config): |
864 | return None | 867 | return None |
@@ -873,7 +876,13 @@ class RandomTester: | |||
873 | for i in range(SOFT_CONFIG_CNT, len(CONFIG_ORDER)): | 876 | for i in range(SOFT_CONFIG_CNT, len(CONFIG_ORDER)): |
874 | key = CONFIG_ARGMAP[CONFIG_ORDER[i]] | 877 | key = CONFIG_ARGMAP[CONFIG_ORDER[i]] |
875 | val = config[i] | 878 | val = config[i] |
876 | args.append('%s=%s' % (key, val)) | 879 | if val == 'true' or val == 'false': |
880 | if val == 'true': | ||
881 | args.append('%s' % key) | ||
882 | #end | ||
883 | else: | ||
884 | args.append('%s=%s' % (key, val)) | ||
885 | #end | ||
877 | #end | 886 | #end |
878 | return args | 887 | return args |
879 | #end | 888 | #end |
@@ -924,7 +933,8 @@ class RandomTester: | |||
924 | tr = RandomTestResult(self.round_num, | 933 | tr = RandomTestResult(self.round_num, |
925 | config, | 934 | config, |
926 | result.encode_time.mean, | 935 | result.encode_time.mean, |
927 | result.encode_size) | 936 | result.encode_size, |
937 | result.decode_time.mean) | ||
928 | 938 | ||
929 | self.results.append(tr) | 939 | self.results.append(tr) |
930 | 940 | ||
@@ -1060,7 +1070,6 @@ def RunRandomRcsTest(rcsf): | |||
1060 | #end | 1070 | #end |
1061 | 1071 | ||
1062 | def RunSampleTest(d, files): | 1072 | def RunSampleTest(d, files): |
1063 | # TODO: consolidate w/ the above | ||
1064 | print 'testing %s with %d files' % (d, len(files)) | 1073 | print 'testing %s with %d files' % (d, len(files)) |
1065 | configs = [] | 1074 | configs = [] |
1066 | while len(files) > 1: | 1075 | while len(files) > 1: |
@@ -1106,6 +1115,11 @@ if __name__ == "__main__": | |||
1106 | #RunSpeedTest() | 1115 | #RunSpeedTest() |
1107 | 1116 | ||
1108 | #rcsf = GetTestRcsFiles() | 1117 | #rcsf = GetTestRcsFiles() |
1118 | #RunRandomRcsTest(rcsf) | ||
1119 | |||
1120 | RunSampleDataTest() | ||
1121 | |||
1122 | # Other | ||
1109 | 1123 | ||
1110 | #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9'])) | 1124 | #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9'])) |
1111 | #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-S', 'djw'])) | 1125 | #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-S', 'djw'])) |
@@ -1117,9 +1131,6 @@ if __name__ == "__main__": | |||
1117 | #x1r = rcsf.AllPairsByDate(Xdelta1RunClass()) | 1131 | #x1r = rcsf.AllPairsByDate(Xdelta1RunClass()) |
1118 | #ReportPairs('xdelta1', x1r) | 1132 | #ReportPairs('xdelta1', x1r) |
1119 | 1133 | ||
1120 | #RunRandomRcsTest(rcsf) | ||
1121 | RunSampleDataTest() | ||
1122 | |||
1123 | except CommandError: | 1134 | except CommandError: |
1124 | pass | 1135 | pass |
1125 | else: | 1136 | else: |