summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorjosh.macdonald <jmacd@users.noreply.github.com>2007-02-05 10:30:28 +0000
committerjosh.macdonald <jmacd@users.noreply.github.com>2007-02-05 10:30:28 +0000
commitd3ab114a4f748456e63fbd0f19c89d52c8d740ae (patch)
tree0e52bceaff48d99f8096f6bd84bc05c4fe1bf276
parent29ec813ed97ebd4a02d594bb2a07f570ee1f604b (diff)
Add a .swig for python, working on xd3_encode_mememory() and
xd3_decode_memory(). Not working. Progress on scoring in xdelta3-regtest.py. Update FAST encoder w/ results of 9,11,13,16-llook experiments.
-rwxr-xr-xxdelta3/Makefile30
-rwxr-xr-xxdelta3/setup.py4
-rwxr-xr-xxdelta3/xdelta3-cfgs.h6
-rwxr-xr-xxdelta3/xdelta3-python.h7
-rwxr-xr-xxdelta3/xdelta3-regtest.py316
-rwxr-xr-xxdelta3/xdelta3-test.h10
-rwxr-xr-xxdelta3/xdelta3.prj12
-rwxr-xr-xxdelta3/xdelta3.swig29
8 files changed, 291 insertions, 123 deletions
diff --git a/xdelta3/Makefile b/xdelta3/Makefile
index 16df004..e5e5eeb 100755
--- a/xdelta3/Makefile
+++ b/xdelta3/Makefile
@@ -1,6 +1,8 @@
1# xdelta 3 - delta compression tools and library 1# xdelta 3 - delta compression tools and library
2# Copyright (C) 2001, 2003, 2004, 2005, 2006. Joshua P. MacDonald 2# Copyright (C) 2001, 2003, 2004, 2005, 2006. Joshua P. MacDonald
3 3
4CC=gcc-4.1.1
5
4SOURCES = xdelta3-cfgs.h \ 6SOURCES = xdelta3-cfgs.h \
5 xdelta3-decode.h \ 7 xdelta3-decode.h \
6 xdelta3-djw.h \ 8 xdelta3-djw.h \
@@ -17,6 +19,8 @@ TARGETS = xdelta3-debug \
17 xdelta3 \ 19 xdelta3 \
18 xdelta3-debug2 \ 20 xdelta3-debug2 \
19 xdelta3-debug3 \ 21 xdelta3-debug3 \
22 xdelta3.o \
23 xdelta3module.so \
20 xdelta3-32 \ 24 xdelta3-32 \
21 xdelta3-64 \ 25 xdelta3-64 \
22 xdelta3-everything \ 26 xdelta3-everything \
@@ -24,7 +28,7 @@ TARGETS = xdelta3-debug \
24 xdelta3-64-O \ 28 xdelta3-64-O \
25 xdelta3-Op \ 29 xdelta3-Op \
26 xdelta3-decoder xdelta3-decoder-nomain.o \ 30 xdelta3-decoder xdelta3-decoder-nomain.o \
27 xdelta3-nosec.o xdelta3-all.o xdelta3-fgk.o xdelta3-djw.o \ 31 xdelta3-nosec.o xdelta3-all.o xdelta3-fgk.o \
28 xdelta3-noext xdelta3-tools xdelta3-tune \ 32 xdelta3-noext xdelta3-tools xdelta3-tune \
29 xdelta3-notools \ 33 xdelta3-notools \
30 $(PYTGT) \ 34 $(PYTGT) \
@@ -90,6 +94,27 @@ xdelta3-debug3: $(SOURCES)
90$(PYTGT): $(SOURCES) 94$(PYTGT): $(SOURCES)
91 $(PYTHON) setup.py install --verbose --compile --force 95 $(PYTHON) setup.py install --verbose --compile --force
92 96
97xdelta3_wrap.c xdelta3.py: xdelta3.swig
98 swig -python xdelta3.swig
99
100xdelta3.o: $(SOURCES)
101 $(CC) -O3 -Wall -Wshadow -c xdelta3.c -DSECONDARY_DJW=1 -o xdelta3.o
102
103xdelta3_wrap.o: xdelta3_wrap.c
104 $(CC) -DXD3_DEBUG=0 \
105 -DXD3_USE_LARGEFILE64=1 \
106 -DSECONDARY_DJW=1 \
107 -DXD3_MAIN=0 \
108 -DHAVE_CONFIG_H \
109 -I/usr/include/python2.4 \
110 -I/usr/lib/python2.4/config \
111 -fpic \
112 -c xdelta3_wrap.c
113
114xdelta3module.so: xdelta3_wrap.o xdelta3.o
115 ld -shared xdelta3.o xdelta3_wrap.o -o xdelta3module.so /usr/lib/libpython2.4.so
116 cp -f xdelta3module.so /usr/lib/python2.4/site-packages/xdelta3module.so
117
93xdelta3-decoder: $(SOURCES) 118xdelta3-decoder: $(SOURCES)
94 $(CC) -O2 -Wall -Wshadow xdelta3.c \ 119 $(CC) -O2 -Wall -Wshadow xdelta3.c \
95 -DXD3_ENCODER=0 -DXD3_MAIN=1 -DSECONDARY_FGK=0 -DSECONDARY_DJW=0 \ 120 -DXD3_ENCODER=0 -DXD3_MAIN=1 -DSECONDARY_FGK=0 -DSECONDARY_DJW=0 \
@@ -146,9 +171,6 @@ xdelta3-all.o: $(SOURCES)
146xdelta3-fgk.o: $(SOURCES) 171xdelta3-fgk.o: $(SOURCES)
147 $(CC) -O2 -Wall -Wshadow -c xdelta3.c -DSECONDARY_FGK=1 -DSECONDARY_DJW=0 -o xdelta3-fgk.o 172 $(CC) -O2 -Wall -Wshadow -c xdelta3.c -DSECONDARY_FGK=1 -DSECONDARY_DJW=0 -o xdelta3-fgk.o
148 173
149xdelta3-djw.o: $(SOURCES)
150 $(CC) -O2 -Wall -Wshadow -c xdelta3.c -DSECONDARY_FGK=0 -DSECONDARY_DJW=1 -o xdelta3-djw.o
151
152xdelta3-noext: $(SOURCES) 174xdelta3-noext: $(SOURCES)
153 $(CC) -O2 -Wall -Wshadow xdelta3.c -DXD3_MAIN=1 -DEXTERNAL_COMPRESSION=0 -o xdelta3-noext 175 $(CC) -O2 -Wall -Wshadow xdelta3.c -DXD3_MAIN=1 -DEXTERNAL_COMPRESSION=0 -o xdelta3-noext
154 176
diff --git a/xdelta3/setup.py b/xdelta3/setup.py
index faee42b..24b80de 100755
--- a/xdelta3/setup.py
+++ b/xdelta3/setup.py
@@ -31,7 +31,7 @@ if is_cygwin:
31 extcomp = 0 31 extcomp = 0
32#end 32#end
33 33
34xdelta3_ext = Extension('xdelta3', 34xdelta3_ext = Extension('xdelta3main',
35 ['xdelta3.c'], 35 ['xdelta3.c'],
36 define_macros = [ 36 define_macros = [
37 ('PYTHON_MODULE',1), 37 ('PYTHON_MODULE',1),
@@ -56,6 +56,6 @@ REL='0n'
56 56
57# This provides xdelta3.main(), which calls the xdelta3 command-line main() 57# This provides xdelta3.main(), which calls the xdelta3 command-line main()
58# from python. 58# from python.
59setup(name='xdelta3', 59setup(name='xdelta3main',
60 version=REL, 60 version=REL,
61 ext_modules=[xdelta3_ext]) 61 ext_modules=[xdelta3_ext])
diff --git a/xdelta3/xdelta3-cfgs.h b/xdelta3/xdelta3-cfgs.h
index 955ff51..c2f4756 100755
--- a/xdelta3/xdelta3-cfgs.h
+++ b/xdelta3/xdelta3-cfgs.h
@@ -61,12 +61,12 @@
61#define LLOOK 9 61#define LLOOK 9
62#define LSTEP 8 62#define LSTEP 8
63#define SLOOK 4 63#define SLOOK 4
64#define SCHAIN 5 64#define SCHAIN 4
65#define SLCHAIN 1 65#define SLCHAIN 1
66#define SSMATCH 0 66#define SSMATCH 0
67#define TRYLAZY 1 67#define TRYLAZY 1
68#define MAXLAZY 5 68#define MAXLAZY 18
69#define LONGENOUGH 4 69#define LONGENOUGH 35
70#define PROMOTE 0 70#define PROMOTE 0
71 71
72#include "xdelta3.c" 72#include "xdelta3.c"
diff --git a/xdelta3/xdelta3-python.h b/xdelta3/xdelta3-python.h
index 845da48..a91b7f2 100755
--- a/xdelta3/xdelta3-python.h
+++ b/xdelta3/xdelta3-python.h
@@ -71,16 +71,17 @@ PyObject *xdelta3_main_cmdline (PyObject *self, PyObject *args)
71 } 71 }
72 return result; 72 return result;
73} 73}
74
74static PyMethodDef xdelta3_methods[] = { 75static PyMethodDef xdelta3_methods[] = {
75 { "main", xdelta3_main_cmdline, METH_VARARGS, "xdelta3 main()" }, 76 { "main", xdelta3_main_cmdline, METH_VARARGS, "xdelta3 main()" },
76 { NULL, NULL } 77 { NULL, NULL }
77}; 78};
78 79
79DL_EXPORT(void) initxdelta3 (void) 80DL_EXPORT(void) initxdelta3main (void)
80{ 81{
81 PyObject *m, *d; 82 PyObject *m, *d;
82 m = Py_InitModule ("xdelta3", xdelta3_methods); 83 m = Py_InitModule ("xdelta3main", xdelta3_methods);
83 d = PyModule_GetDict (m); 84 d = PyModule_GetDict (m);
84 pyxd3_error = PyErr_NewException ("xdelta3.error", NULL, NULL); 85 pyxd3_error = PyErr_NewException ("xdelta3main.error", NULL, NULL);
85 PyDict_SetItemString (d, "error", pyxd3_error); 86 PyDict_SetItemString (d, "error", pyxd3_error);
86} 87}
diff --git a/xdelta3/xdelta3-regtest.py b/xdelta3/xdelta3-regtest.py
index 9e90f8b..33f0715 100755
--- a/xdelta3/xdelta3-regtest.py
+++ b/xdelta3/xdelta3-regtest.py
@@ -24,7 +24,7 @@
24# TODO: Test IOPT (1.5 vs. greedy) 24# TODO: Test IOPT (1.5 vs. greedy)
25 25
26import os, sys, math, re, time, types, array, random 26import os, sys, math, re, time, types, array, random
27import xdelta3 27import xdelta3main
28 28
29HIST_SIZE = 10 # the number of buckets 29HIST_SIZE = 10 # the number of buckets
30MIN_SIZE = 0 30MIN_SIZE = 0
@@ -42,9 +42,45 @@ MAX_TRIALS = 15
42MIN_STDDEV_PCT = 1.5 # stop 42MIN_STDDEV_PCT = 1.5 # stop
43MAX_RUN = 1000 * 1000 * 10 43MAX_RUN = 1000 * 1000 * 10
44 44
45# How many results per round
46MAX_RESULTS = 100
47KEEP_P = (0.3)
48FAST_P = (0.0)
49SLOW_P = (0.0)
50FILE_P = (0.50)
51
52CONFIG_ORDER = [ 'large_look',
53 'large_step',
54 'small_look',
55 'small_chain',
56 'small_lchain',
57 'ssmatch',
58 'trylazy',
59 'max_lazy',
60 'long_enough',
61 'promote' ]
62
63def INPUT_SPEC(rand):
64 return {
65 'large_look' : lambda d: rand.choice([9]),
66 'large_step' : lambda d: rand.choice([4, 5]),
67
68 'small_chain' : lambda d: rand.randrange(4, 65, 4),
69 'small_lchain' : lambda d: rand.randrange(1, 17, 1),
70
71 'max_lazy' : lambda d: rand.choice([18]),
72 'long_enough' : lambda d: rand.choice([35]),
73
74 'small_look' : lambda d: 4,
75 'promote' : lambda d: 0,
76 'trylazy' : lambda d: 1,
77 'ssmatch' : lambda d: 0,
78 }
79
80
45# 81#
46# 82#
47RCSDIR = '/mnt/polaroid/Polaroid/orbit_linux/home/jmacd/PRCS' 83#RCSDIR = '/mnt/polaroid/Polaroid/orbit_linux/home/jmacd/PRCS'
48RCSDIR = '/tmp/PRCS_read_copy' 84RCSDIR = '/tmp/PRCS_read_copy'
49#RCSDIR = 'G:/jmacd/PRCS' 85#RCSDIR = 'G:/jmacd/PRCS'
50 86
@@ -60,6 +96,9 @@ BAR_STATE = 1
60REV_STATE = 2 96REV_STATE = 2
61DATE_STATE = 3 97DATE_STATE = 3
62 98
99#
100IGNORE_FILENAME = re.compile('.*\\.(gif|jpg).*')
101
63# rcs output 102# rcs output
64RE_TOTREV = re.compile('total revisions: (\\d+)') 103RE_TOTREV = re.compile('total revisions: (\\d+)')
65RE_BAR = re.compile('----------------------------') 104RE_BAR = re.compile('----------------------------')
@@ -69,6 +108,22 @@ RE_DATE = re.compile('date: ([^;]+);.*')
69RE_HDRSZ = re.compile('VCDIFF header size: +(\\d+)') 108RE_HDRSZ = re.compile('VCDIFF header size: +(\\d+)')
70RE_EXTCOMP = re.compile('XDELTA ext comp.*') 109RE_EXTCOMP = re.compile('XDELTA ext comp.*')
71 110
111# Testwide defaults
112testwide_encode_args = [
113
114 # secondary compression on or off
115 #'-S', 'djw',
116
117 # do not measure instruction buffer effects
118 '-I', '0',
119
120 # do not attempt external decompression
121 '-D'
122 ]
123
124def c2s(c):
125 return ' '.join(['%02d' % x for x in c])
126
72# 127#
73# exceptions 128# exceptions
74class SkipRcsException: 129class SkipRcsException:
@@ -92,8 +147,8 @@ class RcsVersion:
92 self.vstr = vstr 147 self.vstr = vstr
93 def __cmp__(self,other): 148 def __cmp__(self,other):
94 return cmp(self.date, other.date) 149 return cmp(self.date, other.date)
95 def Print(self): 150 def __str__(self):
96 print '%s %s' % (self.vstr, self.date) 151 return str(self.vstr)
97# 152#
98# one rcsfile 153# one rcsfile
99class RcsFile: 154class RcsFile:
@@ -201,7 +256,7 @@ class RcsFile:
201 if os.stat(self.Verf(v)).st_size < MIN_SIZE or \ 256 if os.stat(self.Verf(v)).st_size < MIN_SIZE or \
202 os.stat(self.Verf(v+1)).st_size < MIN_SIZE: 257 os.stat(self.Verf(v+1)).st_size < MIN_SIZE:
203 continue 258 continue
204 259
205 result = TimeRun(runnable.Runner(self.Verf(v), 260 result = TimeRun(runnable.Runner(self.Verf(v),
206 self.Vstr(v), 261 self.Vstr(v),
207 self.Verf(v+1), 262 self.Verf(v+1),
@@ -213,7 +268,7 @@ class RcsFile:
213 result.time.mean, 268 result.time.mean,
214 result.trials) 269 result.trials)
215 ntrials.append(result) 270 ntrials.append(result)
216 271
217 os.remove(self.Verf(self.totrev-1)) 272 os.remove(self.Verf(self.totrev-1))
218 os.remove(self.Verf(self.totrev-2)) 273 os.remove(self.Verf(self.totrev-2))
219 return ntrials 274 return ntrials
@@ -412,7 +467,7 @@ class StatList:
412def RunCommand(args): 467def RunCommand(args):
413 #print "run command", args 468 #print "run command", args
414 p = os.spawnvp(os.P_WAIT, args[0], args) 469 p = os.spawnvp(os.P_WAIT, args[0], args)
415 if p != 0: 470 if p != 0:
416 raise CommandError(args, 'exited %d' % p) 471 raise CommandError(args, 'exited %d' % p)
417 472
418def RunCommandIO(args,infn,outfn): 473def RunCommandIO(args,infn,outfn):
@@ -431,7 +486,7 @@ def RunCommandIO(args,infn,outfn):
431def RunXdelta3(args): 486def RunXdelta3(args):
432 try: 487 try:
433 #print 'RUN', args 488 #print 'RUN', args
434 xdelta3.main(args) 489 xdelta3main.main(args)
435 except Exception, e: 490 except Exception, e:
436 raise CommandError(args, "xdelta3.main exception") 491 raise CommandError(args, "xdelta3.main exception")
437 492
@@ -439,7 +494,7 @@ class GzipInfo:
439 def __init__(self,target,delta): 494 def __init__(self,target,delta):
440 self.tgtsize = os.stat(target).st_size 495 self.tgtsize = os.stat(target).st_size
441 self.dsize = os.stat(delta).st_size 496 self.dsize = os.stat(delta).st_size
442 497
443class Xdelta3Info: 498class Xdelta3Info:
444 def __init__(self,target,delta): 499 def __init__(self,target,delta):
445 self.extcomp = 0 # TODO: I removed some code that called printhdr 500 self.extcomp = 0 # TODO: I removed some code that called printhdr
@@ -464,14 +519,13 @@ class Xdelta3Pair:
464 self.old = old 519 self.old = old
465 self.oldv = oldv 520 self.oldv = oldv
466 self.new = new 521 self.new = new
467 self.newv = newv 522 self.newv = newv
468 return self 523 return self
469 524
470 def Run(self,trial): 525 def Run(self,trial):
471 526
472 # TODO: move '-S djw' somewhere else
473 encode_args = self.extra + \ 527 encode_args = self.extra + \
474 [ '-S', 'djw' ] + \ 528 testwide_encode_args + \
475 [self.encode_args, 529 [self.encode_args,
476 self.presrc, 530 self.presrc,
477 self.old, 531 self.old,
@@ -498,6 +552,7 @@ class Xdelta3Pair:
498 except CommandError: 552 except CommandError:
499 print 'encode args: %s' % ' '.join(encode_args) 553 print 'encode args: %s' % ' '.join(encode_args)
500 print 'decode args: %s' % ' '.join(decode_args) 554 print 'decode args: %s' % ' '.join(decode_args)
555 raise CommandError("Run failed")
501 556
502def Test(): 557def Test():
503 rcsf = RcsFinder() 558 rcsf = RcsFinder()
@@ -529,7 +584,8 @@ class Xdelta3Run1:
529 def __init__(self,file): 584 def __init__(self,file):
530 self.file = file 585 self.file = file
531 def Run(self,trial): 586 def Run(self,trial):
532 RunXdelta3(['-efq', self.file, DFILE]) 587 RunXdelta3(testwide_encode_args +
588 ['-efq', self.file, DFILE])
533 if trial > 0: 589 if trial > 0:
534 return None 590 return None
535 return Xdelta3Info(self.file,DFILE) 591 return Xdelta3Info(self.file,DFILE)
@@ -554,39 +610,23 @@ def ReportSpeed(L,tr,desc):
554 print '%s 0-run length %u: dsize %u: time %.3f ms: encode %.0f B/sec: in %u trials' % \ 610 print '%s 0-run length %u: dsize %u: time %.3f ms: encode %.0f B/sec: in %u trials' % \
555 (desc, L, tr.r1.dsize, tr.time.mean * 1000.0, ((L+tr.r1.dsize) / tr.time.mean), tr.trials) 611 (desc, L, tr.r1.dsize, tr.time.mean * 1000.0, ((L+tr.r1.dsize) / tr.time.mean), tr.trials)
556 612
557def MakeBigFiles(rcsf):
558 rand = random.Random()
559 f1 = open(TMPDIR + "/big.1", "w")
560 f2 = open(TMPDIR + "/big.2", "w")
561 f1sz = 0
562 f2sz = 0
563 for file in rcsf.rcsfiles:
564 if file.versions < 2:
565 continue
566 r1 = 0
567 r2 = 0
568 while r1 == r2:
569 r1 = rand.randint(0, len(file.versions) - 1)
570 r2 = rand.randint(0, len(file.versions) - 1)
571 f1sz += file.AppendVersion(f1, r1)
572 f2sz += file.AppendVersion(f2, r2)
573
574 print 'from %u; to %u' % (f1sz, f2sz)
575 f1.close()
576 f2.close()
577 return (TMPDIR + "/big.1",
578 TMPDIR + "/big.2")
579
580class RandomTestResult: 613class RandomTestResult:
581 def __init__(self, round, config, runtime, compsize): 614 def __init__(self, round, config, runtime, compsize):
582 self.round = round 615 self.round = round
583 self.myconfig = config 616 self.myconfig = config
584 self.runtime = runtime 617 self.runtime = runtime
585 self.compsize = compsize 618 self.compsize = compsize
619 self.score = None
620 self.time_pos = None
621 self.size_pos = None
622 self.score_pos = None
586 #end 623 #end
587 624
588 def __str__(self): 625 def __str__(self):
589 return '%.4f %d [%s]' % (self.time(), self.size(), ' '.join([str(x) for x in self.config()])) 626 return 'time %.6f%s size %d%s << %s >>' % (
627 self.time(), ((self.time_pos != None) and (" (%s)" % self.time_pos) or ""),
628 self.size(), ((self.size_pos != None) and (" (%s)" % self.size_pos) or ""),
629 c2s(self.config()))
590 #end 630 #end
591 631
592 def time(self): 632 def time(self):
@@ -600,10 +640,14 @@ class RandomTestResult:
600 def config(self): 640 def config(self):
601 return self.myconfig 641 return self.myconfig
602 #end 642 #end
643
644 def score(self):
645 return self.score
646 #end
603#end 647#end
604 648
605def PosInAlist(l, e): 649def PosInAlist(l, e):
606 for i in range(len(l)): 650 for i in range(0, len(l)):
607 if l[i][1] == e: 651 if l[i][1] == e:
608 return i; 652 return i;
609 #end 653 #end
@@ -611,56 +655,38 @@ def PosInAlist(l, e):
611 return -1 655 return -1
612#end 656#end
613 657
614# How many results per round 658test_totals = {}
615MAX_RESULTS = 500
616 659
617class RandomTester: 660class RandomTester:
618 def __init__(self, old_results): 661 def __init__(self, old_results):
619 self.old_configs = old_results 662 self.old_configs = old_results
663
664 # these get reset each round so we don't test the same config twice
620 self.results = [] 665 self.results = []
666
621 self.trial_num = 0 667 self.trial_num = 0
622 self.round_num = 0 668 self.round_num = 0
623 self.random = random.Random() 669 self.random = random.Random()
624 #end 670 #end
625 671
672 def Reset(self):
673 self.results = []
674 #end
675
626 def HasEnoughResults(self): 676 def HasEnoughResults(self):
627 return len(self.results) >= MAX_RESULTS 677 return len(self.results) >= MAX_RESULTS
628 #end 678 #end
629 679
630 def RandomConfig(self): 680 def RandomConfig(self):
631 681
632 input_ranges = [
633 (9, 9, 9, 'large_look'),
634 (1, 4.5, 8, 'large_step'),
635 (4, 4, 4, 'small_look'), # Note: disabled
636 (1, 10, 30, 'small_chain'),
637 (1, 3.5, 6, 'small_lchain'),
638 (0, 0, 0, 'ssmatch'), # Note: disabled
639 (1, 1, 1, 'trylazy'), # Note: enabled
640 (1, 128, 256, 'max_lazy'),
641 (1, 256, 512, 'long_enough'),
642 (0, 0, 0, 'promote'), # Note: disabled
643 ]
644
645 config = [] 682 config = []
646 map = {} 683 map = {}
647 684
648 for input in input_ranges: 685 for key in CONFIG_ORDER:
649 minv = input[0] 686
650 mean = input[1] 687 val = map[key] = (INPUT_SPEC(self.random)[key])(map)
651 maxv = input[2]
652 name = input[3]
653 if minv == maxv:
654 val = minv
655 else:
656 val = -1
657 while val < minv or val > maxv:
658 val = int(self.random.expovariate(1.0 / mean) + 0.5)
659 #end
660 #end
661 688
662 config.append(val) 689 config.append(val)
663 map[name] = val
664 #end 690 #end
665 691
666 if map['small_chain'] < map['small_lchain']: 692 if map['small_chain'] < map['small_lchain']:
@@ -669,8 +695,43 @@ class RandomTester:
669 if map['large_look'] < map['small_look']: 695 if map['large_look'] < map['small_look']:
670 return None 696 return None
671 697
698 for r in self.results:
699 if c2s(r.config()) == c2s(config):
700 return None
701 #end
702 #end
703
672 return config 704 return config
673 705
706 def MakeBigFiles(self, rcsf):
707 f1 = open(TMPDIR + "/big.1", "w")
708 f2 = open(TMPDIR + "/big.2", "w")
709 population = []
710 for file in rcsf.rcsfiles:
711 if len(file.versions) < 2:
712 continue
713 population.append(file)
714 #end
715 f1sz = 0
716 f2sz = 0
717 fcount = int(len(population) * FILE_P)
718 assert fcount > 0
719 for file in self.random.sample(population, fcount):
720 m = IGNORE_FILENAME.match(file.fname)
721 if m != None:
722 continue
723 #end
724 r1, r2 = self.random.sample(xrange(0, len(file.versions)), 2)
725 f1sz += file.AppendVersion(f1, r1)
726 f2sz += file.AppendVersion(f2, r2)
727 #end
728
729 print 'from %u; to %u' % (f1sz, f2sz)
730 f1.close()
731 f2.close()
732 return (TMPDIR + "/big.1",
733 TMPDIR + "/big.2")
734
674 def RandomBigRun(self, f1, f2): 735 def RandomBigRun(self, f1, f2):
675 config = None 736 config = None
676 if len(self.old_configs) > 0: 737 if len(self.old_configs) > 0:
@@ -683,10 +744,7 @@ class RandomTester:
683 #end 744 #end
684 745
685 runner = Xdelta3Pair() 746 runner = Xdelta3Pair()
686 runner.extra = ['-I', 747 runner.extra = [ '-C', ','.join([str(x) for x in config]) ]
687 '0',
688 '-D',
689 '-C', ','.join([str(x) for x in config])]
690 result = TimeRun(runner.Runner(f1, 1, f2, 2)) 748 result = TimeRun(runner.Runner(f1, 1, f2, 2))
691 749
692 tr = RandomTestResult(self.round_num, 750 tr = RandomTestResult(self.round_num,
@@ -696,7 +754,7 @@ class RandomTester:
696 754
697 self.results.append(tr) 755 self.results.append(tr)
698 756
699 print 'Trial %d: %s in %u trials' % \ 757 print 'Test %d: %s in %u trials' % \
700 (self.trial_num, 758 (self.trial_num,
701 tr, 759 tr,
702 result.trials) 760 result.trials)
@@ -706,54 +764,111 @@ class RandomTester:
706 #end 764 #end
707 765
708 def ScoreTests(self): 766 def ScoreTests(self):
709 mint = float(min([test.time() for test in self.results]))
710 maxt = float(max([test.time() for test in self.results]))
711 mins = float(min([test.size() for test in self.results]))
712 maxs = float(max([test.size() for test in self.results]))
713
714 scored = [] 767 scored = []
715 timed = [] 768 timed = []
716 sized = [] 769 sized = []
717 770
771 t_min = float(min([test.time() for test in self.results]))
772 t_max = float(max([test.time() for test in self.results]))
773 s_min = float(min([test.size() for test in self.results]))
774 s_max = float(max([test.size() for test in self.results]))
775
776 # These are the major axes of an ellipse, after normalizing for the
777 # mininum values. Time should be major, size should be minor.
778 time_major = (t_max / t_min)
779 size_minor = (s_max / s_min)
780
781 # Dimensions of the rectangular region bounding the results.
782 t_rect = time_major - 1.0
783 s_rect = size_minor - 1.0
784
785 assert(time_major >= size_minor)
786
787 rect_ratio = s_rect / t_rect
788
789 assert(rect_ratio <= 1.000001)
790
718 for test in self.results: 791 for test in self.results:
719 792
720 # This scores ellipse has x-major (time) and y-minor (size) 793 # Transform the major min/max region linearly to normalize the
721 ntime = (test.time()) / float(maxt) 794 # min-max variation in time (major) and size (minor).
722 nsize = (test.size()) / float(maxs) 795
796 s_norm = test.size() / s_min
797 t_norm = 1.0 + rect_ratio * (test.time() / t_min - 1.0)
723 798
724 wntime = ntime * (maxs / mins) 799 assert t_norm >= 1.0
725 wnsize = nsize * (maxt / mint) 800 assert t_norm <= size_minor + 0.000001
726 score = math.sqrt(wntime * wntime + wnsize * wnsize) 801
727 scored.append((score, test)) 802 # Projects the coords onto a min-unit circle. Use the
728 timed.append((test.time(), test, score)) 803 # root-mean-square. Smaller scores are better, 1.0 is the minimum.
729 sized.append((test.size(), test, score)) 804 test.score = math.sqrt(t_norm * t_norm / 2.0 + s_norm * s_norm / 2.0)
805
806 scored.append((test.score, test))
807 timed.append((test.time(), test))
808 sized.append((test.size(), test))
730 #end 809 #end
810
731 scored.sort() 811 scored.sort()
732 timed.sort() 812 timed.sort()
733 sized.sort() 813 sized.sort()
734 for (score, test) in scored:
735 spos = PosInAlist(sized, test)
736 tpos = PosInAlist(timed, test)
737 print 'Score %f: %s (%d, %d)' % (score, test, spos, tpos)
738 #end
739 814
740 sized = sized[0:MAX_RESULTS/2] 815 best_by_size = []
741 timed = timed[0:MAX_RESULTS/2] 816 best_by_time = []
742 817
743 for (size, test, score) in sized: 818 print 'Worst: %s' % scored[len(scored)-1][1]
744 print 'Size: %s (%f)' % (test, score) 819 scored = [x[1] for x in scored[0:int(MAX_RESULTS * KEEP_P)]]
820
821 for fast in [x[1] for x in timed[0:int(MAX_RESULTS * FAST_P)]]:
822 if fast in scored:
823 continue
824 print 'Carry fast: %s' % (fast)
825 scored.append(fast)
745 #end 826 #end
746 827
747 for (time, test, score) in timed: 828 for slow in [x[1] for x in sized[0:int(MAX_RESULTS * SLOW_P)]]:
748 print 'Time: %s (%f)' % (test, score) 829 if slow in scored:
830 continue
831 print 'Carry slow: %s' % (slow)
832 scored.append(slow)
833 #end
834
835 # Do not carry slow. It causes bad compressors to perpetuate extra
836 # weight.
837 for test in scored:
838 test.size_pos = PosInAlist(sized, test)
839 test.time_pos = PosInAlist(timed, test)
749 #end 840 #end
750 841
751 self.results = []
752 r = [] 842 r = []
753 for (score, test) in scored[0:MAX_RESULTS/2]: 843 pos = 0
844 for test in scored:
845 pos += 1
846 test.score_pos = pos
847 c = c2s(test.config())
848 if not test_totals.has_key(c):
849 test_totals[c] = [test]
850 else:
851 test_totals[c].append(test)
852 #end
853 s = 0.0
754 r.append(test.config()) 854 r.append(test.config())
855 all_r = test_totals[c]
856 for t in all_r:
857 s += float(t.score_pos)
858 #end
859 if len(all_r) == 1:
860 stars = ''
861 elif len(all_r) >= 10:
862 stars = ' ***'
863 elif len(all_r) >= int(1/KEEP_P):
864 stars = ' **'
865 else:
866 stars = ' *'
867 print 'Score: %0.6f %s (%.1f%s%s)' % \
868 (test.score, test, s / len(all_r), stars, (len(all_r) > 2) and (' in %d' % len(all_r)) or "")
755 #end 869 #end
756 870
871
757 return r 872 return r
758 #end 873 #end
759#end 874#end
@@ -776,14 +891,15 @@ if __name__ == "__main__":
776 configs = [] 891 configs = []
777 892
778 while 1: 893 while 1:
779 f1, f2 = MakeBigFiles(rcsf)
780 #f1 = '/tmp/big.1' 894 #f1 = '/tmp/big.1'
781 #f2 = '/tmp/big.2' 895 #f2 = '/tmp/big.2'
782 test = RandomTester(configs) 896 test = RandomTester(configs)
897 f1, f2 = test.MakeBigFiles(rcsf)
783 while not test.HasEnoughResults(): 898 while not test.HasEnoughResults():
784 test.RandomBigRun(f1, f2) 899 test.RandomBigRun(f1, f2)
785 #end 900 #end
786 configs = test.ScoreTests() 901 configs = test.ScoreTests()
902 test.Reset()
787 #end 903 #end
788 #end 904 #end
789 905
diff --git a/xdelta3/xdelta3-test.h b/xdelta3/xdelta3-test.h
index b83e4c1..3658488 100755
--- a/xdelta3/xdelta3-test.h
+++ b/xdelta3/xdelta3-test.h
@@ -2217,11 +2217,11 @@ static int
2217test_in_memory (xd3_stream *stream, int ignore) 2217test_in_memory (xd3_stream *stream, int ignore)
2218{ 2218{
2219 // test_text is 256 bytes 2219 // test_text is 256 bytes
2220 char ibuf[sizeof(test_text)]; 2220 uint8_t ibuf[sizeof(test_text)];
2221 char dbuf[sizeof(test_text)]; 2221 uint8_t dbuf[sizeof(test_text)];
2222 char obuf[sizeof(test_text)]; 2222 uint8_t obuf[sizeof(test_text)];
2223 int size = sizeof(test_text); 2223 usize_t size = sizeof(test_text);
2224 int dsize, osize; 2224 usize_t dsize, osize;
2225 int r1, r2; 2225 int r1, r2;
2226 int eflags = SECONDARY_DJW ? XD3_SEC_DJW : 0; 2226 int eflags = SECONDARY_DJW ? XD3_SEC_DJW : 0;
2227 2227
diff --git a/xdelta3/xdelta3.prj b/xdelta3/xdelta3.prj
index 0740f58..67bb489 100755
--- a/xdelta3/xdelta3.prj
+++ b/xdelta3/xdelta3.prj
@@ -1,11 +1,11 @@
1;; -*- Prcs -*- 1;; -*- Prcs -*-
2(Created-By-Prcs-Version 1 3 4) 2(Created-By-Prcs-Version 1 3 4)
3(Project-Description "") 3(Project-Description "")
4(Project-Version xdelta3 0 12) 4(Project-Version xdelta3 0 13)
5(Parent-Version xdelta3 0 11) 5(Parent-Version xdelta3 0 12)
6(Version-Log "fixes buffer-allocation in xdelta3-decode related to secondary, adds new -6 default compression level") 6(Version-Log "fix /dont_run_xdelta3_test_as_root stupid")
7(New-Version-Log "") 7(New-Version-Log "")
8(Checkin-Time "Sun, 04 Feb 2007 01:49:07 -0800") 8(Checkin-Time "Sun, 04 Feb 2007 02:47:02 -0800")
9(Checkin-Login jmacd) 9(Checkin-Login jmacd)
10(Populate-Ignore ()) 10(Populate-Ignore ())
11(Project-Keywords 11(Project-Keywords
@@ -23,11 +23,11 @@
23 (xdelta3-python.h (xdelta3/4_xdelta3-py 1.3 644)) 23 (xdelta3-python.h (xdelta3/4_xdelta3-py 1.3 644))
24 (xdelta3-regtest.py (xdelta3/10_xdelta3-re 1.7 744)) 24 (xdelta3-regtest.py (xdelta3/10_xdelta3-re 1.7 744))
25 (xdelta3-second.h (xdelta3/3_xdelta3-se 1.4 744)) 25 (xdelta3-second.h (xdelta3/3_xdelta3-se 1.4 744))
26 (xdelta3-test.h (xdelta3/2_xdelta3-te 1.6 744)) 26 (xdelta3-test.h (xdelta3/2_xdelta3-te 1.7 744))
27 (xdelta3.c (xdelta3/16_xdelta3.c 1.7 744)) 27 (xdelta3.c (xdelta3/16_xdelta3.c 1.7 744))
28 (xdelta3.h (xdelta3/1_xdelta3.h 1.6 744)) 28 (xdelta3.h (xdelta3/1_xdelta3.h 1.6 744))
29 29
30 (Makefile (xdelta3/0_Makefile 1.5 744)) 30 (Makefile (xdelta3/0_Makefile 1.6 744))
31 (setup.py (xdelta3/11_setup.py 1.3 744)) 31 (setup.py (xdelta3/11_setup.py 1.3 744))
32 32
33 (draft-korn-vcdiff.txt (xdelta3/b/22_draft-korn 1.1 644)) 33 (draft-korn-vcdiff.txt (xdelta3/b/22_draft-korn 1.1 644))
diff --git a/xdelta3/xdelta3.swig b/xdelta3/xdelta3.swig
new file mode 100755
index 0000000..4397b9d
--- /dev/null
+++ b/xdelta3/xdelta3.swig
@@ -0,0 +1,29 @@
1%module xdelta3
2%{
3#include "xdelta3.h"
4
5DL_EXPORT(void) initxdelta3 (void)
6{
7 SWIG_init();
8}
9
10%}
11
12int xd3_encode_memory (const uint8_t *input,
13 usize_t input_size,
14 const uint8_t *source,
15 usize_t source_size,
16 uint8_t *output,
17 usize_t *output_size,
18 usize_t avail_output,
19 int flags);
20
21int xd3_decode_memory (const uint8_t *input,
22 usize_t input_size,
23 const uint8_t *source,
24 usize_t source_size,
25 uint8_t *output,
26 usize_t *output_size,
27 usize_t avail_output,
28 int flags);
29