Work on tuning tests.

3 files changed, 148 insertions, 50 deletions

diff --git a/xdelta3/setup.py b/xdelta3/setup.py
index ffde8ad..84b6cbf 100755
--- a/xdelta3/setup.py
+++ b/xdelta3/setup.py
@@ -21,16 +21,19 @@ from distutils.core import setup, Extension
 xdelta3_ext = Extension('xdelta3',
                         ['xdelta3.c'],
                         define_macros = [
-                                         ('EXTCOMP',1),
                                          ('REGRESSION_TEST',1),
                                          ('SECONDARY_DJW',1),
                                          ('SECONDARY_FGK',1),
                                          ('VCDIFF_TOOLS',1),
                                          ('XD3_DEBUG',1),
                                          ('XD3_POSIX',1),
+                                         ('EXTERNAL_COMPRESSION',1),
                                          ('XD3_USE_LARGEFILE64',1),
                                          ('PYTHON_MODULE',1),
-                                         ])
+                                         ],
+                        extra_compile_args = [ '-O3',
+                                               '-funroll-loops',
+                                               ])
 
 # $Format: "REL='$Xdelta3Version$'" $
 REL='0k'
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h
index e80b332..7d03eb7 100755
--- a/xdelta3/xdelta3-main.h
+++ b/xdelta3/xdelta3-main.h
@@ -242,7 +242,7 @@ static int         option_stdout             = 0;
 static int         option_force              = 0;
 static int         option_verbose            = 0;
 static int         option_quiet              = 0;
-static int         option_level              = 6;
+static int         option_level              = 5;
 static int         option_use_appheader      = 1;
 static uint8_t*    option_appheader          = NULL;
 static int         option_use_secondary      = /* until-standardized, leave this off */ 0;
@@ -352,6 +352,51 @@ main_config (void)
   return EXIT_SUCCESS;
 }
 
+static void
+reset_defaults()
+{
+  option_stdout = 0;
+  option_force = 0;
+  option_verbose = 0;
+  option_quiet = 0;
+  option_level = 5;
+  option_use_appheader = 1;
+  option_appheader = NULL;
+  option_use_secondary = 0;
+  option_secondary = NULL;
+  option_use_checksum = 1;
+  option_use_altcodetable = 0;
+  option_smatch_config = NULL;
+  option_no_compress = 0;
+  option_no_output = 0;
+  option_source_filename = NULL;
+  option_winsize = XD3_DEFAULT_WINSIZE;
+  option_srcwinsz = XD3_DEFAULT_SRCWINSZ;
+  option_srcwinsz_set = 0;
+  option_profile_cnt = 0;
+#if EXTERNAL_COMPRESSION
+  option_decompress_inputs  = 1;
+  option_recompress_outputs = 1;
+#endif
+#if VCDIFF_TOOLS
+  option_print_cpymode = 1;
+#endif
+  program_name = NULL;
+  appheader_used = NULL;
+  main_bdata = NULL;
+  lru_size = 0;
+  lru = NULL;
+  do_not_lru = 0;
+  lru_hits   = 0;
+  lru_misses = 0;
+  lru_filled = 0;
+  allow_fake_source = 0;
+  option_winsize = XD3_DEFAULT_WINSIZE;
+  option_srcwinsz = XD3_DEFAULT_SRCWINSZ;
+  option_srcwinsz_set = 0;
+  option_smatch_config = NULL;
+}
+
 static void*
 main_malloc1 (usize_t size)
 {
@@ -2295,8 +2340,8 @@ main_input (xd3_cmd     cmd,
           config.smatcher_soft.long_enough   = values[8];
           config.smatcher_soft.promote       = values[9];
         }
-      else if (option_level < 5) { config.smatch_cfg = XD3_SMATCH_FAST; }
-      else                       { config.smatch_cfg = XD3_SMATCH_SLOW; }
+      else if (option_level <= 5) { config.smatch_cfg = XD3_SMATCH_FAST; }
+      else                        { config.smatch_cfg = XD3_SMATCH_SLOW; }
       break;
 #endif
     case CMD_DECODE:
@@ -2353,6 +2398,11 @@ main_input (xd3_cmd     cmd,
       return EXIT_FAILURE;
     }
 
+  if (option_verbose > 1)
+    {
+      XPR(NT "scanner configuration: %s\n", stream.smatcher.name); 
+    }
+
   /* This times each window. */
   get_millisecs_since ();
 
@@ -2661,6 +2711,8 @@ main (int argc, char **argv)
   main_file_init (& ofile);
   main_file_init (& sfile);
 
+  reset_defaults();
+
  go:  /* Go. */
   cmd = CMD_NONE;
   sfilename = NULL;
diff --git a/xdelta3/xdelta3-regtest.py b/xdelta3/xdelta3-regtest.py
index 4cb3a35..bf1693d 100755
--- a/xdelta3/xdelta3-regtest.py
+++ b/xdelta3/xdelta3-regtest.py
@@ -560,50 +560,89 @@ def ReportSpeed(L,tr,desc):
           (desc, L, tr.r1.dsize, tr.time.mean * 1000.0, ((L+tr.r1.dsize) / tr.time.mean), tr.trials, tr.reps)
 
 def BigFileRuns(rcsf):
+    while 1:
+
+        rand = random.Random()
+        f1 = open(TMPDIR + "/big.1", "w")
+        f2 = open(TMPDIR + "/big.2", "w")
+        f1sz = 0
+        f2sz = 0
+        for file in rcsf.rcsfiles:
+            if file.versions < 2:
+                continue
+            r1 = 0
+            r2 = 0
+            while r1 == r2:
+                r1 = rand.randint(0, len(file.versions) - 1)
+                r2 = rand.randint(0, len(file.versions) - 1)
+            f1sz += file.AppendVersion(f1, r1)
+            f2sz += file.AppendVersion(f2, r2)
+
+        f1.close()
+        f2.close()
+
+        print "Test input sizes: %d %d" % (f1sz, f2sz)
+
+        BigFileRun(TMPDIR + "/big.1",
+                   TMPDIR + "/big.2")
+        #continue
+    #end
+
+def BigFileRun(f1, f2):
+                   
+    testcases = [
+        # large_look large_step small_look small_chain small_lchain
+        # ssmatch try_lazy max_lazy long_enough promote
+        ['-DC', '16,32,4,2,2,1,0,0,64,0'],
+        ['-DC', '24,32,4,2,2,1,0,0,64,0'],
+        ['-DC', '32,32,4,2,2,1,0,0,64,0'],
+    ]
+
+    for test in testcases:
+        runner = Xdelta3Pair()
+        runner.extra = ['-DC', test]
+        result = TimeRun(runner.Runner(f1, 1, f2, 2))
+
+        print 'test %s dsize %d: time %.7f: in %u/%u trials' % \
+              (test,
+               result.r1.dsize,
+               result.time.mean,
+               result.trials,
+               result.reps)
+    #end
+    return 1
+
+def RandomBigRun(f1, f2):
+
+    input_ranges = [
+        (8, 32, 4),
+        (8, 64, 8),
+        (4, 6, 1),
+        (1, 8, 1),
+        (1, 4, 1),
+        (0, 1, 1),
+        (0, 1, 1),
+        (8, 128, 32),
+        (8, 128, 32),
+        (0, 1, 1),
+    ]
+
+    config = []
     rand = random.Random()
-    f1 = open(TMPDIR + "/big.1", "w")
-    f2 = open(TMPDIR + "/big.2", "w")
-    f1sz = 0
-    f2sz = 0
-    for file in rcsf.rcsfiles:
-        if file.versions < 2:
-            continue
-        r1 = 0
-        r2 = 0
-        while r1 == r2:
-            r1 = rand.randint(0, len(file.versions) - 1)
-            r2 = rand.randint(0, len(file.versions) - 1)
-        f1sz += file.AppendVersion(f1, r1)
-        f2sz += file.AppendVersion(f2, r2)
-
-    f1.close()
-    f2.close()
-
-    print "Test input sizes: %d %d" % (f1sz, f2sz)
 
-    while 1:
+    for input in input_ranges:
+        config.append(str(rand.randrange(input[0], input[1] + 1, input[2])))
 
-        extras = [
-            ['-1'],
-            ['-C', '32,32,4,2,2,1,0,0,64,0'],
-            ['-9'],
-            ['-C', '64,64,4,128,16,0,1,8,128,0'],
-            ]
-
-        for extra in extras:
-            runner = Xdelta3Pair()
-            runner.extra = extra
-            result = TimeRun(runner.Runner(TMPDIR + "/big.1", 1,
-                                           TMPDIR + "/big.2", 2))
-
-            print 'testing %s dsize %d: time %.7f: in %u/%u trials' % \
-                  (extra,
-                   result.r1.dsize,
-                   result.time.mean,
-                   result.trials,
-                   result.reps)
-        # continue
-    # end
+    runner = Xdelta3Pair()
+    runner.extra = ['-DC', ','.join(config)]
+    result = TimeRun(runner.Runner(f1, 1, f2, 2))
+
+    print 'test %s dsize %d: time %.7f: in %u/%u trials' % \
+          (','.join(config),
+           result.r1.dsize,
+           result.time.mean,
+           result.trials,
+           result.reps)
 
 def RunSpeed():
     for L in Decimals(MAX_RUN):
@@ -616,11 +655,15 @@ def RunSpeed():
 if __name__ == "__main__":
     try:
         os.mkdir(TMPDIR)
-        rcsf = Test()
+        #rcsf = Test()
         #rcsf.PairsByDate(Xdelta3Pair())
         #RunSpeed()
-        BigFileRuns(rcsf)
+        #BigFileRuns(rcsf)
+        #BigFileRun("/tmp/big.1", "/tmp/big.2")
+        while 1:
+            RandomBigRun("/tmp/big.1", "/tmp/big.2")
     except CommandError:
         pass
-
-    RunCommand(['rm', '-rf', TMPDIR])
+    else:
+        #RunCommand(['rm', '-rf', TMPDIR])
+        pass