The rabin-karp checksum looks better in testing but doesn't really

seem to improve things in practice.  Removed HASH_PRIME.

6 files changed, 190 insertions, 132 deletions

diff --git a/xdelta3/examples/Makefile b/xdelta3/examples/Makefile
index f1caa57..7bc575c 100755
--- a/xdelta3/examples/Makefile
+++ b/xdelta3/examples/Makefile
@@ -1,4 +1,4 @@
-#CFLAGS = -g -Wall -I.. -DXD3_DEBUG=1
+#CFLAGS = -g -Wall -I.. -DXD3_DEBUG=1 -DNDEBUG=0
 CFLAGS = -O3 -Wall -I.. -DXD3_DEBUG=0 -fno-builtin -DNDEBUG=1
 # -pg
 
diff --git a/xdelta3/examples/checksum_test.cc b/xdelta3/examples/checksum_test.cc
index 40e2df1..15c6087 100644
--- a/xdelta3/examples/checksum_test.cc
+++ b/xdelta3/examples/checksum_test.cc
@@ -48,12 +48,6 @@ struct plain {
     }
 };
 
-struct permute {
-    int operator()(const uint8_t &c) {
-        return __single_hash[c];
-    }
-};
-
 template <typename Word>
 struct hhash {  // take "h" of the high-bits as a hash value for this
                 // checksum, which are the most "distant" in terms of the
@@ -152,6 +146,34 @@ struct rabin_karp {
     Word  incr_state;
 };
 
+MEMBER
+struct adler32_cksum {
+    typedef Word word_type;
+    typedef Permute permute_type;
+    typedef Hash hash_type;
+
+    enum { cksum_size = CksumSize,
+           cksum_skip = CksumSkip, 
+           compaction = Compaction,
+    };
+
+    Word step(const uint8_t *ptr) {
+        return xd3_lcksum (ptr, cksum_size);
+    }
+
+    Word state0(const uint8_t *ptr) {
+        incr_state = step(ptr);
+        return incr_state;
+    }
+
+    Word incr(const uint8_t *ptr) {
+        LARGE_CKSUM_UPDATE(incr_state, ptr - 1, cksum_size);
+        return incr_state;
+    }
+
+    Word  incr_state;
+};
+
 // TESTS
 
 template <typename Word>
@@ -222,12 +244,15 @@ struct test_result_base {
     }
     virtual void reset() = 0;
     virtual void print() = 0;
-    virtual void print_accum() = 0;
     virtual void get(const uint8_t* buf, const int buf_size, int iters) = 0;
     virtual void stat() = 0;
     virtual int count() = 0;
     virtual int dups() = 0;
     virtual double uniqueness() = 0;
+    virtual double fullness() = 0;
+    virtual double collisions() = 0;
+    virtual double coverage() = 0;
+    virtual double compression() = 0;
     virtual double time() = 0;
     virtual double score() = 0;
     virtual void set_score(double min_dups_frac, double min_time) = 0;
@@ -345,6 +370,18 @@ struct test_result : public test_result_base {
         return (double) h_buckets_full / (1 << h_bits);
     }
 
+    double collisions() {
+        return (double) colls() / fstats.unique;
+    }
+
+    double coverage() {
+        return (double) h_buckets_full / uniqueness() / count();
+    }
+
+    double compression() {
+        return 1.0 - coverage();
+    }
+
     double time() {
         return (double) accum_millis / accum_iters;
     }
@@ -353,8 +390,9 @@ struct test_result : public test_result_base {
         return h_score;
     }
 
-    void set_score(double min_uniqueness, double min_time) {
-        h_score = time();
+    void set_score(double min_compression, double min_time) {
+        h_score = (compression() - 0.99 * min_compression)
+                * (time() - 0.99 * min_time);
     }
 
     double total_time() {
@@ -381,29 +419,21 @@ struct test_result : public test_result_base {
         accum_size += test_size;
     }
 
-    void print_accum() {
-        printf("%s: (%u#%u) count %u dups %0.2f%% %.4f MB/s score %0.6f\n",
-               test_name,
-               cksum_size,
-               cksum_skip,
-               total_count(),
-               100.0 * total_dups() / total_count(),
-               0.001 * accum_size / time(),
-               h_score);
-    }
-
     void print() {
         if (fstats.count != count()) {
             fprintf(stderr, "internal error: %d != %d\n", fstats.count, count());
             abort();
         }
-        printf("%s: (%u#%u) count %u uniq %0.2f%% full %0.4f%% of 1<<%d @ %.4f MB/s %u iters\n",
+        printf("%s: (%u#%u) count %u uniq %0.2f%% full %u (%0.4f%% coll %0.4f%%) covers %0.2f%% w/ 2^%d @ %.4f MB/s %u iters\n",
                test_name,
                cksum_size,
                cksum_skip,
                count(),
                100.0 * uniqueness(),
+               h_buckets_full,
                100.0 * fullness(),
+               100.0 * collisions(),
+               100.0 * coverage(),
                h_bits,
                0.001 * accum_iters * test_size / accum_millis,
                accum_iters);
@@ -457,6 +487,7 @@ struct test_result : public test_result_base {
 
     void get(const uint8_t* buf, const int buf_size, int test_iters) {
         rabin_karp<SELF> test;
+        //adler32_cksum<SELF> test;
         hash_type hash;
         const uint8_t *ptr;
         const uint8_t *end;
@@ -545,6 +576,17 @@ struct test_result : public test_result_base {
     }
 };
 
+template <typename Word>
+void print_array(const char *tname) {
+    printf("static const %s hash_multiplier[64] = {\n", tname);
+    Word p = 1;
+    for (int i = 0; i < 64; i++) {
+        printf("  %uU,\n", p);
+        p *= good_word<Word>();
+    }
+    printf("};\n", tname);
+}
+
 int main(int argc, char** argv) {
   int i;
   uint8_t *buf = NULL;
@@ -556,21 +598,42 @@ int main(int argc, char** argv) {
     return 1;
   }
 
+  //print_array<uint32_t>("uint32_t");
+
 #define TEST(T,Z,S,P,H,C) test_result<T,Z,S,P,H<T>,C> \
       _ ## T ## _ ## Z ## _ ## S ## _ ## P ## _ ## H ## _ ## C \
       (#T "_" #Z "_" #S "_" #P "_" #H "_" #C)
 
+#if 0
+
+  TEST(uint32_t, 4, SKIP, plain, hhash, 0); /* x */ \
+  TEST(uint32_t, 4, SKIP, plain, hhash, 1); /* x */ \
+  TEST(uint32_t, 4, SKIP, plain, hhash, 2); /* x */ \
+  TEST(uint32_t, 4, SKIP, plain, hhash, 3); /* x */ \
+
+#endif
+
 #define TESTS(SKIP) \
+  TEST(uint32_t, 9, SKIP, plain, hhash, 0); /* x */ \
+  TEST(uint32_t, 9, SKIP, plain, hhash, 1); /* x */ \
+  TEST(uint32_t, 9, SKIP, plain, hhash, 2); /* x */ \
+  TEST(uint32_t, 9, SKIP, plain, hhash, 3)
+  
+#define TESTS_ALL(SKIP) \
   TEST(uint32_t, 3, SKIP, plain, hhash, 0); \
   TEST(uint32_t, 3, SKIP, plain, hhash, 1); \
   TEST(uint32_t, 4, SKIP, plain, hhash, 0); /* x */ \
   TEST(uint32_t, 4, SKIP, plain, hhash, 1); /* x */ \
+  TEST(uint32_t, 4, SKIP, plain, hhash, 2); /* x */ \
+  TEST(uint32_t, 4, SKIP, plain, hhash, 3); /* x */ \
   TEST(uint32_t, 5, SKIP, plain, hhash, 0); \
   TEST(uint32_t, 5, SKIP, plain, hhash, 1); \
   TEST(uint32_t, 8, SKIP, plain, hhash, 0); \
   TEST(uint32_t, 8, SKIP, plain, hhash, 1); \
   TEST(uint32_t, 9, SKIP, plain, hhash, 0); /* x */ \
   TEST(uint32_t, 9, SKIP, plain, hhash, 1); /* x */ \
+  TEST(uint32_t, 9, SKIP, plain, hhash, 2); /* x */ \
+  TEST(uint32_t, 9, SKIP, plain, hhash, 3); /* x */ \
   TEST(uint32_t, 11, SKIP, plain, hhash, 0); /* x */ \
   TEST(uint32_t, 11, SKIP, plain, hhash, 1); /* x */ \
   TEST(uint32_t, 13, SKIP, plain, hhash, 0); \
@@ -584,24 +647,22 @@ int main(int argc, char** argv) {
   TEST(uint32_t, 34, SKIP, plain, hhash, 0); \
   TEST(uint32_t, 34, SKIP, plain, hhash, 1); \
   TEST(uint32_t, 55, SKIP, plain, hhash, 0); \
-  TEST(uint32_t, 55, SKIP, plain, hhash, 1); \
-  TEST(uint32_t, 89, SKIP, plain, hhash, 0); \
-  TEST(uint32_t, 89, SKIP, plain, hhash, 1)
+  TEST(uint32_t, 55, SKIP, plain, hhash, 1)
 
   TESTS(1); // *
-  TESTS(2); // *
-  TESTS(3); // *
-  TESTS(5); // *
-  TESTS(8); // *
-  TESTS(9);
-  TESTS(11);
-  TESTS(13); // *
+//   TESTS(2); // *
+//   TESTS(3); // *
+//   TESTS(5); // *
+//   TESTS(8); // *
+//   TESTS(9);
+//   TESTS(11);
+//   TESTS(13); // *
   TESTS(15);
-  TESTS(16);
-  TESTS(21); // *
-  TESTS(34); // *
-  TESTS(55); // *
-  TESTS(89); // *
+//   TESTS(16);
+//   TESTS(21); // *
+//   TESTS(34); // *
+//   TESTS(55); // *
+//   TESTS(89); // *
 
   for (i = 1; i < argc; i++) {
     if ((ret = read_whole_file(argv[i],
@@ -614,7 +675,7 @@ int main(int argc, char** argv) {
             argv[i], buf_len);
 
     double min_time = -1.0;
-    double min_uniqueness = 0.0;
+    double min_compression = 0.0;
 
     for (vector<test_result_base*>::iterator i = all_tests.begin();
          i != all_tests.end(); ++i) {
@@ -633,7 +694,7 @@ int main(int argc, char** argv) {
         test->stat();
 
         if (min_time < 0.0) {
-            min_uniqueness = test->uniqueness();
+            min_compression = test->compression();
             min_time = test->time();
         }
 
@@ -641,26 +702,26 @@ int main(int argc, char** argv) {
             min_time = test->time();
         }
 
-        if (min_uniqueness > test->uniqueness()) {
-            min_uniqueness = test->uniqueness();
+        if (min_compression > test->compression()) {
+            min_compression = test->compression();
         }
 
         test->print();
     }
 
-    for (vector<test_result_base*>::iterator i = all_tests.begin();
-         i != all_tests.end(); ++i) {
-        test_result_base *test = *i;
-        test->set_score(min_uniqueness, min_time);
-    }   
+//     for (vector<test_result_base*>::iterator i = all_tests.begin();
+//       i != all_tests.end(); ++i) {
+//      test_result_base *test = *i;
+//      test->set_score(min_compression, min_time);
+//     }        
 
-    sort(all_tests.begin(), all_tests.end(), compare_h());
+//     sort(all_tests.begin(), all_tests.end(), compare_h());
     
-    for (vector<test_result_base*>::iterator i = all_tests.begin();
-         i != all_tests.end(); ++i) {
-        test_result_base *test = *i;
-        test->print_accum();
-    }   
+//     for (vector<test_result_base*>::iterator i = all_tests.begin();
+//       i != all_tests.end(); ++i) {
+//      test_result_base *test = *i;
+//      test->print();
+//     }        
     
     free(buf);
     buf = NULL;
diff --git a/xdelta3/xdelta3-hash.h b/xdelta3/xdelta3-hash.h
index 5d12804..462c323 100644
--- a/xdelta3/xdelta3-hash.h
+++ b/xdelta3/xdelta3-hash.h
@@ -41,8 +41,7 @@
   else { run_c = (c); run_l = 1; } } while (0)
 
 /* Update the checksum state. */
-#define OLD_LARGE_CKSUM 1
-#if OLD_LARGE_CKSUM
+#if ADLER_LARGE_CKSUM
 #define LARGE_CKSUM_UPDATE(cksum,base,look)                              \
   do {                                                                   \
     uint32_t old_c = PERMUTE((base)[0]);                                    \
@@ -52,16 +51,41 @@
     (cksum) = (high << 16) | low;                                        \
   } while (0)
 #else
-#define LARGE_CKSUM_UPDATE(cksum,base,look)                              \
+
+// This is a table of powers of 1597334677 (a good MLCG multiplier for 32-bit
+// modular arithmetic).  TODO: add citation for "linear congruential
+// generators of different sizes and good lattice structure"
+static const uint32_t hash_multiplier = 1597334677U;
+static const uint32_t hash_multiplier_powers[64] = {
+  1U,          1597334677U, 1664532153U, 1152009645U, 
+  438510001U,  2554380293U, 2226829033U, 2237430173U, 
+  785718369U,  1614047349U, 2189918233U, 3004453517U, 
+  2504269841U, 1873956325U, 3298675273U, 184041597U, 
+  412048577U,  513892437U,  1512523129U, 420197229U, 
+  3195470449U, 3365769157U, 782672297U,  1830755165U, 
+  3607904545U, 2926038069U, 2592473U,    2764040269U, 
+  2237745361U, 1146488229U, 3329146121U, 975735357U, 
+  4047063425U, 3720555541U, 992599097U,  2284876077U, 
+  1371042609U, 2776575877U, 857709673U,  1933315357U, 
+  18085345U,   3544121333U, 2019536281U, 657247757U, 
+  3376546193U, 235703653U,  3581067209U, 1515267069U, 
+  2578040385U, 3954624469U, 25854713U,   2236907245U, 
+  3587404785U, 2101452613U, 498181929U,  2150584029U, 
+  1830304417U, 3629515701U, 1929108569U, 4095149005U, 
+  3948038737U, 2377245989U, 565564041U,  309202365U, 
+};
+
+#define LARGE_CKSUM_UPDATE(cksum,base,look) \
   do { \
-    // linear congruential generators of different
-    // sizes and good lattice structure
-  } while (1)
+    cksum = (cksum * hash_multiplier) - \
+      (hash_multiplier_powers[look] * PERMUTE(base[0])) + \
+      PERMUTE(base[look]); \
+  } while (0)
 #endif
 
-/* Multiply and add hash function */
 #if ARITH_SMALL_CKSUM
-#define SMALL_CKSUM_UPDATE(cksum,base,look) (cksum) = ((*(unsigned long*)(base+1)) * 71143)
+#define SMALL_CKSUM_UPDATE(cksum,base,look) \
+  (cksum) = ((*(uint32_t*)(base+1)) * 1597334677U)
 #else
 #define SMALL_CKSUM_UPDATE LARGE_CKSUM_UPDATE
 #endif
@@ -118,40 +142,17 @@ static const uint16_t __single_hash[256] =
  Ctable stuff
  ***********************************************************************/
 
-#if HASH_PRIME
-static const usize_t __primes[] =
-{
-  11, 19, 37, 73, 109,
-  163, 251, 367, 557, 823,
-  1237, 1861, 2777, 4177, 6247,
-  9371, 14057, 21089, 31627, 47431,
-  71143, 106721, 160073, 240101, 360163,
-  540217, 810343, 1215497, 1823231, 2734867,
-  4102283, 6153409, 9230113, 13845163, 20767711,
-  31151543, 46727321, 70090921, 105136301, 157704401,
-  236556601, 354834919, 532252367, 798378509, 1197567719,
-  1796351503
-};
-
-static const usize_t __nprimes = SIZEOF_ARRAY (__primes);
-#endif
-
 static inline usize_t
 xd3_checksum_hash (const xd3_hash_cfg *cfg, const usize_t cksum)
 {
-#if HASH_PRIME
-  /* If the table is prime compute the modulus. */
-  return (cksum % cfg->size);
-#else
-  /* If the table is power-of-two compute the mask.*/
   return (cksum ^ (cksum >> cfg->shift)) & cfg->mask;
-#endif
 }
 
 /***********************************************************************
  Cksum function
  ***********************************************************************/
 
+#if ADLER_LARGE_CKSUM
 static inline uint32_t
 xd3_lcksum (const uint8_t *seg, const int ln)
 {
@@ -167,6 +168,18 @@ xd3_lcksum (const uint8_t *seg, const int ln)
 
   return ((high & 0xffff) << 16) | (low & 0xffff);
 }
+#else
+static inline uint32_t
+xd3_lcksum (const uint8_t *seg, const int ln)
+{
+  int i, j;
+  uint32_t h = 0;
+  for (i = 0, j = ln - 1; i < ln; ++i, --j) {
+    h += PERMUTE(seg[i]) * hash_multiplier_powers[j];
+  }
+  return h;
+}
+#endif
 
 #if ARITH_SMALL_CKSUM
 static inline usize_t
@@ -182,7 +195,6 @@ xd3_scksum (const uint8_t *seg, const int ln)
 #endif
 
 #if XD3_ENCODER
-#if !HASH_PRIME
 static usize_t
 xd3_size_log2 (usize_t slots)
 {
@@ -193,7 +205,8 @@ xd3_size_log2 (usize_t slots)
     {
       if (slots < (1U << i))
         {
-          bits = i-1;
+          bits = i-1; /* TODO: this is compaction=1 in checksum_test.cc and
+                       * should not be fixed at 1. */
           break;
         }
     }
@@ -201,37 +214,18 @@ xd3_size_log2 (usize_t slots)
   return bits;
 }
 #endif
-#endif
 
 static void
 xd3_size_hashtable (xd3_stream    *stream,
                     usize_t        slots,
                     xd3_hash_cfg  *cfg)
 {
-  /* initialize ctable: the number of hash buckets is computed from the table
-   * of primes or the nearest power-of-two, in both cases rounding down in
-   * favor of using less memory. */
-
-#if HASH_PRIME
-  usize_t i;
-
-  cfg->size = __primes[__nprimes-1];
-
-  for (i = 1; i < __nprimes; i += 1)
-    {
-      if (slots < __primes[i])
-        {
-          cfg->size = __primes[i-1];
-          break;
-        }
-    }
-#else
   int bits = xd3_size_log2 (slots);
 
+  /* TODO: there's a 32-bit assumption here */
   cfg->size  = (1 << bits);
   cfg->mask  = (cfg->size - 1);
-  cfg->shift = min (32 - bits, 16);
-#endif
+  cfg->shift = 32 - bits;
 }
 
 #endif
diff --git a/xdelta3/xdelta3-main.h b/xdelta3/xdelta3-main.h
index 5c02a75..58f4d31 100644
--- a/xdelta3/xdelta3-main.h
+++ b/xdelta3/xdelta3-main.h
@@ -375,9 +375,9 @@ main_config (void)
   DP(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE);
   DP(RINT "sizeof(int)=%d\n", sizeof(int));
   DP(RINT "sizeof(uint32_t)=%d\n", sizeof(uint32_t));
-  DP(RINT "sizeof(uint64_t)=%d\n", sizeof(uint64_t));  
+  DP(RINT "sizeof(uint64_t)=%d\n", sizeof(uint64_t));
   DP(RINT "sizeof(usize_t)=%d\n", sizeof(usize_t));
-  DP(RINT "sizeof(xoff_t)=%d\n", sizeof(xoff_t));  
+  DP(RINT "sizeof(xoff_t)=%d\n", sizeof(xoff_t));
 
   return EXIT_SUCCESS;
 }
diff --git a/xdelta3/xdelta3.c b/xdelta3/xdelta3.c
index 723a62a..8c14ef2 100644
--- a/xdelta3/xdelta3.c
+++ b/xdelta3/xdelta3.c
@@ -337,7 +337,7 @@ typedef enum {
 
 typedef enum {
   SEC_NOFLAGS     = 0,
-  SEC_COUNT_FREQS = (1 << 0), /* OPT: Not implemented: Could eliminate first pass of Huffman... */
+  SEC_COUNT_FREQS = (1 << 0), /* Note: Not implemented: eliminate 1st Huffman pass. */
 } xd3_secondary_flags;
 
 typedef enum {
@@ -378,34 +378,40 @@ XD3_MAKELIST(xd3_rlist, xd3_rinst, link);
 #define CODE_TABLE_STRING_SIZE (6 * 256) /* Should fit a code table string. */
 #define CODE_TABLE_VCDIFF_SIZE (6 * 256) /* Should fit a compressed code table string */
 
-#define SECONDARY_ANY (SECONDARY_DJW || SECONDARY_FGK) /* True if any secondary compressor is used. */
+#define SECONDARY_ANY (SECONDARY_DJW || SECONDARY_FGK)
 
-#define ALPHABET_SIZE      256  /* Used in test code--size of the secondary compressor alphabet. */
+#define ALPHABET_SIZE      256  /* Used in test code--size of the secondary
+                                 * compressor alphabet. */
 
-#define HASH_PRIME         0    /* Old hashing experiments */
-#define HASH_PERMUTE       1
-#define ARITH_SMALL_CKSUM  1
+#define HASH_PERMUTE       1    /* The input is permuted by random nums */
+#define ARITH_SMALL_CKSUM  1    /* Simple small checksum function -- faster than RK */
+#define ADLER_LARGE_CKSUM  1    /* Adler checksum vs. RK checksum */
 
-#define HASH_CKOFFSET      1U   /* Table entries distinguish "no-entry" from offset 0 using this offset. */
+#define HASH_CKOFFSET      1U   /* Table entries distinguish "no-entry" from
+                                 * offset 0 using this offset. */
 
 #define MIN_SMALL_LOOK    2U    /* Match-optimization stuff. */
 #define MIN_LARGE_LOOK    2U
 #define MIN_MATCH_OFFSET  1U
-#define MAX_MATCH_SPLIT   18U   /* VCDIFF code table: 18 is the default limit for direct-coded ADD sizes */
-
-#define LEAST_MATCH_INCR  0   /* The least number of bytes an overlapping match must beat
-                               * the preceding match by.  This is a bias for the lazy
-                               * match optimization.  A non-zero value means that an
-                               * adjacent match has to be better by more than the step
+#define MAX_MATCH_SPLIT   18U   /* VCDIFF code table: 18 is the default limit
+                                 * for direct-coded ADD sizes */
+
+#define LEAST_MATCH_INCR  0   /* The least number of bytes an overlapping
+                               * match must beat the preceding match by.  This
+                               * is a bias for the lazy match optimization.  A
+                               * non-zero value means that an adjacent match
+                               * has to be better by more than the step
                                * between them.  0. */
 
 #define MIN_MATCH         4U  /* VCDIFF code table: MIN_MATCH=4 */
 #define MIN_ADD           1U  /* 1 */
-#define MIN_RUN           8U  /* The shortest run, if it is shorter than this an immediate
-                               * add/copy will be just as good.  ADD1/COPY6 = 1I+1D+1A bytes,
-                               * RUN18 = 1I+1D+1A. */
+#define MIN_RUN           8U  /* The shortest run, if it is shorter than this
+                               * an immediate add/copy will be just as good.
+                               * ADD1/COPY6 = 1I+1D+1A bytes, RUN18 =
+                               * 1I+1D+1A. */
 
-#define MAX_MODES         9  /* Maximum number of nodes used for compression--does not limit decompression. */
+#define MAX_MODES         9  /* Maximum number of nodes used for
+                              * compression--does not limit decompression. */
 
 #define ENC_SECTS         4  /* Number of separate output sections. */
 
@@ -5067,12 +5073,6 @@ XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream)
 
           IF_DEBUG (xd3_verify_large_state (stream, inp, lcksum));
 
-          /* Note: To handle large checksum duplicates, this code
-           * should be rearranged to resemble the small_match case
-           * more.  But how much of the code can be truly shared?  The
-           * main difference is the need for xd3_source_extend_match
-           * to work outside of xd3_string_match, in the case where
-           * inputs are identical. */
           if (stream->large_table[linx] != 0)
             {
               /* the match_setup will fail if the source window has
diff --git a/xdelta3/xdelta3.h b/xdelta3/xdelta3.h
index 1dda36e..48d766f 100644
--- a/xdelta3/xdelta3.h
+++ b/xdelta3/xdelta3.h
@@ -119,6 +119,9 @@ typedef ULONGLONG      uint64_t;
 #define XD3_USE_LARGEFILE64 1
 #endif
 
+/* TODO: note that SIZEOF_USIZE_T is never set to 8, although it should be for
+ * a 64bit platform.  OTOH, may be that using 32bits is appropriate even on a
+ * 64bit platform because we allocate large arrays of these values. */
 #if XD3_USE_LARGEFILE64
 #define __USE_FILE_OFFSET64 1 /* GLIBC: for 64bit fileops, ... ? */
 typedef uint64_t xoff_t;