Xdelta3.Internal

1 files changed, 962 insertions, 0 deletions

diff --git a/haskell/Xdelta3/Internal.hsc b/haskell/Xdelta3/Internal.hsc
new file mode 100644
index 0000000..4740861
--- /dev/null
+++ b/haskell/Xdelta3/Internal.hsc
@@ -0,0 +1,962 @@
+{-# LANGUAGE BangPatterns    #-}
+{-# LANGUAGE PatternSynonyms #-}
+module Xdelta3.Internal where
+
+import Data.Word
+import Foreign.C.Types
+import Foreign.C.String
+import Foreign.Marshal.Alloc
+import Foreign.Ptr
+
+-- #define HAVE_CONFIG_H 1
+
+#define SIZEOF_SIZE_T __SIZEOF_SIZE_T__
+#define SIZEOF_UNSIGNED_INT __SIZEOF_INT__
+#define SIZEOF_UNSIGNED_LONG __SIZEOF_LONG__
+#define SIZEOF_UNSIGNED_LONG_LONG __SIZEOF_LONG_LONG__
+#define static_assert(...)
+
+#include <xdelta3.h>
+
+
+-- To learn more about Xdelta, start by reading xdelta3.c.  If you are
+-- ready to use the API, continue reading here.  There are two
+-- interfaces -- xd3_encode_input and xd3_decode_input -- plus a dozen
+-- or so related calls.  This interface is styled after Zlib.
+
+
+-- | Default configured value of stream->winsize.  If the program
+-- supplies xd3_encode_input() with data smaller than winsize the
+-- stream will automatically buffer the input, otherwise the input
+-- buffer is used directly.
+pattern XD3_DEFAULT_WINSIZE = #const XD3_DEFAULT_WINSIZE
+
+-- | Default total size of the source window used in xdelta3-main.h
+pattern XD3_DEFAULT_SRCWINSZ = #const XD3_DEFAULT_SRCWINSZ
+
+-- When Xdelta requests a memory allocation for certain buffers, it
+-- rounds up to units of at least this size.  The code assumes (and
+-- asserts) that this is a power-of-two.
+pattern XD3_ALLOCSIZE = #const XD3_ALLOCSIZE
+
+
+-- The XD3_HARDMAXWINSIZE parameter is a safety mechanism to protect
+-- decoders against malicious files.  The decoder will never decode a
+-- window larger than this.  If the file specifies VCD_TARGET the
+-- decoder may require two buffers of this size.
+--
+-- 8-16MB is reasonable, probably don't need to go larger.
+pattern XD3_HARDMAXWINSIZE = #const XD3_HARDMAXWINSIZE
+
+
+-- The IOPT_SIZE value sets the size of a buffer used to batch
+-- overlapping copy instructions before they are optimized by picking
+-- the best non-overlapping ranges.  The larger this buffer, the
+-- longer a forced xd3_srcwin_setup() decision is held off.  Setting
+-- this value to 0 causes an unlimited buffer to be used. */
+pattern XD3_DEFAULT_IOPT_SIZE = #const XD3_DEFAULT_IOPT_SIZE
+
+-- The maximum distance backward to search for small matches */
+pattern XD3_DEFAULT_SPREVSZ = #const XD3_DEFAULT_SPREVSZ
+
+-- The default compression level */
+pattern XD3_DEFAULT_LEVEL = #const XD3_DEFAULT_LEVEL
+
+pattern XD3_DEFAULT_SECONDARY_LEVEL = #const XD3_DEFAULT_SECONDARY_LEVEL
+
+pattern XD3_USE_LARGEFILE64 = #const XD3_USE_LARGEFILE64
+
+-- The source window size is limited to 2GB unless
+-- XD3_USE_LARGESIZET is defined to 1. */
+pattern XD3_USE_LARGESIZET = #const XD3_USE_LARGESIZET
+
+
+-- Sizes and addresses within VCDIFF windows are represented as usize_t
+--
+-- For source-file offsets and total file sizes, total input and
+-- output counts, the xoff_t type is used.  The decoder and encoder
+-- generally check for overflow of the xoff_t size (this is tested at
+-- the 32bit boundary [xdelta3-test.h]).
+
+-- Settings based on the size of xoff_t (32 vs 64 file offsets) */
+#if XD3_USE_LARGEFILE64
+-- xoff_t is a 64-bit type */
+pattern C__USE_FILE_OFFSET64 = #const __USE_FILE_OFFSET64
+
+pattern C_FILE_OFFSET_BITS = #const _FILE_OFFSET_BITS
+#endif
+
+#if SIZEOF_XOFF_T == 8
+type Xoff_t = Word64
+pattern Q = #const_str Q
+#elif SIZEOF_XOFF_T == 4
+type Xoff_t = Word32
+pattern Q = ""
+#endif
+
+pattern SIZEOF_XOFF_T = #const SIZEOF_XOFF_T
+
+#if SIZEOF_USIZE_T == 8
+type Usize_t = Word64
+pattern W = #const_str W
+#elif SIZEOF_USIZE_T == 4
+type Usize_t = Word32
+pattern W = ""
+#endif
+
+pattern SIZEOF_USIZE_T = #const SIZEOF_USIZE_T
+
+pattern SIZEOF_SIZE_T = #const SIZEOF_SIZE_T
+pattern Z = #const_str Z
+
+pattern USE_UINT32 = #const USE_UINT32
+pattern USE_UINT64 = #const USE_UINT64
+
+pattern UNALIGNED_OK = #const UNALIGNED_OK
+
+pattern XD3_ENCODER = #const XD3_ENCODER
+
+-- The code returned when main() fails, also defined in system
+-- includes.
+pattern EXIT_FAILURE = #const EXIT_FAILURE
+
+-- REGRESSION TEST enables the "xdelta3 test" command, which runs a
+-- series of self-tests.
+pattern REGRESSION_TEST = #const REGRESSION_TEST
+
+pattern PYTHON_MODULE = #const PYTHON_MODULE
+
+pattern SWIG_MODULE = #const SWIG_MODULE
+
+pattern NOT_MAIN = #const NOT_MAIN
+
+-- There are three string matching functions supplied: one fast, one
+-- slow (default), and one soft-configurable.  To disable any of
+-- these, use the following definitions.
+pattern XD3_BUILD_SLOW = #const XD3_BUILD_SLOW
+pattern XD3_BUILD_FAST = #const XD3_BUILD_FAST
+pattern XD3_BUILD_FASTER = #const XD3_BUILD_FASTER
+pattern XD3_BUILD_FASTEST = #const XD3_BUILD_FASTEST
+pattern XD3_BUILD_SOFT = #const XD3_BUILD_SOFT
+pattern XD3_BUILD_DEFAULT = #const XD3_BUILD_DEFAULT
+
+
+-- newtype Xd3_stream             = Xd3_stream           ( Ptr Xd3_stream           )
+-- newtype Xd3_source             = Xd3_source           ( Ptr Xd3_source           )
+newtype Xd3_hash_cfg           = Xd3_hash_cfg         ( Ptr Xd3_hash_cfg         )
+newtype Xd3_smatcher           = Xd3_smatcher         ( Ptr Xd3_smatcher         )
+newtype Xd3_rinst              = Xd3_rinst            ( Ptr Xd3_rinst            )
+newtype Xd3_dinst              = Xd3_dinst            ( Ptr Xd3_dinst            )
+newtype Xd3_hinst              = Xd3_hinst            ( Ptr Xd3_hinst            )
+newtype Xd3_winst              = Xd3_winst            ( Ptr Xd3_winst            )
+newtype Xd3_rpage              = Xd3_rpage            ( Ptr Xd3_rpage            )
+newtype Xd3_addr_cache         = Xd3_addr_cache       ( Ptr Xd3_addr_cache       )
+newtype Xd3_output             = Xd3_output           ( Ptr Xd3_output           )
+newtype Xd3_desect             = Xd3_desect           ( Ptr Xd3_desect           )
+newtype Xd3_iopt_buflist       = Xd3_iopt_buflist     ( Ptr Xd3_iopt_buflist     )
+newtype Xd3_rlist              = Xd3_rlist            ( Ptr Xd3_rlist            )
+newtype Xd3_sec_type           = Xd3_sec_type         ( Ptr Xd3_sec_type         )
+-- newtype Xd3_sec_cfg            = Xd3_sec_cfg          ( Ptr Xd3_sec_cfg          )
+newtype Xd3_sec_stream         = Xd3_sec_stream       ( Ptr Xd3_sec_stream       )
+-- newtype Xd3_config             = Xd3_config           ( Ptr Xd3_config           )
+newtype Xd3_code_table_desc    = Xd3_code_table_desc  ( Ptr Xd3_code_table_desc  )
+newtype Xd3_code_table_sizes   = Xd3_code_table_sizes ( Ptr Xd3_code_table_sizes )
+newtype Xd3_slist              = Xd3_slist            ( Ptr Xd3_slist            )
+newtype Xd3_whole_state        = Xd3_whole_state      ( Ptr Xd3_whole_state      )
+newtype Xd3_wininfo            = Xd3_wininfo          ( Ptr Xd3_wininfo          )
+
+
+
+-- The stream configuration has three callbacks functions, all of
+-- which may be supplied with NULL values.  If config->getblk is
+-- provided as NULL, the stream returns XD3_GETSRCBLK. */
+
+-- typedef void* (xd3_alloc_func) (void       *opaque,
+--                                 size_t      items,
+--                                 usize_t     size);
+foreign export ccall hs_alloc :: Ptr ()
+                              -> #type size_t
+                              -> #type usize_t
+                              -> IO ( Ptr                   ( ))
+hs_alloc _ items size = mallocBytes (fromIntegral items * fromIntegral size)
+
+-- typedef void  (xd3_free_func) (void *opaque,
+--                                void *address);
+foreign export ccall hs_free :: Ptr () -> Ptr () -> IO ()
+hs_free _ address = free address
+
+{-
+/* third callback */
+typedef int    (xd3_getblk_func)   (xd3_stream *stream,
+                                    xd3_source *source,
+                                    xoff_t      blkno);
+
+typedef const xd3_dinst* (xd3_code_table_func) (void);
+-
+-}
+
+-- Type used for short snprintf calls. */
+-- typedef struct { char buf[48]; } shortbuf;
+data Shortbuf = Shortbuf {-# UNPACK #-} !Word64
+                         {-# UNPACK #-} !Word64
+                         {-# UNPACK #-} !Word64
+                         {-# UNPACK #-} !Word64
+                         {-# UNPACK #-} !Word64
+                         {-# UNPACK #-} !Word64
+
+
+
+-- XPR(NT "", ...) (used by main) prefixes an "xdelta3: " to the output. */
+-- void xprintf(const char *fmt, ...) PRINTF_ATTRIBUTE(1,2);
+-- foreign import ccall "xdelta3.h xprintf" xprintf :: ?
+
+pattern NT = #const_str NT
+pattern NTR = #const_str NTR
+pattern RINT = #const_str RINT
+
+
+-----------------------------------------------------------------
+-- PUBLIC ENUMS
+--------------------------------------------------------------------
+
+-- | These are the five ordinary status codes returned by the
+-- xd3_encode_input() and xd3_decode_input() state machines.
+--
+-- An application must be prepared to handle these five return
+-- values from either xd3_encode_input or xd3_decode_input except
+-- in the case of no-source compression in which case XD3_GETSRCBLK
+-- is never returned.  More detailed comments for these are given in
+-- xd3_encode_input and xd3_decode_input comments below.
+newtype Xd3_rvalues = Xd3_rvalues CInt
+
+-- | need input
+pattern XD3_INPUT         = Xd3_rvalues (#const XD3_INPUT)
+
+-- | have output
+pattern XD3_OUTPUT        = Xd3_rvalues (#const XD3_OUTPUT)
+
+-- | need a block of source input (with no xd3_getblk function) a chance to do non-blocking read.
+pattern XD3_GETSRCBLK     = Xd3_rvalues (#const XD3_GETSRCBLK)
+
+-- | (decode-only) after the initial VCDIFF & first window header
+pattern XD3_GOTHEADER     = Xd3_rvalues (#const XD3_GOTHEADER)
+
+-- | notification: returned before a window is processed giving a chance to XD3_SKIP_WINDOW or not XD3_SKIP_EMIT that window.
+pattern XD3_WINSTART      = Xd3_rvalues (#const XD3_WINSTART)
+
+-- | notification: returned after encode/decode & output for a window
+pattern XD3_WINFINISH     = Xd3_rvalues (#const XD3_WINFINISH)
+
+-- | (encoder only) may be returned by getblk() if the block is too old
+pattern XD3_TOOFARBACK    = Xd3_rvalues (#const XD3_TOOFARBACK)
+
+-- | internal error
+pattern XD3_INTERNAL      = Xd3_rvalues (#const XD3_INTERNAL)
+
+-- | invalid config
+pattern XD3_INVALID       = Xd3_rvalues (#const XD3_INVALID)
+
+-- | invalid input/decoder error
+pattern XD3_INVALID_INPUT = Xd3_rvalues (#const XD3_INVALID_INPUT)
+
+-- | when secondary compression finds no improvement.
+pattern XD3_NOSECOND      = Xd3_rvalues (#const XD3_NOSECOND)
+
+-- | currently VCD_TARGET VCD_CODETABLE
+pattern XD3_UNIMPLEMENTED = Xd3_rvalues (#const XD3_UNIMPLEMENTED)
+
+
+
+
+-- | special values in config->flags
+newtype Xd3_flags = Xd3_flags Word32
+
+-- used by VCDIFF tools, see xdelta3-main.h.--/
+pattern XD3_JUST_HDR       = Xd3_flags (#const XD3_JUST_HDR)
+
+-- used by VCDIFF tools see xdelta3-main.h.--/
+pattern XD3_SKIP_WINDOW    = Xd3_flags (#const XD3_SKIP_WINDOW)
+
+-- | used by VCDIFF tools, see xdelta3-main.h. */
+pattern XD3_SKIP_EMIT      = Xd3_flags (#const XD3_SKIP_EMIT)
+-- | flush the stream buffer to prepare for xd3_stream_close(). */
+pattern XD3_FLUSH          = Xd3_flags (#const XD3_FLUSH)
+
+-- | use DJW static huffman */
+pattern XD3_SEC_DJW        = Xd3_flags (#const XD3_SEC_DJW)
+-- | use FGK adaptive huffman */
+pattern XD3_SEC_FGK        = Xd3_flags (#const XD3_SEC_FGK)
+-- | use LZMA secondary */
+pattern XD3_SEC_LZMA       = Xd3_flags (#const XD3_SEC_LZMA)
+
+pattern XD3_SEC_TYPE       = Xd3_flags (#const XD3_SEC_TYPE)
+
+-- | disable secondary compression of the data section. */
+pattern XD3_SEC_NODATA     = Xd3_flags (#const XD3_SEC_NODATA)
+-- | disable secondary compression of the inst section. */
+pattern XD3_SEC_NOINST     = Xd3_flags (#const XD3_SEC_NOINST)
+-- | disable secondary compression of the addr section. */
+pattern XD3_SEC_NOADDR     = Xd3_flags (#const XD3_SEC_NOADDR)
+
+pattern XD3_SEC_NOALL      = Xd3_flags (#const XD3_SEC_NOALL)
+
+-- | enable checksum computation in the encoder. */
+pattern XD3_ADLER32        = Xd3_flags (#const XD3_ADLER32)
+-- | disable checksum verification in the decoder. */
+pattern XD3_ADLER32_NOVER  = Xd3_flags (#const XD3_ADLER32_NOVER)
+
+-- | disable ordinary data * compression feature, only search * the source, not the target. */
+pattern XD3_NOCOMPRESS     = Xd3_flags (#const XD3_NOCOMPRESS)
+-- | disable the "1.5-pass * algorithm", instead use greedy * matching.  Greedy is off by * default. */
+pattern XD3_BEGREEDY       = Xd3_flags (#const XD3_BEGREEDY)
+-- | used by "recode". */
+pattern XD3_ADLER32_RECODE = Xd3_flags (#const XD3_ADLER32_RECODE)
+
+-- 4 bits to set the compression level the same as the command-line
+-- setting -1 through -9 Xd3_flags (-0 corresponds to the XD3_NOCOMPRESS flag
+-- and is independent of compression level).  This is for
+-- convenience especially with xd3_encode_memoryXd3_flags (). */
+
+pattern XD3_COMPLEVEL_SHIFT = Xd3_flags (#const XD3_COMPLEVEL_SHIFT)
+pattern XD3_COMPLEVEL_MASK  = Xd3_flags (#const XD3_COMPLEVEL_MASK)
+pattern XD3_COMPLEVEL_1     = Xd3_flags (#const XD3_COMPLEVEL_1)
+pattern XD3_COMPLEVEL_2     = Xd3_flags (#const XD3_COMPLEVEL_2)
+pattern XD3_COMPLEVEL_3     = Xd3_flags (#const XD3_COMPLEVEL_3)
+pattern XD3_COMPLEVEL_6     = Xd3_flags (#const XD3_COMPLEVEL_6)
+pattern XD3_COMPLEVEL_9     = Xd3_flags (#const XD3_COMPLEVEL_9)
+
+-- | The values of this enumeration are set in xd3_config using the
+-- smatch_cfg variable.  It can be set to default, slow, fast, etc.,
+-- and soft.
+data Xd3_smatch_cfg
+    = XD3_SMATCH_DEFAULT -- ^ Flags may contain XD3_COMPLEVEL bits, else default.
+    | XD3_SMATCH_SLOW
+    | XD3_SMATCH_FAST
+    | XD3_SMATCH_FASTER
+    | XD3_SMATCH_FASTEST
+    | XD3_SMATCH_SOFT
+ deriving Enum
+
+#if 0
+
+----------------------------------------------------------------------
+-- PRIVATE ENUMS
+----------------------------------------------------------------------
+
+-- | stream->match_state is part of the xd3_encode_input state machine
+--  for source matching:
+--
+--  1. the XD3_GETSRCBLK block-read mechanism means reentrant matching
+--  2. this state spans encoder windows: a match and end-of-window
+--  will continue in the next 3. the initial target byte and source
+--  byte are a presumed match, to avoid some computation in case the
+--  inputs are identical.
+data Xd3_match_state
+    = MATCH_TARGET    -- ^ in this state, attempt to match the start of the
+                      -- target with the previously set source address (initially
+                      -- 0).
+    | MATCH_BACKWARD  -- ^ currently expanding a match backward in the
+                      -- source/target.
+    | MATCH_FORWARD   -- ^ currently expanding a match forward in the
+                      -- source/target.
+    | MATCH_SEARCHING -- ^ currently searching for a match.
+ deriving Enum
+
+
+-- The xd3_encode_input state machine steps through these states in
+-- the following order.  The matcher is reentrant and returns
+-- XD3_INPUT whenever it requires more data.  After receiving
+-- XD3_INPUT, if the application reads EOF it should call
+-- xd3_stream_close().
+data Xd3_encode_input
+    = ENC_INIT    -- ^ xd3_encode_input has never been called.
+    | ENC_INPUT   -- ^ waiting for xd3_avail_input () to be called.
+    | ENC_SEARCH  -- ^ currently searching for matches.
+    | ENC_INSTR   -- ^ currently formatting output.
+    | ENC_FLUSH   -- ^ currently emitting output.
+    | ENC_POSTOUT -- ^ after an output section.
+    | ENC_POSTWIN -- ^ after all output sections.
+    | ENC_ABORTED -- ^ abort.
+ deriving Enum
+
+-- | The xd3_decode_input state machine steps through these states in
+-- the following order.  The matcher is reentrant and returns
+-- XD3_INPUT whenever it requires more data.  After receiving
+-- XD3_INPUT, if the application reads EOF it should call
+-- xd3_stream_close().
+--
+-- 0-8:   the VCDIFF header
+-- 9-18:  the VCDIFF window header
+-- 19-21: the three primary sections: data, inst, addr
+-- 22:    producing output: returns XD3_OUTPUT, possibly XD3_GETSRCBLK,
+-- 23:    return XD3_WINFINISH, set state=9 to decode more input
+data Xd3_decode_state
+  = DEC_VCHEAD -- ^ VCDIFF header
+  | DEC_HDRIND -- ^ header indicator
+
+  | DEC_SECONDID -- ^ secondary compressor ID
+
+  | DEC_TABLEN -- ^ code table length
+  | DEC_NEAR   -- ^ code table near
+  | DEC_SAME   -- ^ code table same
+  | DEC_TABDAT -- ^ code table data
+
+  | DEC_APPLEN -- ^ application data length
+  | DEC_APPDAT -- ^ application data
+
+  | DEC_WININD -- ^ window indicator
+
+  | DEC_CPYLEN -- ^ copy window length
+  | DEC_CPYOFF -- ^ copy window offset
+
+  | DEC_ENCLEN -- ^ length of delta encoding
+  | DEC_TGTLEN -- ^ length of target window
+  | DEC_DELIND -- ^ delta indicator
+
+  | DEC_DATALEN -- ^ length of ADD+RUN data
+  | DEC_INSTLEN -- ^ length of instruction data
+  | DEC_ADDRLEN -- ^ length of address data
+
+  | DEC_CKSUM -- ^ window checksum
+
+  | DEC_DATA -- ^ data section
+  | DEC_INST -- ^ instruction section
+  | DEC_ADDR -- ^ address section
+
+  | DEC_EMIT -- ^ producing data
+
+  | DEC_FINISH -- ^ window finished
+
+  | DEC_ABORTED -- ^ xd3_abort_stream
+  deriving Enum
+
+-------------------------------------------------------------
+-- internal types
+--------------------------------------------------------------
+
+-- skipped.
+
+#endif
+
+---------------------------------------------------------------------
+-- public types
+---------------------------------------------------------------------
+
+-- | Settings for the secondary compressor.
+data Xd3_sec_cfg = Xd3_sec_cfg
+  { data_type   :: Int     -- ^  Which section. (set automatically)
+  , ngroups     :: Usize_t -- ^  Number of DJW Huffman groups.
+  , sector_size :: Usize_t -- ^  Sector size.
+  , inefficient :: Int     -- ^  If true, ignore efficiency check [avoid XD3_NOSECOND].
+  }
+
+-- | This is the user-visible stream configuration. */
+data Xd3_config = Xd3_config
+  { winsize   :: Usize_t -- ^  The encoder window size.
+  , sprevsz   :: Usize_t -- ^  How far back small string matching goes
+  , iopt_size :: Usize_t -- ^  entries in the instruction-optimizing buffer
+
+   -- xd3_getblk_func   *getblk;        /* The three callbacks. */
+   -- xd3_alloc_func    *alloc;
+   -- xd3_free_func     *freef;
+   -- void              *opaque;        /* Not used. */
+
+  , flags         :: Xd3_flags      -- ^  stream->flags are initialized from xd3_config & never modified by the library.  Use xd3_set_flags to modify flags settings mid-stream.
+  , sec_data      :: Xd3_sec_cfg    -- ^  Secondary compressor config: data
+  , sec_inst      :: Xd3_sec_cfg    -- ^  Secondary compressor config: inst
+  , sec_addr      :: Xd3_sec_cfg    -- ^  Secondary compressor config: addr
+  , smatch_cfg    :: Xd3_smatch_cfg -- ^  See enum: use fields below  for soft config
+  , smatcher_soft :: Xd3_smatcher
+  }
+
+-- | The primary source file object. You create one of these objects and
+-- initialize the first four fields.  This library maintains the next
+-- 5 fields.  The configured getblk implementation is responsible for
+-- setting the final 3 fields when called (and/or when XD3_GETSRCBLK
+-- is returned).
+data Xd3_source = Xd3_source
+ {
+  -- you set
+    blksize     :: Usize_t -- ^  block size
+  , name        :: String  -- ^ its name, for debug/print purposes
+  , ioh         :: Ptr ()  -- ^ opaque handle */
+  , max_winsize :: Xoff_t  -- ^  maximum visible buffer
+
+  -- getblk sets
+  , curblkno :: Xoff_t    -- ^  current block number: client sets after getblk request
+  , onblk    :: Usize_t   -- ^  number of bytes on current block: client sets,  must be >= 0 and <= blksize
+  , curblk   :: Ptr Word8 -- ^ current block array: client sets after getblk request */
+
+  -- xd3 sets
+  , srclen        :: Usize_t -- ^  length of this source window
+  , srcbase       :: Xoff_t  -- ^  offset of this source window in the source itself
+  , shiftby       :: Usize_t -- ^  for power-of-two blocksizes
+  , maskby        :: Usize_t -- ^  for power-of-two blocksizes
+  , cpyoff_blocks :: Xoff_t  -- ^  offset of dec_cpyoff in blocks
+  , cpyoff_blkoff :: Usize_t -- ^  offset of copy window in blocks, remainder
+  , getblkno      :: Xoff_t  -- ^  request block number: xd3 sets current getblk request
+
+  -- See xd3_getblk()
+  , max_blkno :: Xoff_t  -- ^  Maximum block, if eof is known, otherwise, equals frontier_blkno (initially 0).
+  , onlastblk :: Usize_t -- ^  Number of bytes on max_blkno
+  , eof_known :: Int     -- ^  Set to true when the first partial block is read.
+}
+
+-- | The primary xd3_stream object, used for encoding and decoding.  You
+-- may access only two fields: avail_out, next_out.  Use the methods
+-- above to operate on xd3_stream.
+
+data Xd3_stream = Xd3_stream
+  {
+  -- input state */
+    next_in  :: Ptr Word8 -- ^ next input byte */
+  , avail_in :: Usize_t   -- ^  number of bytes available at next_in
+  , total_in :: Xoff_t    -- ^  how many bytes in
+
+  -- output state */
+  , next_out       :: Ptr Word8 -- ^ next output byte */
+  , avail_out      :: Usize_t   -- ^  number of bytes available at next_out
+  , space_out      :: Usize_t   -- ^  total out space
+  , current_window :: Xoff_t    -- ^  number of windows encoded/decoded
+  , total_out      :: Xoff_t    -- ^  how many bytes out
+
+  -- to indicate an error, xd3 sets */
+  , msg :: Maybe String -- last error message, NULL if no error */
+
+  -- source configuration */
+  , src :: Xd3_source       -- ^ source array */
+
+  -- encoder memory configuration */
+  -- , winsize        :: Usize_t -- ^  suggested window size
+  -- , sprevsz        :: Usize_t -- ^  small string, previous window size (power of 2)
+  , sprevmask      :: Usize_t -- ^  small string, previous window size mask
+  -- , iopt_size      :: Usize_t
+  , iopt_unlimited :: Usize_t
+
+  -- general configuration */
+  -- xd3_getblk_func  *getblk;           /* set nxtblk, nxtblkno to scanblkno */
+  -- xd3_alloc_func   *alloc;            /* malloc function */
+  -- xd3_free_func    *free;             /* free function */
+  , opaque :: Ptr ()   -- ^  private data object passed to alloc, free, and getblk
+  -- , flags  :: Uint32_t -- ^  various options
+
+  -- secondary compressor configuration */
+  -- , sec_data :: Xd3_sec_cfg -- ^  Secondary compressor config: data
+  -- , sec_inst :: Xd3_sec_cfg -- ^  Secondary compressor config: inst
+  -- , sec_addr :: Xd3_sec_cfg -- ^  Secondary compressor config: addr
+
+  , smatcher :: Xd3_smatcher
+
+  {-
+
+  usize_t           *large_table;      /* table of large checksums */
+  , large_hash :: xd3_hash_cfg       -- ^  large hash config
+
+  usize_t           *small_table;      /* table of small checksums */
+  xd3_slist         *small_prev;       /* table of previous offsets, circular linked list */
+  , small_reset :: int                -- ^  true if small table should be reset
+
+  , small_hash :: xd3_hash_cfg       -- ^  small hash config
+  , acache :: xd3_addr_cache     -- ^  the vcdiff address cache
+  , enc_state :: xd3_encode_state   -- ^  state of the encoder
+
+  , taroff :: usize_t            -- ^  base offset of the target input
+  , input_position :: usize_t            -- ^  current input position
+  , min_match :: usize_t            -- ^  current minimum match length, avoids redundent matches
+  , unencoded_offset :: usize_t            -- ^  current input, first * unencoded offset. this value is <= the first instruction's position in the iopt buffer, if there is at least one match in the buffer.
+
+  /* SRCWIN */
+  , srcwin_decided :: int                -- ^  boolean: true if srclen and srcbase have been decided.
+  , srcwin_decided_early :: int                -- ^  boolean: true if srclen and srcbase were decided early.
+  , srcwin_cksum_pos :: xoff_t             -- ^  Source checksum position
+
+  /* MATCH */
+  , match_state       :: xd3_match_state    -- ^  encoder match state
+  , match_srcpos      :: xoff_t             -- ^  current match source position relative to srcbase
+  , match_last_srcpos :: xoff_t             -- ^  previously attempted srcpos, to avoid loops.
+  , match_minaddr     :: xoff_t             -- ^  smallest matching address to set window params (reset each window xd3_encode_reset)
+  , match_maxaddr     :: xoff_t             -- ^  largest matching address to set window params (reset each window xd3_encode_reset)
+  , match_back        :: usize_t            -- ^  match extends back so far
+  , match_maxback     :: usize_t            -- ^  match extends back maximum
+  , match_fwd         :: usize_t            -- ^  match extends forward so far
+  , match_maxfwd      :: usize_t            -- ^  match extends forward maximum
+
+  , maxsrcaddr :: xoff_t             -- ^  address of the last source match (across windows)
+
+  uint8_t          *buf_in;           /* for saving buffered input */
+  , buf_avail :: usize_t           -- ^  amount of saved input
+  const uint8_t    *buf_leftover;     /* leftover content of next_in (i.e., user's buffer) */
+  , buf_leftavail :: usize_t            -- ^  amount of leftover content
+
+  xd3_output       *enc_current;      /* current output buffer */
+  xd3_output       *enc_free;         /* free output buffers */
+  xd3_output       *enc_heads[4];     /* array of encoded outputs: head of chain */
+  xd3_output       *enc_tails[4];     /* array of encoded outputs: tail of chain */
+  uint32_t          recode_adler32;   /* set the adler32 checksum * during "recode". */
+
+  , iopt_used :: xd3_rlist         -- ^  instruction optimizing buffer
+  xd3_rlist         iopt_free;
+  xd3_rinst        *iout;             /* next single instruction */
+  xd3_iopt_buflist *iopt_alloc;
+
+  const uint8_t    *enc_appheader;    /* application header to encode */
+  usize_t            enc_appheadsz;    /* application header size */
+
+  /* decoder stuff */
+  , dec_state :: xd3_decode_state  -- ^  current DEC_XXX value
+  , dec_hdr_ind :: usize_t           -- ^  VCDIFF header indicator
+  , dec_win_ind :: usize_t           -- ^  VCDIFF window indicator
+  , dec_del_ind :: usize_t           -- ^  VCDIFF delta indicator
+
+  uint8_t           dec_magic[4];     /* First four bytes */
+  , dec_magicbytes :: usize_t           -- ^  Magic position.
+
+  , dec_secondid :: usize_t           -- ^  Optional secondary compressor ID.
+
+  , dec_codetblsz :: usize_t           -- ^  Optional code table: length.
+  uint8_t          *dec_codetbl;      /* Optional code table: storage. */
+  , dec_codetblbytes :: usize_t           -- ^  Optional code table: position.
+
+  , dec_appheadsz :: usize_t           -- ^  Optional application header: size.
+  uint8_t          *dec_appheader;    /* Optional application header: storage */
+  , dec_appheadbytes :: usize_t           -- ^  Optional application header: position.
+
+  , dec_cksumbytes :: usize_t            -- ^  Optional checksum: position.
+  uint8_t           dec_cksum[4];     /* Optional checksum: storage. */
+  uint32_t          dec_adler32;      /* Optional checksum: value. */
+
+  , dec_cpylen :: usize_t            -- ^  length of copy window (VCD_SOURCE or VCD_TARGET)
+  , dec_cpyoff :: xoff_t             -- ^  offset of copy window (VCD_SOURCE or VCD_TARGET)
+  , dec_enclen :: usize_t            -- ^  length of delta encoding
+  , dec_tgtlen :: usize_t            -- ^  length of target window
+
+#if USE_UINT64
+  uint64_t          dec_64part;       /* part of a decoded uint64_t */
+#endif
+#if USE_UINT32
+  uint32_t          dec_32part;       /* part of a decoded uint32_t */
+#endif
+
+  , dec_winstart :: xoff_t            -- ^  offset of the start of current target window
+  , dec_window_count :: xoff_t            -- ^  == current_window + 1 in DEC_FINISH
+  , dec_winbytes :: usize_t            -- ^  bytes of the three sections so far consumed
+  , dec_hdrsize :: usize_t            -- ^  VCDIFF + app header size
+
+  const uint8_t    *dec_tgtaddrbase;  /* Base of decoded target addresses (addr >= dec_cpylen). */
+  const uint8_t    *dec_cpyaddrbase;  /* Base of decoded copy addresses (addr < dec_cpylen). */
+
+  , dec_position :: usize_t            -- ^  current decoder position counting the cpylen offset
+  , dec_maxpos :: usize_t            -- ^  maximum decoder position counting the cpylen offset
+  xd3_hinst         dec_current1;     /* current instruction */
+  xd3_hinst         dec_current2;     /* current instruction */
+
+  uint8_t          *dec_buffer;       /* Decode buffer */
+  uint8_t          *dec_lastwin;      /* In case of VCD_TARGET, the last target window. */
+  , dec_lastlen :: usize_t            -- ^  length of the last target window
+  , dec_laststart :: xoff_t            -- ^  offset of the start of last target window
+  , dec_lastspace :: usize_t            -- ^  allocated space of last target window, for reuse
+
+  , inst_sect :: xd3_desect        -- ^  staging area for decoding window sections
+  xd3_desect        addr_sect;
+  xd3_desect        data_sect;
+
+  xd3_code_table_func       *code_table_func;
+  const xd3_dinst           *code_table;
+  const xd3_code_table_desc *code_table_desc;
+  xd3_dinst                 *code_table_alloc;
+
+  /* secondary compression */
+  const xd3_sec_type *sec_type;
+  xd3_sec_stream     *sec_stream_d;
+  xd3_sec_stream     *sec_stream_i;
+  xd3_sec_stream     *sec_stream_a;
+
+  /* state for reconstructing whole files (e.g., for merge), this only
+   * supports loading USIZE_T_MAX instructions, adds, etc. */
+  xd3_whole_state     whole_target;
+
+  /* statistics */
+  xoff_t            n_scpy;
+  xoff_t            n_tcpy;
+  xoff_t            n_add;
+  xoff_t            n_run;
+
+  xoff_t            l_scpy;
+  xoff_t            l_tcpy;
+  xoff_t            l_add;
+  xoff_t            l_run;
+
+  usize_t           i_slots_used;
+
+#if XD3_DEBUG
+  usize_t            large_ckcnt;
+
+  /* memory usage */
+  usize_t            alloc_cnt;
+  usize_t            free_cnt;
+#endif
+    -}
+
+  }
+
+---------------------------------------------------------------------------
+-- PUBLIC FUNCTIONS
+----------------------------------------------------------------------------
+
+-- | This function configures an xd3_stream using the provided in-memory
+-- input buffer, source buffer, output buffer, and flags.  The output
+-- array must be large enough or else ENOSPC will be returned.  This
+-- is the simplest in-memory encoding interface.
+foreign import ccall "xdelta3.h xd3_encode_memory"
+    xd3_encode_memory :: Ptr Word8     -- input
+                        -> Usize_t     -- input_size
+                        -> Ptr Word8   -- source
+                        -> Usize_t     -- source_size
+                        -> Ptr Word8   -- output_buffer
+                        -> Ptr Usize_t -- output_size
+                        -> Usize_t     -- avail_output
+                        -> Int         -- flags
+                        -> IO CInt
+
+-- | The reverse of xd3_encode_memory.
+foreign import ccall "xdelta3.h xd3_decode_memory"
+    xd3_decode_memory :: Ptr Word8   -- input,
+                      -> Usize_t     -- input_size,
+                      -> Ptr Word8   -- source,
+                      -> Usize_t     -- source_size,
+                      -> Ptr Word8   -- output_buf,
+                      -> Ptr Usize_t -- output_size,
+                      -> Usize_t     -- avail_output,
+                      -> Int         -- flags
+                      -> IO CInt
+
+-- | This function encodes an in-memory input using a pre-configured
+-- xd3_stream.  This allows the caller to set a variety of options
+-- which are not available in the xd3_encode/decode_memory()
+-- functions.
+--
+-- The output array must be large enough to hold the output or else
+-- ENOSPC is returned.  The source (if any) should be set using
+-- xd3_set_source_and_size() with a single-block xd3_source.  This
+-- calls the underlying non-blocking interfaces,
+-- xd3_encode/decode_input(), handling the necessary input/output
+-- states.  This method may be considered a reference for any
+-- application using xd3_encode_input() directly.
+--
+--   xd3_stream stream;
+--   xd3_config config;
+--   xd3_source src;
+--
+--   memset (& src, 0, sizeof (src));
+--   memset (& stream, 0, sizeof (stream));
+--   memset (& config, 0, sizeof (config));
+--
+--   if (source != NULL)
+--     {
+--       src.size = source_size;
+--       src.blksize = source_size;
+--       src.curblkno = 0;
+--       src.onblk = source_size;
+--       src.curblk = source;
+--       src.max_winsize = source_size;
+--       xd3_set_source(&stream, &src);
+--     }
+--
+--   config.flags = flags;
+--   config.winsize = input_size;
+--
+--   ... set smatcher, appheader, encoding-table, compression-level, etc.
+--
+--   xd3_config_stream(&stream, &config);
+--   xd3_encode_stream(&stream, ...);
+--   xd3_free_stream(&stream);
+foreign import ccall "xdelta3.h xd3_encode_stream"
+    xd3_encode_stream :: Ptr Xd3_stream -- stream,
+                      -> Ptr Word8      -- input,
+                      -> Usize_t        -- input_size,
+                      -> Ptr Word8      -- output,
+                      -> Ptr Usize_t    -- output_size,
+                      -> Usize_t        -- avail_output);
+                      -> IO Int
+
+-- | The reverse of xd3_encode_stream.
+foreign import ccall "xdelta3.h xd3_decode_stream"
+    xd3_decode_stream :: Ptr Xd3_stream -- stream,
+                      -> Ptr Word8      -- input,
+                      -> Usize_t        -- input_size,
+                      -> Ptr Word8      -- output,
+                      -> Ptr Usize_t    -- output_size,
+                      -> Usize_t        -- avail_size);
+                      -> IO Int
+
+-- | This is the non-blocking interface.
+--
+-- Handling input and output states is the same for encoding or
+-- decoding using the xd3_avail_input() and xd3_consume_output()
+-- routines, inlined below.
+--
+-- Return values:
+--
+--   XD3_INPUT: the process requires more input: call
+--               xd3_avail_input() then repeat
+--
+--   XD3_OUTPUT: the process has more output: read stream->next_out,
+--               stream->avail_out, then call xd3_consume_output(),
+--               then repeat
+--
+--   XD3_GOTHEADER: (decoder-only) notification returned following the
+--               VCDIFF header and first window header.  the decoder
+--               may use the header to configure itself.
+--
+--   XD3_WINSTART: a general notification returned once for each
+--               window except the 0-th window, which is implied by
+--               XD3_GOTHEADER.  It is recommended to use a
+--               switch-stmt such as:
+--
+--                 ...
+--               again:
+--                 switch ((ret = xd3_decode_input (stream))) {
+--                    case XD3_GOTHEADER: {
+--                      assert(stream->current_window == 0);
+--                      stuff;
+--                    }
+--                    // fallthrough
+--                    case XD3_WINSTART: {
+--                      something(stream->current_window);
+--                      goto again;
+--                    }
+--                    ...
+--
+--   XD3_WINFINISH: a general notification, following the complete
+--               input & output of a window.  at this point,
+--               stream->total_in and stream->total_out are consistent
+--               for either encoding or decoding.
+--
+--   XD3_GETSRCBLK: If the xd3_getblk() callback is NULL, this value
+--               is returned to initiate a non-blocking source read.
+foreign import ccall "xdelta3.h xd3_decode_input" xd3_decode_input  :: Ptr Xd3_stream -> IO Xd3_rvalues
+foreign import ccall "xdelta3.h xd3_encode_input" xd3_encode_input  :: Ptr Xd3_stream -> IO Xd3_rvalues
+
+-- | The xd3_config structure is used to initialize a stream - all data
+-- is copied into stream so config may be a temporary variable.  See
+-- the [documentation] or comments on the xd3_config structure.
+foreign import ccall "xdelta3.h xd3_config_stream" xd3_config_stream :: Ptr Xd3_stream -> Ptr xd3_config -> IO Xd3_rvalues
+
+-- | Since Xdelta3 doesn't open any files, xd3_close_stream is just an
+-- error check that the stream is in a proper state to be closed: this
+-- means the encoder is flushed and the decoder is at a window
+-- boundary.  The application is responsible for freeing any of the
+-- resources it supplied.
+foreign import ccall "xdelta3.h xd3_close_stream" xd3_close_stream :: Ptr Xd3_stream -> IO Xd3_rvalues
+
+-- | This arranges for closes the stream to succeed.  Does not free the
+-- stream.
+foreign import ccall "xdelta3.h xd3_abort_stream" xd3_abort_stream :: Ptr Xd3_stream  -> IO ()
+
+-- | xd3_free_stream frees all memory allocated for the stream.  The
+-- application is responsible for freeing any of the resources it
+-- supplied.
+foreign import ccall "xdelta3.h xd3_free_stream" xd3_free_stream :: Ptr Xd3_stream -> IO ()
+
+-- | This function informs the encoder or decoder that source matching
+-- (i.e., delta-compression) is possible.  For encoding, this should
+-- be called before the first xd3_encode_input.  A NULL source is
+-- ignored.  For decoding, this should be called before the first
+-- window is decoded, but the appheader may be read first
+-- (XD3_GOTHEADER).  After decoding the header, call xd3_set_source()
+-- if you have a source file.  Note: if (stream->dec_win_ind & VCD_SOURCE)
+-- is true, it means the first window expects there to be a source file.
+foreign import ccall "xdelta3.h xd3_set_source" xd3_set_source :: Ptr Xd3_stream -> Ptr Xd3_source -> IO CInt
+
+-- | If the source size is known, call this instead of xd3_set_source().
+-- to avoid having stream->getblk called (and/or to avoid XD3_GETSRCBLK).
+--
+-- Follow these steps:
+--
+-- >   xd3_source source;
+-- >   memset(&source, 0, sizeof(source));
+-- >   source.blksize  = size;
+-- >   source.onblk    = size;
+-- >   source.curblk   = buf;
+-- >   source.curblkno = 0;
+-- >   int ret = xd3_set_source_and_size(&stream, &source, size);
+-- >   ...
+foreign import ccall "xdelta3.h xd3_set_source_and_size" xd3_set_source_and_size :: Ptr Xd3_stream -> Ptr Xd3_source -> Xoff_t -> IO CInt
+
+-- | This should be called before the first call to xd3_encode_input()
+-- to include application-specific data in the VCDIFF header. */
+foreign import ccall "xdelta3.h xd3_set_appheader" xd3_set_appheader :: Ptr Xd3_stream  -> Ptr Word8 -> Usize_t -> IO ()
+
+-- xd3_get_appheader may be called in the decoder after XD3_GOTHEADER.
+-- For convenience, the decoder always adds a single byte padding to
+-- the end of the application header, which is set to zero in case the
+-- application header is a string.
+foreign import ccall "xdelta3.h xd3_get_appheader" xd3_get_appheader :: Ptr Xd3_stream -> Ptr (Ptr Word8) -> Ptr Usize_t -> IO Xd3_rvalues
+
+-- | To generate a VCDIFF encoded delta with xd3_encode_init() from
+-- another format, use:
+--
+--   xd3_encode_init_partial() -- initialze encoder state (w/o hash tables)
+--   xd3_init_cache() -- reset VCDIFF address cache
+--   xd3_found_match() -- to report a copy instruction
+--
+-- set stream->enc_state to ENC_INSTR and call xd3_encode_input as usual.
+foreign import ccall "xdelta3.h xd3_encode_init_partial" xd3_encode_init_partial :: Ptr Xd3_stream -> IO Xd3_rvalues
+foreign import ccall "xdelta3.h xd3_init_cache" xd3_init_cache :: Ptr Xd3_addr_cache -> IO ()
+foreign import ccall "xdelta3.h xd3_found_match" xd3_found_match :: Ptr Xd3_stream ->  Usize_t -> Usize_t -> Xoff_t -> CInt -> IO Xd3_rvalues
+
+
+-- | Gives an error string for xdelta3-speficic errors, returns NULL for
+-- system errors
+foreign import ccall "xdelta3.h xd3_strerror" xd3_strerror :: Xd3_rvalues -> Ptr CChar
+
+
+-- For convenience, zero & initialize the xd3_config structure with
+-- specified flags. */
+foreign import ccall "xdelta3.h xd3_init_config" xd3_init_config :: Ptr Xd3_config -> Word32 -> IO ()
+
+-- | This supplies some input to the stream.
+--
+-- For encoding, if the input is larger than the configured window
+-- size (xd3_config.winsize), the entire input will be consumed and
+-- encoded anyway.  If you wish to strictly limit the window size,
+-- limit the buffer passed to xd3_avail_input to the window size.
+--
+-- For encoding, if the input is smaller than the configured window
+-- size (xd3_config.winsize), the library will create a window-sized
+-- buffer and accumulate input until a full-sized window can be
+-- encoded.  XD3_INPUT will be returned.  The input must remain valid
+-- until the next time xd3_encode_input() returns XD3_INPUT.
+--
+-- For decoding, the input will be consumed entirely before XD3_INPUT
+-- is returned again.
+foreign import ccall "xdelta3.h xd3_avail_input" xd3_avail_input :: Ptr Xd3_stream -> Ptr Word8 -> Usize_t -> IO ()
+
+-- | This acknowledges receipt of output data, must be called after any
+-- XD3_OUTPUT return.
+foreign import ccall "xdelta3.h xd3_consume_output" xd3_consume_output :: Ptr Xd3_stream -> IO ()
+
+-- | These are set for each XD3_WINFINISH return.
+foreign import ccall "xdelta3.h xd3_encoder_used_source" xd3_encoder_used_source :: Ptr Xd3_stream -> IO Bool
+foreign import ccall "xdelta3.h xd3_encoder_srcbase" xd3_encoder_srcbase :: Ptr xd3_stream -> IO Xoff_t
+foreign import ccall "xdelta3.h xd3_encoder_srclen" xd3_encoder_srclen :: Ptr Xd3_stream -> IO Usize_t
+
+
+-- | Checks for legal flag changes.
+foreign import ccall "xdelta3.h xd3_set_flags" xd3_set_flags :: Ptr Xd3_stream -> Xd3_flags -> IO ()
+
+-- | Gives some extra information about the latest library error, if any
+-- is known.
+foreign import ccall "xdelta3.h xd3_errstring" xd3_errstring :: Ptr Xd3_stream -> CString
+
+-- | 64-bit divisions are expensive, which is why we require a
+-- power-of-two source->blksize.  To relax this restriction is
+-- relatively easy, see the history for xd3_blksize_div(). */
+foreign import ccall "xdelta3.h xd3_blksize_div" xd3_blksize_div :: Xoff_t -> Ptr Xd3_source -> Ptr Xoff_t -> Usize_t -> IO ()
+
+foreign import ccall "xdelta3.h xd3_blksize_add" xd3_blksize_add :: Ptr Xoff_t -> Ptr Usize_t -> Ptr Xd3_source -> Usize_t -> IO ()
+
+pattern XD3_NOOP = #const XD3_NOOP
+pattern XD3_ADD  = #const XD3_ADD
+pattern XD3_RUN  = #const XD3_RUN
+pattern XD3_CPY  = #const XD3_CPY