path: root/haskell/Xdelta3.hsc

{-# LANGUAGE BangPatterns    #-}
{-# LANGUAGE PatternSynonyms #-}
module Xdelta3 where

import Data.Word
import Foreign.C.Types
import Foreign.C.String
import Foreign.Marshal.Alloc
import Foreign.Ptr

-- #define HAVE_CONFIG_H 1

#define SIZEOF_SIZE_T __SIZEOF_SIZE_T__
#define SIZEOF_UNSIGNED_INT __SIZEOF_INT__
#define SIZEOF_UNSIGNED_LONG __SIZEOF_LONG__
#define SIZEOF_UNSIGNED_LONG_LONG __SIZEOF_LONG_LONG__
#define static_assert(...)

#include <xdelta3.h>


-- To learn more about Xdelta, start by reading xdelta3.c.  If you are
-- ready to use the API, continue reading here.  There are two
-- interfaces -- xd3_encode_input and xd3_decode_input -- plus a dozen
-- or so related calls.  This interface is styled after Zlib.


-- | Default configured value of stream->winsize.  If the program
-- supplies xd3_encode_input() with data smaller than winsize the
-- stream will automatically buffer the input, otherwise the input
-- buffer is used directly.
pattern XD3_DEFAULT_WINSIZE = #const XD3_DEFAULT_WINSIZE

-- | Default total size of the source window used in xdelta3-main.h
pattern XD3_DEFAULT_SRCWINSZ = #const XD3_DEFAULT_SRCWINSZ

-- When Xdelta requests a memory allocation for certain buffers, it
-- rounds up to units of at least this size.  The code assumes (and
-- asserts) that this is a power-of-two.
pattern XD3_ALLOCSIZE = #const XD3_ALLOCSIZE


-- The XD3_HARDMAXWINSIZE parameter is a safety mechanism to protect
-- decoders against malicious files.  The decoder will never decode a
-- window larger than this.  If the file specifies VCD_TARGET the
-- decoder may require two buffers of this size.
--
-- 8-16MB is reasonable, probably don't need to go larger.
pattern XD3_HARDMAXWINSIZE = #const XD3_HARDMAXWINSIZE


-- The IOPT_SIZE value sets the size of a buffer used to batch
-- overlapping copy instructions before they are optimized by picking
-- the best non-overlapping ranges.  The larger this buffer, the
-- longer a forced xd3_srcwin_setup() decision is held off.  Setting
-- this value to 0 causes an unlimited buffer to be used. */
pattern XD3_DEFAULT_IOPT_SIZE = #const XD3_DEFAULT_IOPT_SIZE

-- The maximum distance backward to search for small matches */
pattern XD3_DEFAULT_SPREVSZ = #const XD3_DEFAULT_SPREVSZ

-- The default compression level */
pattern XD3_DEFAULT_LEVEL = #const XD3_DEFAULT_LEVEL

pattern XD3_DEFAULT_SECONDARY_LEVEL = #const XD3_DEFAULT_SECONDARY_LEVEL

pattern XD3_USE_LARGEFILE64 = #const XD3_USE_LARGEFILE64

-- The source window size is limited to 2GB unless
-- XD3_USE_LARGESIZET is defined to 1. */
pattern XD3_USE_LARGESIZET = #const XD3_USE_LARGESIZET


-- Sizes and addresses within VCDIFF windows are represented as usize_t
--
-- For source-file offsets and total file sizes, total input and
-- output counts, the xoff_t type is used.  The decoder and encoder
-- generally check for overflow of the xoff_t size (this is tested at
-- the 32bit boundary [xdelta3-test.h]).

-- Settings based on the size of xoff_t (32 vs 64 file offsets) */
#if XD3_USE_LARGEFILE64
-- xoff_t is a 64-bit type */
pattern C__USE_FILE_OFFSET64 = #const __USE_FILE_OFFSET64

pattern C_FILE_OFFSET_BITS = #const _FILE_OFFSET_BITS
#endif

#if SIZEOF_XOFF_T == 8
type Xoff_t = Word64
pattern Q = #const_str Q
#elif SIZEOF_XOFF_T == 4
type Xoff_t = Word32
pattern Q = ""
#endif

pattern SIZEOF_XOFF_T = #const SIZEOF_XOFF_T

#if SIZEOF_USIZE_T == 8
type Usize_t = Word64
pattern W = #const_str W
#elif SIZEOF_USIZE_T == 4
type Usize_t = Word32
pattern W = ""
#endif

pattern SIZEOF_USIZE_T = #const SIZEOF_USIZE_T

pattern SIZEOF_SIZE_T = #const SIZEOF_SIZE_T
pattern Z = #const_str Z

pattern USE_UINT32 = #const USE_UINT32
pattern USE_UINT64 = #const USE_UINT64

pattern UNALIGNED_OK = #const UNALIGNED_OK

pattern XD3_ENCODER = #const XD3_ENCODER

-- The code returned when main() fails, also defined in system
-- includes.
pattern EXIT_FAILURE = #const EXIT_FAILURE

-- REGRESSION TEST enables the "xdelta3 test" command, which runs a
-- series of self-tests.
pattern REGRESSION_TEST = #const REGRESSION_TEST

pattern PYTHON_MODULE = #const PYTHON_MODULE

pattern SWIG_MODULE = #const SWIG_MODULE

pattern NOT_MAIN = #const NOT_MAIN

-- There are three string matching functions supplied: one fast, one
-- slow (default), and one soft-configurable.  To disable any of
-- these, use the following definitions.
pattern XD3_BUILD_SLOW = #const XD3_BUILD_SLOW
pattern XD3_BUILD_FAST = #const XD3_BUILD_FAST
pattern XD3_BUILD_FASTER = #const XD3_BUILD_FASTER
pattern XD3_BUILD_FASTEST = #const XD3_BUILD_FASTEST
pattern XD3_BUILD_SOFT = #const XD3_BUILD_SOFT
pattern XD3_BUILD_DEFAULT = #const XD3_BUILD_DEFAULT


-- newtype Xd3_stream             = Xd3_stream           ( Ptr Xd3_stream           )
-- newtype Xd3_source             = Xd3_source           ( Ptr Xd3_source           )
newtype Xd3_hash_cfg           = Xd3_hash_cfg         ( Ptr Xd3_hash_cfg         )
newtype Xd3_smatcher           = Xd3_smatcher         ( Ptr Xd3_smatcher         )
newtype Xd3_rinst              = Xd3_rinst            ( Ptr Xd3_rinst            )
newtype Xd3_dinst              = Xd3_dinst            ( Ptr Xd3_dinst            )
newtype Xd3_hinst              = Xd3_hinst            ( Ptr Xd3_hinst            )
newtype Xd3_winst              = Xd3_winst            ( Ptr Xd3_winst            )
newtype Xd3_rpage              = Xd3_rpage            ( Ptr Xd3_rpage            )
newtype Xd3_addr_cache         = Xd3_addr_cache       ( Ptr Xd3_addr_cache       )
newtype Xd3_output             = Xd3_output           ( Ptr Xd3_output           )
newtype Xd3_desect             = Xd3_desect           ( Ptr Xd3_desect           )
newtype Xd3_iopt_buflist       = Xd3_iopt_buflist     ( Ptr Xd3_iopt_buflist     )
newtype Xd3_rlist              = Xd3_rlist            ( Ptr Xd3_rlist            )
newtype Xd3_sec_type           = Xd3_sec_type         ( Ptr Xd3_sec_type         )
-- newtype Xd3_sec_cfg            = Xd3_sec_cfg          ( Ptr Xd3_sec_cfg          )
newtype Xd3_sec_stream         = Xd3_sec_stream       ( Ptr Xd3_sec_stream       )
-- newtype Xd3_config             = Xd3_config           ( Ptr Xd3_config           )
newtype Xd3_code_table_desc    = Xd3_code_table_desc  ( Ptr Xd3_code_table_desc  )
newtype Xd3_code_table_sizes   = Xd3_code_table_sizes ( Ptr Xd3_code_table_sizes )
newtype Xd3_slist              = Xd3_slist            ( Ptr Xd3_slist            )
newtype Xd3_whole_state        = Xd3_whole_state      ( Ptr Xd3_whole_state      )
newtype Xd3_wininfo            = Xd3_wininfo          ( Ptr Xd3_wininfo          )



-- The stream configuration has three callbacks functions, all of
-- which may be supplied with NULL values.  If config->getblk is
-- provided as NULL, the stream returns XD3_GETSRCBLK. */

-- typedef void* (xd3_alloc_func) (void       *opaque,
--                                 size_t      items,
--                                 usize_t     size);
foreign export ccall hs_alloc :: Ptr ()
                              -> #type size_t
                              -> #type usize_t
                              -> IO ( Ptr                   ( ))
hs_alloc _ items size = mallocBytes (fromIntegral items * fromIntegral size)

-- typedef void  (xd3_free_func) (void *opaque,
--                                void *address);
foreign export ccall hs_free :: Ptr () -> Ptr () -> IO ()
hs_free _ address = free address

{-
/* third callback */
typedef int    (xd3_getblk_func)   (xd3_stream *stream,
				    xd3_source *source,
				    xoff_t      blkno);

typedef const xd3_dinst* (xd3_code_table_func) (void);
-
-}

-- Type used for short snprintf calls. */
-- typedef struct { char buf[48]; } shortbuf;
data Shortbuf = Shortbuf {-# UNPACK #-} !Word64
                         {-# UNPACK #-} !Word64
                         {-# UNPACK #-} !Word64
                         {-# UNPACK #-} !Word64
                         {-# UNPACK #-} !Word64
                         {-# UNPACK #-} !Word64



-- XPR(NT "", ...) (used by main) prefixes an "xdelta3: " to the output. */
-- void xprintf(const char *fmt, ...) PRINTF_ATTRIBUTE(1,2);
-- foreign import ccall "xdelta3.h xprintf" xprintf :: ?

pattern NT = #const_str NT
pattern NTR = #const_str NTR
pattern RINT = #const_str RINT


-----------------------------------------------------------------
-- PUBLIC ENUMS
--------------------------------------------------------------------

-- | These are the five ordinary status codes returned by the
-- xd3_encode_input() and xd3_decode_input() state machines.
--
-- An application must be prepared to handle these five return
-- values from either xd3_encode_input or xd3_decode_input except
-- in the case of no-source compression in which case XD3_GETSRCBLK
-- is never returned.  More detailed comments for these are given in
-- xd3_encode_input and xd3_decode_input comments below.
newtype Xd3_rvalues = Xd3_rvalues CInt

-- | need input
pattern XD3_INPUT         = Xd3_rvalues (#const XD3_INPUT)

-- | have output
pattern XD3_OUTPUT        = Xd3_rvalues (#const XD3_OUTPUT)

-- | need a block of source input (with no xd3_getblk function) a chance to do non-blocking read.
pattern XD3_GETSRCBLK     = Xd3_rvalues (#const XD3_GETSRCBLK)

-- | (decode-only) after the initial VCDIFF & first window header
pattern XD3_GOTHEADER     = Xd3_rvalues (#const XD3_GOTHEADER)

-- | notification: returned before a window is processed giving a chance to XD3_SKIP_WINDOW or not XD3_SKIP_EMIT that window.
pattern XD3_WINSTART      = Xd3_rvalues (#const XD3_WINSTART)

-- | notification: returned after encode/decode & output for a window
pattern XD3_WINFINISH     = Xd3_rvalues (#const XD3_WINFINISH)

-- | (encoder only) may be returned by getblk() if the block is too old
pattern XD3_TOOFARBACK    = Xd3_rvalues (#const XD3_TOOFARBACK)

-- | internal error
pattern XD3_INTERNAL      = Xd3_rvalues (#const XD3_INTERNAL)

-- | invalid config
pattern XD3_INVALID       = Xd3_rvalues (#const XD3_INVALID)

-- | invalid input/decoder error
pattern XD3_INVALID_INPUT = Xd3_rvalues (#const XD3_INVALID_INPUT)

-- | when secondary compression finds no improvement.
pattern XD3_NOSECOND      = Xd3_rvalues (#const XD3_NOSECOND)

-- | currently VCD_TARGET VCD_CODETABLE
pattern XD3_UNIMPLEMENTED = Xd3_rvalues (#const XD3_UNIMPLEMENTED)




-- | special values in config->flags
newtype Xd3_flags = Xd3_flags Word32

-- used by VCDIFF tools, see xdelta3-main.h.--/
pattern XD3_JUST_HDR       = Xd3_flags (#const XD3_JUST_HDR)

-- used by VCDIFF tools see xdelta3-main.h.--/
pattern XD3_SKIP_WINDOW    = Xd3_flags (#const XD3_SKIP_WINDOW)

-- | used by VCDIFF tools, see xdelta3-main.h. */
pattern XD3_SKIP_EMIT      = Xd3_flags (#const XD3_SKIP_EMIT)
-- | flush the stream buffer to prepare for xd3_stream_close(). */
pattern XD3_FLUSH          = Xd3_flags (#const XD3_FLUSH)

-- | use DJW static huffman */
pattern XD3_SEC_DJW        = Xd3_flags (#const XD3_SEC_DJW)
-- | use FGK adaptive huffman */
pattern XD3_SEC_FGK        = Xd3_flags (#const XD3_SEC_FGK)
-- | use LZMA secondary */
pattern XD3_SEC_LZMA       = Xd3_flags (#const XD3_SEC_LZMA)

pattern XD3_SEC_TYPE       = Xd3_flags (#const XD3_SEC_TYPE)

-- | disable secondary compression of the data section. */
pattern XD3_SEC_NODATA     = Xd3_flags (#const XD3_SEC_NODATA)
-- | disable secondary compression of the inst section. */
pattern XD3_SEC_NOINST     = Xd3_flags (#const XD3_SEC_NOINST)
-- | disable secondary compression of the addr section. */
pattern XD3_SEC_NOADDR     = Xd3_flags (#const XD3_SEC_NOADDR)

pattern XD3_SEC_NOALL      = Xd3_flags (#const XD3_SEC_NOALL)

-- | enable checksum computation in the encoder. */
pattern XD3_ADLER32        = Xd3_flags (#const XD3_ADLER32)
-- | disable checksum verification in the decoder. */
pattern XD3_ADLER32_NOVER  = Xd3_flags (#const XD3_ADLER32_NOVER)

-- | disable ordinary data * compression feature, only search * the source, not the target. */
pattern XD3_NOCOMPRESS     = Xd3_flags (#const XD3_NOCOMPRESS)
-- | disable the "1.5-pass * algorithm", instead use greedy * matching.  Greedy is off by * default. */
pattern XD3_BEGREEDY       = Xd3_flags (#const XD3_BEGREEDY)
-- | used by "recode". */
pattern XD3_ADLER32_RECODE = Xd3_flags (#const XD3_ADLER32_RECODE)

-- 4 bits to set the compression level the same as the command-line
-- setting -1 through -9 Xd3_flags (-0 corresponds to the XD3_NOCOMPRESS flag
-- and is independent of compression level).  This is for
-- convenience especially with xd3_encode_memoryXd3_flags (). */

pattern XD3_COMPLEVEL_SHIFT = Xd3_flags (#const XD3_COMPLEVEL_SHIFT)
pattern XD3_COMPLEVEL_MASK  = Xd3_flags (#const XD3_COMPLEVEL_MASK)
pattern XD3_COMPLEVEL_1     = Xd3_flags (#const XD3_COMPLEVEL_1)
pattern XD3_COMPLEVEL_2     = Xd3_flags (#const XD3_COMPLEVEL_2)
pattern XD3_COMPLEVEL_3     = Xd3_flags (#const XD3_COMPLEVEL_3)
pattern XD3_COMPLEVEL_6     = Xd3_flags (#const XD3_COMPLEVEL_6)
pattern XD3_COMPLEVEL_9     = Xd3_flags (#const XD3_COMPLEVEL_9)

-- | The values of this enumeration are set in xd3_config using the
-- smatch_cfg variable.  It can be set to default, slow, fast, etc.,
-- and soft.
data Xd3_smatch_cfg
    = XD3_SMATCH_DEFAULT -- ^ Flags may contain XD3_COMPLEVEL bits, else default.
    | XD3_SMATCH_SLOW
    | XD3_SMATCH_FAST
    | XD3_SMATCH_FASTER
    | XD3_SMATCH_FASTEST
    | XD3_SMATCH_SOFT
 deriving Enum

#if 0

----------------------------------------------------------------------
-- PRIVATE ENUMS
----------------------------------------------------------------------

-- | stream->match_state is part of the xd3_encode_input state machine
--  for source matching:
--
--  1. the XD3_GETSRCBLK block-read mechanism means reentrant matching
--  2. this state spans encoder windows: a match and end-of-window
--  will continue in the next 3. the initial target byte and source
--  byte are a presumed match, to avoid some computation in case the
--  inputs are identical.
data Xd3_match_state
    = MATCH_TARGET    -- ^ in this state, attempt to match the start of the
                      -- target with the previously set source address (initially
                      -- 0).
    | MATCH_BACKWARD  -- ^ currently expanding a match backward in the
                      -- source/target.
    | MATCH_FORWARD   -- ^ currently expanding a match forward in the
                      -- source/target.
    | MATCH_SEARCHING -- ^ currently searching for a match.
 deriving Enum


-- The xd3_encode_input state machine steps through these states in
-- the following order.  The matcher is reentrant and returns
-- XD3_INPUT whenever it requires more data.  After receiving
-- XD3_INPUT, if the application reads EOF it should call
-- xd3_stream_close().
data Xd3_encode_input
    = ENC_INIT    -- ^ xd3_encode_input has never been called.
    | ENC_INPUT   -- ^ waiting for xd3_avail_input () to be called.
    | ENC_SEARCH  -- ^ currently searching for matches.
    | ENC_INSTR   -- ^ currently formatting output.
    | ENC_FLUSH   -- ^ currently emitting output.
    | ENC_POSTOUT -- ^ after an output section.
    | ENC_POSTWIN -- ^ after all output sections.
    | ENC_ABORTED -- ^ abort.
 deriving Enum

-- | The xd3_decode_input state machine steps through these states in
-- the following order.  The matcher is reentrant and returns
-- XD3_INPUT whenever it requires more data.  After receiving
-- XD3_INPUT, if the application reads EOF it should call
-- xd3_stream_close().
--
-- 0-8:   the VCDIFF header
-- 9-18:  the VCDIFF window header
-- 19-21: the three primary sections: data, inst, addr
-- 22:    producing output: returns XD3_OUTPUT, possibly XD3_GETSRCBLK,
-- 23:    return XD3_WINFINISH, set state=9 to decode more input
data Xd3_decode_state
  = DEC_VCHEAD -- ^ VCDIFF header
  | DEC_HDRIND -- ^ header indicator

  | DEC_SECONDID -- ^ secondary compressor ID

  | DEC_TABLEN -- ^ code table length
  | DEC_NEAR   -- ^ code table near
  | DEC_SAME   -- ^ code table same
  | DEC_TABDAT -- ^ code table data

  | DEC_APPLEN -- ^ application data length
  | DEC_APPDAT -- ^ application data

  | DEC_WININD -- ^ window indicator

  | DEC_CPYLEN -- ^ copy window length
  | DEC_CPYOFF -- ^ copy window offset

  | DEC_ENCLEN -- ^ length of delta encoding
  | DEC_TGTLEN -- ^ length of target window
  | DEC_DELIND -- ^ delta indicator

  | DEC_DATALEN -- ^ length of ADD+RUN data
  | DEC_INSTLEN -- ^ length of instruction data
  | DEC_ADDRLEN -- ^ length of address data

  | DEC_CKSUM -- ^ window checksum

  | DEC_DATA -- ^ data section
  | DEC_INST -- ^ instruction section
  | DEC_ADDR -- ^ address section

  | DEC_EMIT -- ^ producing data

  | DEC_FINISH -- ^ window finished

  | DEC_ABORTED -- ^ xd3_abort_stream
  deriving Enum

-------------------------------------------------------------
-- internal types
--------------------------------------------------------------

-- skipped.

#endif

---------------------------------------------------------------------
-- public types
---------------------------------------------------------------------

-- | Settings for the secondary compressor.
data Xd3_sec_cfg = Xd3_sec_cfg
  { data_type   :: Int     -- ^  Which section. (set automatically)
  , ngroups     :: Usize_t -- ^  Number of DJW Huffman groups.
  , sector_size :: Usize_t -- ^  Sector size.
  , inefficient :: Int     -- ^  If true, ignore efficiency check [avoid XD3_NOSECOND].
  }

-- | This is the user-visible stream configuration. */
data Xd3_config = Xd3_config
  { winsize   :: Usize_t -- ^  The encoder window size.
  , sprevsz   :: Usize_t -- ^  How far back small string matching goes
  , iopt_size :: Usize_t -- ^  entries in the instruction-optimizing buffer

   -- xd3_getblk_func   *getblk;        /* The three callbacks. */
   -- xd3_alloc_func    *alloc;
   -- xd3_free_func     *freef;
   -- void              *opaque;        /* Not used. */

  , flags         :: Xd3_flags      -- ^  stream->flags are initialized from xd3_config & never modified by the library.  Use xd3_set_flags to modify flags settings mid-stream.
  , sec_data      :: Xd3_sec_cfg    -- ^  Secondary compressor config: data
  , sec_inst      :: Xd3_sec_cfg    -- ^  Secondary compressor config: inst
  , sec_addr      :: Xd3_sec_cfg    -- ^  Secondary compressor config: addr
  , smatch_cfg    :: Xd3_smatch_cfg -- ^  See enum: use fields below  for soft config
  , smatcher_soft :: Xd3_smatcher
  }

-- | The primary source file object. You create one of these objects and
-- initialize the first four fields.  This library maintains the next
-- 5 fields.  The configured getblk implementation is responsible for
-- setting the final 3 fields when called (and/or when XD3_GETSRCBLK
-- is returned).
data Xd3_source = Xd3_source
 {
  -- you set
    blksize     :: Usize_t -- ^  block size
  , name        :: String  -- ^ its name, for debug/print purposes
  , ioh         :: Ptr ()  -- ^ opaque handle */
  , max_winsize :: Xoff_t  -- ^  maximum visible buffer

  -- getblk sets
  , curblkno :: Xoff_t    -- ^  current block number: client sets after getblk request
  , onblk    :: Usize_t   -- ^  number of bytes on current block: client sets,  must be >= 0 and <= blksize
  , curblk   :: Ptr Word8 -- ^ current block array: client sets after getblk request */

  -- xd3 sets
  , srclen        :: Usize_t -- ^  length of this source window
  , srcbase       :: Xoff_t  -- ^  offset of this source window in the source itself
  , shiftby       :: Usize_t -- ^  for power-of-two blocksizes
  , maskby        :: Usize_t -- ^  for power-of-two blocksizes
  , cpyoff_blocks :: Xoff_t  -- ^  offset of dec_cpyoff in blocks
  , cpyoff_blkoff :: Usize_t -- ^  offset of copy window in blocks, remainder
  , getblkno      :: Xoff_t  -- ^  request block number: xd3 sets current getblk request

  -- See xd3_getblk()
  , max_blkno :: Xoff_t  -- ^  Maximum block, if eof is known, otherwise, equals frontier_blkno (initially 0).
  , onlastblk :: Usize_t -- ^  Number of bytes on max_blkno
  , eof_known :: Int     -- ^  Set to true when the first partial block is read.
}

-- | The primary xd3_stream object, used for encoding and decoding.  You
-- may access only two fields: avail_out, next_out.  Use the methods
-- above to operate on xd3_stream.

data Xd3_stream = Xd3_stream
  {
  -- input state */
    next_in  :: Ptr Word8 -- ^ next input byte */
  , avail_in :: Usize_t   -- ^  number of bytes available at next_in
  , total_in :: Xoff_t    -- ^  how many bytes in

  -- output state */
  , next_out       :: Ptr Word8 -- ^ next output byte */
  , avail_out      :: Usize_t   -- ^  number of bytes available at next_out
  , space_out      :: Usize_t   -- ^  total out space
  , current_window :: Xoff_t    -- ^  number of windows encoded/decoded
  , total_out      :: Xoff_t    -- ^  how many bytes out

  -- to indicate an error, xd3 sets */
  , msg :: Maybe String -- last error message, NULL if no error */

  -- source configuration */
  , src :: Xd3_source       -- ^ source array */

  -- encoder memory configuration */
  -- , winsize        :: Usize_t -- ^  suggested window size
  -- , sprevsz        :: Usize_t -- ^  small string, previous window size (power of 2)
  , sprevmask      :: Usize_t -- ^  small string, previous window size mask
  -- , iopt_size      :: Usize_t
  , iopt_unlimited :: Usize_t

  -- general configuration */
  -- xd3_getblk_func  *getblk;           /* set nxtblk, nxtblkno to scanblkno */
  -- xd3_alloc_func   *alloc;            /* malloc function */
  -- xd3_free_func    *free;             /* free function */
  , opaque :: Ptr ()   -- ^  private data object passed to alloc, free, and getblk
  -- , flags  :: Uint32_t -- ^  various options

  -- secondary compressor configuration */
  -- , sec_data :: Xd3_sec_cfg -- ^  Secondary compressor config: data
  -- , sec_inst :: Xd3_sec_cfg -- ^  Secondary compressor config: inst
  -- , sec_addr :: Xd3_sec_cfg -- ^  Secondary compressor config: addr

  , smatcher :: Xd3_smatcher

  {-

  usize_t           *large_table;      /* table of large checksums */
  , large_hash :: xd3_hash_cfg       -- ^  large hash config

  usize_t           *small_table;      /* table of small checksums */
  xd3_slist         *small_prev;       /* table of previous offsets, circular linked list */
  , small_reset :: int                -- ^  true if small table should be reset

  , small_hash :: xd3_hash_cfg       -- ^  small hash config
  , acache :: xd3_addr_cache     -- ^  the vcdiff address cache
  , enc_state :: xd3_encode_state   -- ^  state of the encoder

  , taroff :: usize_t            -- ^  base offset of the target input
  , input_position :: usize_t            -- ^  current input position
  , min_match :: usize_t            -- ^  current minimum match length, avoids redundent matches
  , unencoded_offset :: usize_t            -- ^  current input, first * unencoded offset. this value is <= the first instruction's position in the iopt buffer, if there is at least one match in the buffer.

  /* SRCWIN */
  , srcwin_decided :: int                -- ^  boolean: true if srclen and srcbase have been decided.
  , srcwin_decided_early :: int                -- ^  boolean: true if srclen and srcbase were decided early.
  , srcwin_cksum_pos :: xoff_t             -- ^  Source checksum position

  /* MATCH */
  , match_state       :: xd3_match_state    -- ^  encoder match state
  , match_srcpos      :: xoff_t             -- ^  current match source position relative to srcbase
  , match_last_srcpos :: xoff_t             -- ^  previously attempted srcpos, to avoid loops.
  , match_minaddr     :: xoff_t             -- ^  smallest matching address to set window params (reset each window xd3_encode_reset)
  , match_maxaddr     :: xoff_t             -- ^  largest matching address to set window params (reset each window xd3_encode_reset)
  , match_back        :: usize_t            -- ^  match extends back so far
  , match_maxback     :: usize_t            -- ^  match extends back maximum
  , match_fwd         :: usize_t            -- ^  match extends forward so far
  , match_maxfwd      :: usize_t            -- ^  match extends forward maximum

  , maxsrcaddr :: xoff_t             -- ^  address of the last source match (across windows)

  uint8_t          *buf_in;           /* for saving buffered input */
  , buf_avail :: usize_t           -- ^  amount of saved input
  const uint8_t    *buf_leftover;     /* leftover content of next_in (i.e., user's buffer) */
  , buf_leftavail :: usize_t            -- ^  amount of leftover content

  xd3_output       *enc_current;      /* current output buffer */
  xd3_output       *enc_free;         /* free output buffers */
  xd3_output       *enc_heads[4];     /* array of encoded outputs: head of chain */
  xd3_output       *enc_tails[4];     /* array of encoded outputs: tail of chain */
  uint32_t          recode_adler32;   /* set the adler32 checksum * during "recode". */

  , iopt_used :: xd3_rlist         -- ^  instruction optimizing buffer
  xd3_rlist         iopt_free;
  xd3_rinst        *iout;             /* next single instruction */
  xd3_iopt_buflist *iopt_alloc;

  const uint8_t    *enc_appheader;    /* application header to encode */
  usize_t            enc_appheadsz;    /* application header size */

  /* decoder stuff */
  , dec_state :: xd3_decode_state  -- ^  current DEC_XXX value
  , dec_hdr_ind :: usize_t           -- ^  VCDIFF header indicator
  , dec_win_ind :: usize_t           -- ^  VCDIFF window indicator
  , dec_del_ind :: usize_t           -- ^  VCDIFF delta indicator

  uint8_t           dec_magic[4];     /* First four bytes */
  , dec_magicbytes :: usize_t           -- ^  Magic position.

  , dec_secondid :: usize_t           -- ^  Optional secondary compressor ID.

  , dec_codetblsz :: usize_t           -- ^  Optional code table: length.
  uint8_t          *dec_codetbl;      /* Optional code table: storage. */
  , dec_codetblbytes :: usize_t           -- ^  Optional code table: position.

  , dec_appheadsz :: usize_t           -- ^  Optional application header: size.
  uint8_t          *dec_appheader;    /* Optional application header: storage */
  , dec_appheadbytes :: usize_t           -- ^  Optional application header: position.

  , dec_cksumbytes :: usize_t            -- ^  Optional checksum: position.
  uint8_t           dec_cksum[4];     /* Optional checksum: storage. */
  uint32_t          dec_adler32;      /* Optional checksum: value. */

  , dec_cpylen :: usize_t            -- ^  length of copy window (VCD_SOURCE or VCD_TARGET)
  , dec_cpyoff :: xoff_t             -- ^  offset of copy window (VCD_SOURCE or VCD_TARGET)
  , dec_enclen :: usize_t            -- ^  length of delta encoding
  , dec_tgtlen :: usize_t            -- ^  length of target window

#if USE_UINT64
  uint64_t          dec_64part;       /* part of a decoded uint64_t */
#endif
#if USE_UINT32
  uint32_t          dec_32part;       /* part of a decoded uint32_t */
#endif

  , dec_winstart :: xoff_t            -- ^  offset of the start of current target window
  , dec_window_count :: xoff_t            -- ^  == current_window + 1 in DEC_FINISH
  , dec_winbytes :: usize_t            -- ^  bytes of the three sections so far consumed
  , dec_hdrsize :: usize_t            -- ^  VCDIFF + app header size

  const uint8_t    *dec_tgtaddrbase;  /* Base of decoded target addresses (addr >= dec_cpylen). */
  const uint8_t    *dec_cpyaddrbase;  /* Base of decoded copy addresses (addr < dec_cpylen). */

  , dec_position :: usize_t            -- ^  current decoder position counting the cpylen offset
  , dec_maxpos :: usize_t            -- ^  maximum decoder position counting the cpylen offset
  xd3_hinst         dec_current1;     /* current instruction */
  xd3_hinst         dec_current2;     /* current instruction */

  uint8_t          *dec_buffer;       /* Decode buffer */
  uint8_t          *dec_lastwin;      /* In case of VCD_TARGET, the last target window. */
  , dec_lastlen :: usize_t            -- ^  length of the last target window
  , dec_laststart :: xoff_t            -- ^  offset of the start of last target window
  , dec_lastspace :: usize_t            -- ^  allocated space of last target window, for reuse

  , inst_sect :: xd3_desect        -- ^  staging area for decoding window sections
  xd3_desect        addr_sect;
  xd3_desect        data_sect;

  xd3_code_table_func       *code_table_func;
  const xd3_dinst           *code_table;
  const xd3_code_table_desc *code_table_desc;
  xd3_dinst                 *code_table_alloc;

  /* secondary compression */
  const xd3_sec_type *sec_type;
  xd3_sec_stream     *sec_stream_d;
  xd3_sec_stream     *sec_stream_i;
  xd3_sec_stream     *sec_stream_a;

  /* state for reconstructing whole files (e.g., for merge), this only
   * supports loading USIZE_T_MAX instructions, adds, etc. */
  xd3_whole_state     whole_target;

  /* statistics */
  xoff_t            n_scpy;
  xoff_t            n_tcpy;
  xoff_t            n_add;
  xoff_t            n_run;

  xoff_t            l_scpy;
  xoff_t            l_tcpy;
  xoff_t            l_add;
  xoff_t            l_run;

  usize_t           i_slots_used;

#if XD3_DEBUG
  usize_t            large_ckcnt;

  /* memory usage */
  usize_t            alloc_cnt;
  usize_t            free_cnt;
#endif
    -}

  }

---------------------------------------------------------------------------
-- PUBLIC FUNCTIONS
----------------------------------------------------------------------------

-- | This function configures an xd3_stream using the provided in-memory
-- input buffer, source buffer, output buffer, and flags.  The output
-- array must be large enough or else ENOSPC will be returned.  This
-- is the simplest in-memory encoding interface.
foreign import ccall "xdelta3.h xd3_encode_memory"
    xd3_encode_memory :: Ptr Word8     -- input
                        -> Usize_t     -- input_size
                        -> Ptr Word8   -- source
                        -> Usize_t     -- source_size
                        -> Ptr Word8   -- output_buffer
                        -> Ptr Usize_t -- output_size
                        -> Usize_t     -- avail_output
                        -> Int         -- flags
                        -> IO CInt

-- | The reverse of xd3_encode_memory.
foreign import ccall "xdelta3.h xd3_decode_memory"
    xd3_decode_memory :: Ptr Word8   -- input,
                      -> Usize_t     -- input_size,
                      -> Ptr Word8   -- source,
                      -> Usize_t     -- source_size,
                      -> Ptr Word8   -- output_buf,
                      -> Ptr Usize_t -- output_size,
                      -> Usize_t     -- avail_output,
                      -> Int         -- flags
                      -> IO CInt

-- | This function encodes an in-memory input using a pre-configured
-- xd3_stream.  This allows the caller to set a variety of options
-- which are not available in the xd3_encode/decode_memory()
-- functions.
--
-- The output array must be large enough to hold the output or else
-- ENOSPC is returned.  The source (if any) should be set using
-- xd3_set_source_and_size() with a single-block xd3_source.  This
-- calls the underlying non-blocking interfaces,
-- xd3_encode/decode_input(), handling the necessary input/output
-- states.  This method may be considered a reference for any
-- application using xd3_encode_input() directly.
--
--   xd3_stream stream;
--   xd3_config config;
--   xd3_source src;
--
--   memset (& src, 0, sizeof (src));
--   memset (& stream, 0, sizeof (stream));
--   memset (& config, 0, sizeof (config));
--
--   if (source != NULL)
--     {
--       src.size = source_size;
--       src.blksize = source_size;
--       src.curblkno = 0;
--       src.onblk = source_size;
--       src.curblk = source;
--       src.max_winsize = source_size;
--       xd3_set_source(&stream, &src);
--     }
--
--   config.flags = flags;
--   config.winsize = input_size;
--
--   ... set smatcher, appheader, encoding-table, compression-level, etc.
--
--   xd3_config_stream(&stream, &config);
--   xd3_encode_stream(&stream, ...);
--   xd3_free_stream(&stream);
foreign import ccall "xdelta3.h xd3_encode_stream"
    xd3_encode_stream :: Ptr Xd3_stream -- stream,
                      -> Ptr Word8      -- input,
                      -> Usize_t        -- input_size,
                      -> Ptr Word8      -- output,
                      -> Ptr Usize_t    -- output_size,
                      -> Usize_t        -- avail_output);
                      -> IO Int

-- | The reverse of xd3_encode_stream.
foreign import ccall "xdelta3.h xd3_decode_stream"
    xd3_decode_stream :: Ptr Xd3_stream -- stream,
                      -> Ptr Word8      -- input,
                      -> Usize_t        -- input_size,
                      -> Ptr Word8      -- output,
                      -> Ptr Usize_t    -- output_size,
                      -> Usize_t        -- avail_size);
                      -> IO Int

-- | This is the non-blocking interface.
--
-- Handling input and output states is the same for encoding or
-- decoding using the xd3_avail_input() and xd3_consume_output()
-- routines, inlined below.
--
-- Return values:
--
--   XD3_INPUT: the process requires more input: call
--               xd3_avail_input() then repeat
--
--   XD3_OUTPUT: the process has more output: read stream->next_out,
--               stream->avail_out, then call xd3_consume_output(),
--               then repeat
--
--   XD3_GOTHEADER: (decoder-only) notification returned following the
--               VCDIFF header and first window header.  the decoder
--               may use the header to configure itself.
--
--   XD3_WINSTART: a general notification returned once for each
--               window except the 0-th window, which is implied by
--               XD3_GOTHEADER.  It is recommended to use a
--               switch-stmt such as:
--
--                 ...
--               again:
--                 switch ((ret = xd3_decode_input (stream))) {
--                    case XD3_GOTHEADER: {
--                      assert(stream->current_window == 0);
--                      stuff;
--                    }
--                    // fallthrough
--                    case XD3_WINSTART: {
--                      something(stream->current_window);
--                      goto again;
--                    }
--                    ...
--
--   XD3_WINFINISH: a general notification, following the complete
--               input & output of a window.  at this point,
--               stream->total_in and stream->total_out are consistent
--               for either encoding or decoding.
--
--   XD3_GETSRCBLK: If the xd3_getblk() callback is NULL, this value
--               is returned to initiate a non-blocking source read.
foreign import ccall "xdelta3.h xd3_decode_input" xd3_decode_input  :: Ptr Xd3_stream -> IO Xd3_rvalues
foreign import ccall "xdelta3.h xd3_encode_input" xd3_encode_input  :: Ptr Xd3_stream -> IO Xd3_rvalues

-- | The xd3_config structure is used to initialize a stream - all data
-- is copied into stream so config may be a temporary variable.  See
-- the [documentation] or comments on the xd3_config structure.
foreign import ccall "xdelta3.h xd3_config_stream" xd3_config_stream :: Ptr Xd3_stream -> Ptr xd3_config -> IO Xd3_rvalues

-- | Since Xdelta3 doesn't open any files, xd3_close_stream is just an
-- error check that the stream is in a proper state to be closed: this
-- means the encoder is flushed and the decoder is at a window
-- boundary.  The application is responsible for freeing any of the
-- resources it supplied.
foreign import ccall "xdelta3.h xd3_close_stream" xd3_close_stream :: Ptr Xd3_stream -> IO Xd3_rvalues

-- | This arranges for closes the stream to succeed.  Does not free the
-- stream.
foreign import ccall "xdelta3.h xd3_abort_stream" xd3_abort_stream :: Ptr Xd3_stream  -> IO ()

-- | xd3_free_stream frees all memory allocated for the stream.  The
-- application is responsible for freeing any of the resources it
-- supplied.
foreign import ccall "xdelta3.h xd3_free_stream" xd3_free_stream :: Ptr Xd3_stream -> IO ()

-- | This function informs the encoder or decoder that source matching
-- (i.e., delta-compression) is possible.  For encoding, this should
-- be called before the first xd3_encode_input.  A NULL source is
-- ignored.  For decoding, this should be called before the first
-- window is decoded, but the appheader may be read first
-- (XD3_GOTHEADER).  After decoding the header, call xd3_set_source()
-- if you have a source file.  Note: if (stream->dec_win_ind & VCD_SOURCE)
-- is true, it means the first window expects there to be a source file.
foreign import ccall "xdelta3.h xd3_set_source" xd3_set_source :: Ptr Xd3_stream -> Ptr Xd3_source -> IO CInt

-- | If the source size is known, call this instead of xd3_set_source().
-- to avoid having stream->getblk called (and/or to avoid XD3_GETSRCBLK).
--
-- Follow these steps:
--
-- >   xd3_source source;
-- >   memset(&source, 0, sizeof(source));
-- >   source.blksize  = size;
-- >   source.onblk    = size;
-- >   source.curblk   = buf;
-- >   source.curblkno = 0;
-- >   int ret = xd3_set_source_and_size(&stream, &source, size);
-- >   ...
foreign import ccall "xdelta3.h xd3_set_source_and_size" xd3_set_source_and_size :: Ptr Xd3_stream -> Ptr Xd3_source -> Xoff_t -> IO CInt

-- | This should be called before the first call to xd3_encode_input()
-- to include application-specific data in the VCDIFF header. */
foreign import ccall "xdelta3.h xd3_set_appheader" xd3_set_appheader :: Ptr Xd3_stream  -> Ptr Word8 -> Usize_t -> IO ()

-- xd3_get_appheader may be called in the decoder after XD3_GOTHEADER.
-- For convenience, the decoder always adds a single byte padding to
-- the end of the application header, which is set to zero in case the
-- application header is a string.
foreign import ccall "xdelta3.h xd3_get_appheader" xd3_get_appheader :: Ptr Xd3_stream -> Ptr (Ptr Word8) -> Ptr Usize_t -> IO Xd3_rvalues

-- | To generate a VCDIFF encoded delta with xd3_encode_init() from
-- another format, use:
--
--   xd3_encode_init_partial() -- initialze encoder state (w/o hash tables)
--   xd3_init_cache() -- reset VCDIFF address cache
--   xd3_found_match() -- to report a copy instruction
--
-- set stream->enc_state to ENC_INSTR and call xd3_encode_input as usual.
foreign import ccall "xdelta3.h xd3_encode_init_partial" xd3_encode_init_partial :: Ptr Xd3_stream -> IO Xd3_rvalues
foreign import ccall "xdelta3.h xd3_init_cache" xd3_init_cache :: Ptr Xd3_addr_cache -> IO ()
foreign import ccall "xdelta3.h xd3_found_match" xd3_found_match :: Ptr Xd3_stream ->  Usize_t -> Usize_t -> Xoff_t -> CInt -> IO Xd3_rvalues


-- | Gives an error string for xdelta3-speficic errors, returns NULL for
-- system errors
foreign import ccall "xdelta3.h xd3_strerror" xd3_strerror :: Xd3_rvalues -> Ptr CChar


-- For convenience, zero & initialize the xd3_config structure with
-- specified flags. */
foreign import ccall "xdelta3.h xd3_init_config" xd3_init_config :: Ptr Xd3_config -> Word32 -> IO ()

-- | This supplies some input to the stream.
--
-- For encoding, if the input is larger than the configured window
-- size (xd3_config.winsize), the entire input will be consumed and
-- encoded anyway.  If you wish to strictly limit the window size,
-- limit the buffer passed to xd3_avail_input to the window size.
--
-- For encoding, if the input is smaller than the configured window
-- size (xd3_config.winsize), the library will create a window-sized
-- buffer and accumulate input until a full-sized window can be
-- encoded.  XD3_INPUT will be returned.  The input must remain valid
-- until the next time xd3_encode_input() returns XD3_INPUT.
--
-- For decoding, the input will be consumed entirely before XD3_INPUT
-- is returned again.
foreign import ccall "xdelta3.h xd3_avail_input" xd3_avail_input :: Ptr Xd3_stream -> Ptr Word8 -> Usize_t -> IO ()

-- | This acknowledges receipt of output data, must be called after any
-- XD3_OUTPUT return.
foreign import ccall "xdelta3.h xd3_consume_output" xd3_consume_output :: Ptr Xd3_stream -> IO ()

-- | These are set for each XD3_WINFINISH return.
foreign import ccall "xdelta3.h xd3_encoder_used_source" xd3_encoder_used_source :: Ptr Xd3_stream -> IO Bool
foreign import ccall "xdelta3.h xd3_encoder_srcbase" xd3_encoder_srcbase :: Ptr xd3_stream -> IO Xoff_t
foreign import ccall "xdelta3.h xd3_encoder_srclen" xd3_encoder_srclen :: Ptr Xd3_stream -> IO Usize_t


-- | Checks for legal flag changes.
foreign import ccall "xdelta3.h xd3_set_flags" xd3_set_flags :: Ptr Xd3_stream -> Xd3_flags -> IO ()

-- | Gives some extra information about the latest library error, if any
-- is known.
foreign import ccall "xdelta3.h xd3_errstring" xd3_errstring :: Ptr Xd3_stream -> CString

-- | 64-bit divisions are expensive, which is why we require a
-- power-of-two source->blksize.  To relax this restriction is
-- relatively easy, see the history for xd3_blksize_div(). */
foreign import ccall "xdelta3.h xd3_blksize_div" xd3_blksize_div :: Xoff_t -> Ptr Xd3_source -> Ptr Xoff_t -> Usize_t -> IO ()

foreign import ccall "xdelta3.h xd3_blksize_add" xd3_blksize_add :: Ptr Xoff_t -> Ptr Usize_t -> Ptr Xd3_source -> Usize_t -> IO ()

pattern XD3_NOOP = #const XD3_NOOP
pattern XD3_ADD  = #const XD3_ADD
pattern XD3_RUN  = #const XD3_RUN
pattern XD3_CPY  = #const XD3_CPY