path: root/src/Network/Tox/Crypto/Handlers.hs

{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE TupleSections #-}
{-# LANGUAGE TypeOperators #-}
module Network.Tox.Crypto.Handlers where

import Network.Tox.NodeId
import Network.Tox.Crypto.Transport
import Network.Tox.DHT.Transport (Cookie(..),CookieData(..), CookieRequest(..), NoSpam(..))
import Network.Tox.DHT.Handlers (Client, cookieRequest, cookieRequestH )
import Crypto.Tox
import Control.Concurrent.STM
import Network.Address
import qualified Data.Map.Strict as Map
import Crypto.Hash
import Control.Applicative
import Control.Monad
import Data.Time.Clock.POSIX
import qualified Data.ByteString  as B
import Data.ByteString (ByteString)
import Control.Lens
import Data.Function
import qualified Data.PacketQueue as PQ
         ;import Data.PacketQueue (PacketQueue)
import Data.Serialize             as S
import Data.Word
import qualified Data.Word64Map as W64
import qualified Data.Set         as Set
import qualified Data.Array.Unboxed as A
import SensibleDir
import System.FilePath
import System.IO.Temp
import System.Environment
import System.Directory
import System.Random -- for ping fuzz
import Control.Concurrent
import GHC.Conc (labelThread)
import PingMachine

-- util, todo: move to another module
maybeToEither :: Maybe b -> Either String b
maybeToEither (Just x) = Right x
maybeToEither Nothing  = Left "maybeToEither"

data NetCryptoSessionStatus = Unaccepted | Accepted | Confirmed
    deriving (Eq,Ord,Show,Enum)


type IOHook addr x = addr -> x -> IO (Maybe (x -> x))
type NetCryptoHook = IOHook NetCryptoSession CryptoData
type MsgTypeArray = A.UArray Word8 Word16
type MsgOutMap = W64.Word64Map Word8
-- type MsgOutMap = A.UArray Word64 Word8 -- if above is too slow, switch to this, but use reasonable bounds

msgOutMapLookup :: Word64 -> MsgOutMap -> Maybe Word8
msgOutMapLookup = W64.lookup

-- | Information, that may be made visible in multiple sessions, as well
-- as displayed in some way to the user via mutiple views.
--
data SessionView = SessionView
        { svNick        :: TVar ByteString
        , svStatus      :: TVar UserStatus
        , svStatusMsg   :: TVar ByteString
        , svNoSpam      :: TVar (Maybe NoSpam)
        , svGroups      :: TVar (Map.Map GroupChatId (Set.Set SockAddr))

        -- allthough these directories are not visible to others on the net
        -- they are included in this type, because it facilitates organizing
        -- the disk according to your public image.

        , svCacheDir    :: FilePath -- ^ directory path used if the session has
                                    -- to use the disk for cache clean up only
                                    -- if space is needed

        , svTmpDir      :: FilePath -- ^ Once off storage goes here, should
                                    -- clean up quickly

        , svConfigDir   :: FilePath      -- ^ profile related storage, etc, never clean up
        , svDownloadDir :: TVar FilePath -- ^ where to put files the user downloads
        }

type SessionID = Word64

data NetCryptoSession = NCrypto
    { ncState              :: TVar NetCryptoSessionStatus
    , ncSessionId          :: SessionID
    , ncTheirPublicKey     :: PublicKey -- Tox id w/o nospam
    , ncTheirBaseNonce     :: TVar Nonce24 -- base nonce + packet number
    , ncMyPacketNonce      :: TVar Nonce24 -- base nonce + packet number
    , ncHandShake          :: TVar (Maybe (Handshake Encrypted))
    , ncCookie             :: TVar (Maybe Cookie) -- ^ Cookie issued by remote peer
    , ncTheirDHTKey        :: PublicKey
    , ncTheirSessionPublic :: Maybe PublicKey
    , ncSessionSecret      :: SecretKey
    , ncSockAddr           :: SockAddr
    , ncHooks              :: TVar (Map.Map MessageType [NetCryptoHook])
    , ncUnrecognizedHook   :: TVar (MessageType -> NetCryptoHook)
    , ncIncomingTypeArray  :: TVar MsgTypeArray -- ^ supported messages, 0 for unsupported,
                                                -- otherwise the messageType, some message types
                                                -- may not be in ncHooks yet, but they should appear
                                                -- here if ncUnrecognizedHook will add them to ncHooks
                                                -- on an as-need basis.
    , ncOutgoingIdMap      :: TVar MsgOutMap
    , ncAllSessions        :: NetCryptoSessions -- ^ may be needed if one net-crypto session
                                                -- needs to possibly start another, as is
                                                -- the case in group chats
    , ncView               :: TVar SessionView
    , ncPacketQueue        :: PacketQueue CryptoData
    , ncBufferStart        :: TVar Word32
    , ncDequeueThread      :: Maybe ThreadId
    , ncPingMachine        :: Maybe PingMachine
    , ncOutgoingQueue      :: PQ.PacketOutQueue (State,Nonce24,TVar MsgOutMap)
                                                CryptoMessage
                                                (CryptoPacket Encrypted)
                                                CryptoData
    }

data NetCryptoSessions = NCSessions
    { netCryptoSessions       :: TVar (Map.Map SockAddr NetCryptoSession)
    , netCryptoSessionsByKey  :: TVar (Map.Map PublicKey [NetCryptoSession])
    , transportCrypto         :: TransportCrypto
    , defaultHooks            :: Map.Map MessageType [NetCryptoHook]
    , defaultUnrecognizedHook :: MessageType -> NetCryptoHook
    , sessionView             :: SessionView
    , msgTypeArray            :: MsgTypeArray
    , inboundQueueCapacity    :: Word32
    , outboundQueueCapacity   :: Word32
    , nextSessionId           :: TVar SessionID
    , announceNewSessionHooks :: TVar [IOHook (Maybe NoSpam) NetCryptoSession]
    }

type NewSessionHook = IOHook (Maybe NoSpam) NetCryptoSession

addNewSessionHook :: NetCryptoSessions -> NewSessionHook -> STM ()
addNewSessionHook allsessions@(NCSessions { announceNewSessionHooks }) hook = modifyTVar announceNewSessionHooks (hook:)

forgetCrypto :: TransportCrypto -> NetCryptoSessions -> NetCryptoSession -> STM ()
forgetCrypto crypto (NCSessions {netCryptoSessions,netCryptoSessionsByKey}) session = do
    let addr = ncSockAddr session
        sid = ncSessionId session
        sPubKey = ncTheirPublicKey session
    byAddrMap <- readTVar netCryptoSessions
 {- byKeyMap  <- readTVar netCryptoSessionsByKey -}
    case Map.lookup addr byAddrMap of
        Nothing -> return () -- already gone
        Just _ -> do
            modifyTVar netCryptoSessions      (Map.delete addr)
            modifyTVar netCryptoSessionsByKey (Map.update (\xs -> case filter (\x -> ncSessionId x /= sid) xs of
                                                                    [] -> Nothing
                                                                    ys -> Just ys) sPubKey)

newSessionsState :: TransportCrypto
                 -> (MessageType -> NetCryptoHook)      -- ^ default hook
                 -> Map.Map MessageType [NetCryptoHook] -- ^ all hooks, can be empty to start
                 -> IO NetCryptoSessions
newSessionsState crypto unrechook hooks = do
    x <- atomically $ newTVar Map.empty
    x2 <- atomically $ newTVar Map.empty
    nick <- atomically $ newTVar B.empty
    status <- atomically $ newTVar Online
    statusmsg <- atomically $ newTVar B.empty
    nospam <- atomically $ newTVar Nothing
    grps <- atomically $ newTVar Map.empty
    pname <- getProgName
    cachedir <- sensibleCacheDirCreateIfMissing pname
    tmpdir <- (</> pname) <$> (getTemporaryDirectory >>= canonicalizePath) -- getCanonicalTemporaryDirectory
    configdir <- sensibleVarLib pname
    homedir <- getHomeDirectory
    svDownloadDir0 <- atomically $ newTVar (homedir </> "Downloads")
    nextSessionId0 <- atomically $ newTVar 0
    announceNewSessionHooks0 <- atomically $ newTVar []
    return NCSessions { netCryptoSessions       = x
                      , netCryptoSessionsByKey  = x2
                      , transportCrypto         = crypto
                      , defaultHooks            = hooks
                      , defaultUnrecognizedHook = unrechook
                      , sessionView             = SessionView
                            { svNick        = nick
                            , svStatus      = status
                            , svStatusMsg   = statusmsg
                            , svNoSpam      = nospam
                            , svGroups      = grps
                            , svCacheDir    = cachedir
                            , svTmpDir      = tmpdir
                            , svConfigDir   = configdir
                            , svDownloadDir = svDownloadDir0
                            }
                      , msgTypeArray            = allMsgTypes -- todo make this a parameter
                      , inboundQueueCapacity    = 200
                      , outboundQueueCapacity   = 400
                      , nextSessionId           = nextSessionId0
                      , announceNewSessionHooks = announceNewSessionHooks0
                      }

data HandshakeParams
        = HParam
            { hpTheirBaseNonce        :: Maybe Nonce24 -- ignore and generate your own
            , hpOtherCookie           :: Cookie
            , hpTheirSessionKeyPublic :: PublicKey
            , hpMySecretKey           :: SecretKey
            , hpCookieRemotePubkey    :: PublicKey
            , hpCookieRemoteDhtkey    :: PublicKey
            }

newHandShakeData :: TransportCrypto -> Nonce24 -> HandshakeParams -> SockAddr -> IO (Maybe HandshakeData)
newHandShakeData crypto basenonce (HParam {hpOtherCookie,hpMySecretKey,hpCookieRemotePubkey,hpCookieRemoteDhtkey}) addr
    = do
    freshCookie
      <- case  nodeInfo (key2id hpCookieRemoteDhtkey) addr  of
            Right nodeinfo -> Just <$> cookieRequestH crypto nodeinfo (CookieRequest hpCookieRemotePubkey)
            Left er        -> return Nothing
    let hinit                = hashInit
        Cookie n24 encrypted = hpOtherCookie
        hctx                 = hashUpdate hinit n24
        hctx'                = hashUpdate hctx encrypted
        digest               = hashFinalize hctx'
-- parameters addr {--> SockAddr -}
--    mbcookie <- case hpOtherCookie of
--                    Nothing -> case (nodeInfo hpCookieRemoteDhtkey addr) of
--                                Right nodeinfo -> cookieRequest crypto netCryptoDHTClient (toPublic hpMySecretKey) nodeinfo
--                                Left er -> return Nothing
--                    Just c -> return (Just c)

    return $
      fmap (\freshCookie' ->
            HandshakeData
                { baseNonce   = basenonce
                , sessionKey  = toPublic hpMySecretKey
                , cookieHash  = digest
                , otherCookie = freshCookie'
                }) freshCookie

type XMessage = CryptoMessage -- todo

ncToWire :: STM (State,Nonce24,TVar MsgOutMap)
         -> Word32{- packet number we expect to recieve -}
         -> Word32{- buffer_end -}
         -> Word32{- packet number -}
         -> XMessage
         -> STM (Maybe (CryptoPacket Encrypted,Word32{-next packet no-}))
ncToWire getState seqno bufend pktno msg = do
    let typ = getMessageType msg
        typ64 = toWord64 typ
    let lsness msg =
            case typ of
                    Msg mid -> lossyness mid
                    GrpMsg KnownLossy _ -> Lossy
                    GrpMsg KnownLossless _ -> Lossless
    (state,n24,msgOutMapVar) <- getState
    msgOutMap <- readTVar msgOutMapVar
    case msgOutMapLookup typ64 msgOutMap of
        Just outid -> do
            let setMessageId (OneByte _)   mid = OneByte (toEnum8 mid)
                setMessageId (TwoByte _ x) mid = TwoByte (toEnum8 mid) x
                setMessageId (UpToN _ x)   mid = UpToN (toEnum8 mid) x
                msg' = setMessageId msg outid
                in case lsness msg of
                        UnknownLossyness -> return Nothing
                        Lossy    -> let cd =
                                         CryptoData
                                            { bufferStart = seqno
                                            , bufferEnd = bufend
                                            , bufferData = msg'
                                            }
                                        plain = encodePlain cd
                                        encrypted = encrypt state plain
                                        pkt = CryptoPacket { pktNonce = nonce24ToWord16 n24, pktData = encrypted }
                                        in return (Just (pkt, pktno))
                        Lossless -> let cd =
                                         CryptoData
                                            { bufferStart = seqno
                                            , bufferEnd = pktno
                                            , bufferData = msg'
                                            }
                                        plain = encodePlain cd
                                        encrypted = encrypt state plain
                                        pkt = CryptoPacket { pktNonce = nonce24ToWord16 n24, pktData = encrypted }
                                        in return (Just (pkt, pktno+1))

-- | Blocks until a packet is available, or the session is terminated.
receiveCrypto :: NetCryptoSession -> IO (Maybe CryptoMessage)
receiveCrypto nc@NCrypto { ncState, ncPacketQueue } = do
    cd <- atomically $ do
            st <- readTVar ncState
            case st of
                Accepted  -> Just <$> PQ.dequeue ncPacketQueue
                Confirmed -> Just <$> PQ.dequeue ncPacketQueue
                _         -> pure Nothing
    forM_ cd $ runCryptoHook nc -- TODO: Is this useful?
    return $ bufferData <$> cd

-- | called when we recieve a crypto handshake with valid cookie
-- TODO set priority on contact addr to 0 if it is older than ForgetPeriod,
-- then increment it regardless. (Keep addr in MinMaxPSQ in Roster.Contact)
freshCryptoSession :: NetCryptoSessions -> SockAddr -> HandshakeParams -> IO ()
freshCryptoSession  sessions
                    addr
                    hp@(HParam
                              { hpTheirBaseNonce = Just theirBaseNonce
                              , hpOtherCookie    = otherCookie
                              , hpTheirSessionKeyPublic = theirSessionKey
                              , hpMySecretKey        = key
                              , hpCookieRemotePubkey = remotePublicKey
                              , hpCookieRemoteDhtkey = remoteDhtPublicKey
                              }) = do
    let crypto = transportCrypto sessions
        allsessions = netCryptoSessions sessions
        allsessionsByKey = netCryptoSessionsByKey sessions
    sessionId <- atomically $ do
        x <- readTVar (nextSessionId sessions)
        modifyTVar    (nextSessionId sessions) (+1)
        return x
    ncState0 <- atomically $ newTVar Accepted
    ncTheirBaseNonce0 <- atomically $ newTVar theirBaseNonce
    n24 <- atomically $ transportNewNonce crypto
    state <- lookupSharedSecret crypto key remoteDhtPublicKey n24
    newBaseNonce <- atomically $ transportNewNonce crypto
    mbMyhandshakeData <- newHandShakeData crypto newBaseNonce hp addr
    let encodeHandshake myhandshakeData = let plain = encodePlain myhandshakeData
    --                                        state = computeSharedSecret key remoteDhtPublicKey n24
                                              encrypted = encrypt state plain
                                              in Handshake { handshakeCookie = otherCookie
                                                           , handshakeNonce = n24
                                                           , handshakeData = encrypted
                                                           }
    let myhandshake= encodeHandshake <$> mbMyhandshakeData
    ncMyPacketNonce0 <- atomically $ newTVar newBaseNonce
    ncHandShake0 <- atomically $ newTVar myhandshake
    cookie0 <- atomically $ newTVar (Just otherCookie)
    newsession <- generateSecretKey
    ncHooks0 <- atomically $ newTVar (defaultHooks sessions)
    ncUnrecognizedHook0 <- atomically $ newTVar (defaultUnrecognizedHook sessions)
    ncIncomingTypeArray0 <- atomically $ newTVar (msgTypeArray sessions)
    let idMap = foldl (\mp (x,y) -> W64.insert x y mp) W64.empty (zip [0..255] [0..255])
    ncOutgoingIdMap0 <- atomically $ newTVar idMap
    ncView0 <- atomically $ newTVar (sessionView sessions)
    pktq <- atomically $ PQ.new (inboundQueueCapacity sessions) 0
    bufstart <- atomically $ newTVar 0
    let toWireIO = do
            f <- lookupNonceFunction crypto newsession theirSessionKey
            atomically $ do
                n24 <- readTVar ncMyPacketNonce0
                let n24plus1 = incrementNonce24 n24
                writeTVar ncMyPacketNonce0 n24plus1
                return (return (f n24, n24, ncOutgoingIdMap0))
    pktoq <- atomically $ PQ.newOutGoing pktq ncToWire toWireIO 0 (outboundQueueCapacity sessions) 0
    let netCryptoSession0 =
            NCrypto { ncState              = ncState0
                    , ncSessionId          = sessionId
                    , ncTheirPublicKey     = remotePublicKey
                    , ncTheirBaseNonce     = ncTheirBaseNonce0
                    , ncMyPacketNonce      = ncMyPacketNonce0
                    , ncHandShake          = ncHandShake0
                    , ncCookie             = cookie0
                    , ncTheirDHTKey        = remoteDhtPublicKey
                    , ncTheirSessionPublic = Just theirSessionKey
                    , ncSessionSecret      = newsession
                    , ncSockAddr           = addr
                    , ncHooks              = ncHooks0
                    , ncUnrecognizedHook   = ncUnrecognizedHook0
                    , ncAllSessions        = sessions
                    , ncIncomingTypeArray  = ncIncomingTypeArray0
                    , ncOutgoingIdMap      = ncOutgoingIdMap0
                    , ncView               = ncView0
                    , ncPacketQueue        = pktq
                    , ncBufferStart        = bufstart
                    , ncDequeueThread      = Nothing -- error "you want the NetCrypto-Dequeue thread id, but is it started?"
                    , ncPingMachine        = Nothing -- error "you want the NetCrypto-PingMachine, but is it started?"
                    , ncOutgoingQueue      = pktoq
                    }
    -- launch dequeue thread
    threadid <- forkIO $ do
        tid <- myThreadId
        labelThread tid ("NetCryptoDequeue." ++ show (key2id remotePublicKey))
        fix $ \loop -> do
            cd <- atomically $ PQ.dequeue pktq
            _ <- runCryptoHook (netCryptoSession0 {ncDequeueThread=Just tid}) cd
            loop
    -- launch ping thread
    fuzz <- randomRIO (0,2000)
    pingMachine <- forkPingMachine ("NetCrypto." ++ show (key2id remotePublicKey)) (15000 + fuzz) 2000
    -- update session with thread ids
    let netCryptoSession = netCryptoSession0 {ncDequeueThread=Just threadid, ncPingMachine=Just pingMachine}
    -- add this session to the lookup maps
    atomically $ do
        modifyTVar allsessions (Map.insert addr netCryptoSession)
        byKeyResult <- readTVar allsessionsByKey >>= return . Map.lookup remotePublicKey
        case byKeyResult of
            Nothing -> modifyTVar allsessionsByKey (Map.insert remotePublicKey [netCryptoSession])
            Just xs -> modifyTVar allsessionsByKey (Map.insert remotePublicKey (netCryptoSession:xs))
    -- run announceNewSessionHooks
    hooks <- atomically $ readTVar (announceNewSessionHooks sessions)
    flip fix (hooks,netCryptoSession) $ \loop (hooks,session) ->
        case hooks of
            [] -> return ()
            (h:hs) -> do 
                r <- h Nothing session
                case r of
                  Just f -> loop (hs, f session)
                  Nothing -> return ()

-- | Called when we get a handshake, but there's already a session entry.
updateCryptoSession :: NetCryptoSessions -> SockAddr -> HandshakeParams -> NetCryptoSession -> IO ()
updateCryptoSession sessions addr hp session = do
    ncState0 <- atomically $ readTVar (ncState session)
    ncTheirBaseNonce0 <- atomically $ readTVar (ncTheirBaseNonce session)
    if (ncState0 >= Accepted)
        -- If the nonce in the handshake and the dht key are both the same as
        -- the ones we have saved, assume we already handled this and this is a
        -- duplicate handshake packet, otherwise disregard everything, and
        -- refresh all state.
        --
     then when (  Just ncTheirBaseNonce0 /= hpTheirBaseNonce hp -- XXX: Do we really want to compare base nonce here?
               || ncTheirDHTKey session /= hpCookieRemoteDhtkey hp
               ) $ freshCryptoSession sessions addr hp
     else if ( Just ncTheirBaseNonce0 /= hpTheirBaseNonce hp)
            then freshCryptoSession sessions addr hp -- basenonce mismatch, trigger refresh
            else atomically $ writeTVar (ncState session) Accepted


cryptoNetHandler :: NetCryptoSessions -> SockAddr -> NetCrypto -> IO (Maybe (NetCrypto -> NetCrypto))
cryptoNetHandler sessions addr (NetHandshake (Handshake (Cookie n24 ecookie) nonce24 encrypted)) = do
    -- Handle Handshake Message
    let crypto = transportCrypto sessions
        allsessions = netCryptoSessions sessions
        anyRight [] f     = return $ Left "missing key"
        anyRight (x:xs) f = f x >>= either (const $ anyRight xs f) (return . Right)
    seckeys <- map fst <$> atomically (readTVar (userKeys crypto))
    symkey <- atomically $ transportSymmetric crypto
    now <- getPOSIXTime
    lr <- fmap join . sequence $ do -- Either Monad
            (CookieData cookieTime remotePubkey remoteDhtkey) <- (decodePlain =<< decryptSymmetric symkey n24 ecookie)
            Right $ do -- IO Monad
            decrypted <- anyRight seckeys $ \key -> do
                            secret <- lookupSharedSecret crypto key remotePubkey nonce24
                            return $ (key,) <$> (decodePlain =<< decrypt secret encrypted)
            return $ do -- Either Monad
            (key,HandshakeData { baseNonce, sessionKey, cookieHash, otherCookie }) <- decrypted
            -- check cookie time < 15 seconds ago
            guard (now - fromIntegral cookieTime < 15)
            -- cookie hash is valid? sha512 of ecookie
            let hinit = hashInit
                hctx = hashUpdate hinit n24
                hctx' = hashUpdate hctx ecookie
                digest = hashFinalize hctx'
            guard (cookieHash == digest)
            -- known friend?
            -- todo TODO, see Roster.hs,
            -- talk to not yet existent Network-Manager to ascertain current permissions
            return
             HParam
                { hpTheirBaseNonce        = Just baseNonce
                , hpOtherCookie           = otherCookie
                , hpTheirSessionKeyPublic = sessionKey
                , hpMySecretKey           = key
                , hpCookieRemotePubkey    = remotePubkey
                , hpCookieRemoteDhtkey    = remoteDhtkey
                }
    case lr of
        Left _ -> return ()
        Right hp@(HParam
                      { hpTheirBaseNonce        = Just theirBaseNonce
                      , hpOtherCookie           = otherCookie
                      , hpTheirSessionKeyPublic = theirSessionKey
                      , hpMySecretKey           = key
                      , hpCookieRemotePubkey    = remotePublicKey
                      , hpCookieRemoteDhtkey    = remoteDhtPublicKey
                      }) -> do
            sessionsmap <- atomically $ readTVar allsessions
            -- Do a lookup, so we can handle the update case differently
            case Map.lookup addr sessionsmap of
                Nothing -> freshCryptoSession sessions addr hp -- create new session
                Just session -> updateCryptoSession sessions addr hp session -- update existing session
    return Nothing


cryptoNetHandler sessions addr (NetCrypto (CryptoPacket nonce16 encrypted)) = do
    let crypto = transportCrypto sessions
        allsessions = netCryptoSessions sessions
    sessionsmap <- atomically $ readTVar allsessions
    -- Handle Encrypted Message
    case Map.lookup addr sessionsmap of
        Nothing -> return Nothing -- drop packet, we have no session
        Just session@(NCrypto { ncIncomingTypeArray, ncState, ncPacketQueue, ncHooks,
                                ncSessionSecret, ncTheirSessionPublic, ncTheirBaseNonce,
                                ncPingMachine}) -> do
            theirBaseNonce <- atomically $ readTVar ncTheirBaseNonce
            -- Try to decrypt message
            let diff :: Word16
                diff      = nonce16 - fromIntegral (last2Bytes theirBaseNonce) -- truncating to Word16
                tempNonce = addtoNonce24 theirBaseNonce (fromIntegral diff)    -- expanding to Word
            lr <- fmap join $ sequence $ do -- Either Monad --
                        pubkey <- maybeToEither ncTheirSessionPublic
                        Right $ do -- IO Monad
                        secret <- lookupSharedSecret crypto ncSessionSecret pubkey tempNonce
                        return $ decodePlain =<< decrypt secret encrypted
            case lr of
                Left _ -> return Nothing -- decryption failed, ignore packet
                Right cd@(CryptoData {bufferStart, bufferEnd, bufferData=cm}) -> do -- decryption succeeded,
                    -- TODO: Why do I need bufferStart & bufferEnd?
                    --
                    --   buffer_start = highest packet number handled + 1
                    --                , recvbuffers buffer_start
                    --
                    --   bufferEnd = sendbuffer buffer_end if lossy, otherwise packet number
                    -- update ncTheirBaseNonce if necessary
                    when (diff > 2 * dATA_NUM_THRESHOLD)$
                        atomically $ do
                            y <- readTVar ncTheirBaseNonce
                            let x = addtoNonce24 y (fromIntegral dATA_NUM_THRESHOLD)
                            writeTVar ncTheirBaseNonce y
                    -- then set session confirmed,
                    atomically $ writeTVar ncState Confirmed
                    -- bump ping machine
                    case ncPingMachine of
                        Just pingMachine -> pingBump pingMachine
                        Nothing -> return ()
                    msgTypes <- atomically $ readTVar ncIncomingTypeArray
                    let msgTyp = cd ^. messageType
                        msgTypMapped16 = msgTypes A.! fromEnum8 (msgID cm)
                        msgTypMapped = fromWord16 $ msgTypMapped16
                        isLossy (GrpMsg KnownLossy _) = True
                        isLossy (Msg mid) | lossyness mid == Lossy = True
                        isLossy _ = False
                    if isLossy msgTypMapped
                        then do atomically $ PQ.observeOutOfBand ncPacketQueue bufferEnd
                                runCryptoHook session cd
                        else do atomically $ PQ.enqueue ncPacketQueue bufferEnd cd
                                return Nothing
    where
        last2Bytes :: Nonce24 -> Word
        last2Bytes (Nonce24 bs) = case S.decode (B.drop 22 bs) of
                                    Right n -> n
                                    _ -> error "unreachable-last2Bytes"
        dATA_NUM_THRESHOLD = 21845 -- = 65535 / 3

-- | TODO: make this accept CrytpoMessage instead
runCryptoHook :: NetCryptoSession -> CryptoData -> IO (Maybe (x -> x))
runCryptoHook session@(NCrypto {ncState, ncHooks,ncSessionSecret,ncTheirSessionPublic,ncTheirBaseNonce,ncIncomingTypeArray})
              cd@(CryptoData {bufferStart, bufferEnd, bufferData=cm}) = do
    hookmap <- atomically $ readTVar ncHooks
    -- run hook
    flip fix (cd,hookmap) $ \lookupAgain (cd,hookmap) -> do
        msgTypes <- atomically $ readTVar ncIncomingTypeArray
        let msgTyp = cd ^. messageType
            msgTypMapped16 = msgTypes A.! fromEnum8 (msgID cm)
            msgTypMapped = fromWord16 $ msgTypMapped16
        if msgTypMapped16 == 0
         then return Nothing
         else
           case Map.lookup msgTypMapped hookmap of
                Nothing -> do -- no recognizing hook, run ncUnrecognizedHook0, loopAgain on result
                             unrecognize <- atomically $ readTVar (ncUnrecognizedHook session)
                             mbConsume <- unrecognize msgTypMapped session cd
                             case mbConsume of
                                Just f -> do
                                           -- ncUnrecognizedHook0 may have updated the hookmap
                                           hookmap' <- atomically $ readTVar ncHooks
                                           lookupAgain (f cd,hookmap')
                                Nothing -> return Nothing
                Just hooks -> flip fix (hooks,cd,msgTypMapped) $ \loop (hooks,cd,typ) -> do
                                let _ = cd :: CryptoData
                                case (hooks,cd) of
                                    ([],_) ->  return Nothing
                                    (hook:more,cd) -> do
                                        r <- hook session cd :: IO (Maybe (CryptoData -> CryptoData))
                                        case r of
                                            Just f -> let newcd = f cd
                                                          newtyp = newcd ^. messageType
                                                          in if newtyp == typ then loop (more,newcd,newtyp)
                                                                              else lookupAgain (newcd,hookmap)
                                            Nothing -> return Nothing -- message consumed

-- | construct a 'MsgTypeArray' for specified types, using their known common positions
--   in the MessageId space if they have such a thing.
mkMsgTypes :: [MessageType] -> MsgTypeArray
mkMsgTypes msgs = let zeros = A.listArray (0,255) (replicate 256 0)
                      in zeros A.// map (\x -> (toIndex x,toWord16 x)) msgs
    where
        toIndex (Msg mid) = fromIntegral . fromEnum  $ mid
        toIndex (GrpMsg KnownLossless nam) = 0x63 -- fromEnum MESSAGE_GROUPCHAT
        toIndex (GrpMsg KnownLossy    nam) = 0xC7 -- fromEnum LOSSY_GROUPCHAT

-- | Handle all Tox messages that this code base is aware of.
allMsgTypes :: MsgTypeArray
allMsgTypes = A.listArray (minBound,maxBound) (0:knownMsgs)
  where
    knownMsgs :: [Word16]
    knownMsgs =
      concat [ map (fromIntegral . fromEnum) [ PacketRequest .. KillPacket ]
             , map (const 0)                 [ 3 .. 15 ]  -- UnspecifiedPacket
             , map (const 0)                 [ 16 .. 23 ] -- MessengerLoseless
             , map (fromIntegral . fromEnum) [ ONLINE .. OFFLINE ]
             , map (const 0)                 [ 26 .. 47 ] -- MessengerLoseless
             , map (fromIntegral . fromEnum) [ NICKNAME .. TYPING ]
             , map (const 0)                 [ 52 .. 63 ] -- MessengerLoseless
             , map (fromIntegral . fromEnum) [ MESSAGE .. ACTION ]
             , map (const 0)                 [ 66 .. 68 ] -- MessengerLoseless
             , map (fromIntegral . fromEnum) [ MSI ]
             , map (const 0)                 [ 70 .. 79 ] -- MessengerLoseless
             , map (fromIntegral . fromEnum) [ FILE_SENDREQUEST .. FILE_DATA ]
             , map (const 0)                 [ 83 .. 95 ] -- MessengerLoseless
             , map (fromIntegral . fromEnum) [ INVITE_GROUPCHAT .. MESSAGE_GROUPCHAT ]
             , map (const 0)                 [ 100 .. 191 ] -- MessengerLoseless
             , map (const 0)                 [ 192 .. 198 ] -- MessengerLossy
             , map (fromIntegral . fromEnum) [ LOSSY_GROUPCHAT ]
             , map (const 0)                 [ 200 .. 255 ] -- All lossy, exept the last
             ]

-- | handles nothing
defaultCryptoDataHooks :: Map.Map MessageType [NetCryptoHook]
defaultCryptoDataHooks = Map.empty

-- | discards all unrecognized packets
defaultUnRecHook :: MessageType -> NetCryptoHook
defaultUnRecHook _ _ _ = return Nothing

-- | use to add a single hook to a specific session.
addCryptoDataHook1 :: Map.Map MessageType [NetCryptoHook] -> MessageType -> NetCryptoHook -> Map.Map MessageType [NetCryptoHook]
addCryptoDataHook1 mp typ hook = case Map.lookup typ mp of
                                    Nothing -> Map.insert typ [hook] mp
                                    Just hooks -> Map.insert typ (hook:hooks) mp