fixes for new release

Author: tittoassini

CHANGELOG

@@ -0,0 +1,7 @@
+Significant and compatibility-breaking changes.
+
+Version 0.3:
+	- Removed 'flatStrict' and 'unflatStrict' (use 'flat' and 'unflat' instead that also encode/decode strictly)
+	- `unflatWith` now takes a decoder for the unpadded value (previously it expected a padded decoder) and decodes the padded value
+	- Added some decoding primitives
+	- Added Data.ByteString.Convert

README.md

@@ -31,10 +31,10 @@ data Direction = North | South | Center | East | West deriving (Show,Generic,Fla
 data List a = Nil | Cons a (List a) deriving (Show,Generic,Flat)
 ```
 
-For encoding, use `flat`, for decoding, use `unflat` (or equivalently: `flatStrict` and `unflatStrict`):
+For encoding, use `flat`, for decoding, use `unflat`:
 
 ```haskell
-unflatStrict . flat $ Cons North (Cons South Nil) :: Decoded (List Direction)
+unflat . flat $ Cons North (Cons South Nil) :: Decoded (List Direction)
 -> Right (Cons North (Cons South Nil))
 ```
 
@@ -119,9 +119,9 @@ Briefly:
  * Encoding: `store` and `flat` are usually faster
  * Decoding: `store`, `cereal` and `flat` are usually faster
 
- One thing that is not shown by the benchmarks is that, if the serialized data is to be transferred over a network, the total total transfer time (encoding time + transmission time + decoding time) is usually dominated by the transmission time and that's where the smaller binaries produced by flat give it a significant advantage.
+ One thing that is not shown by the benchmarks is that, if the serialized data is to be transferred over a network, the total transfer time (encoding time + transmission time + decoding time) is usually dominated by the transmission time and that's where the smaller binaries produced by flat give it a significant advantage.
 
- Consider for example the Cars dataset. As you can see in the following comparison with `store`, the overall top performer for encoding/decoding speed, the total transfer time is actually significantly lower for `flat` for all except the highest transmission speeds.
+ Consider for example the Cars dataset. As you can see in the following comparison with `store`, the overall top performer for encoding/decoding speed, the transfer time is actually significantly lower for `flat` for all except the highest transmission speeds.
 
 ||Store|Flat|
 |---|---|---|

flat.cabal

@@ -1,29 +1,31 @@
 name: flat
-version: 0.2.2
+version: 0.2.3
 synopsis: Principled and efficient bit-oriented binary serialization.
 description: See the <http://github.com/tittoassini/flat online tutorial>.
 homepage: http://github.com/tittoassini/flat
-copyright: Copyright: (c) 2016 Pasqualino `Titto` Assini
-author: Pasqualino `Titto` Assini
-license: BSD3
-license-file: LICENSE
-maintainer: tittoassini@gmail.com
+license:             BSD3
+license-file:        LICENSE
+author:              Pasqualino `Titto` Assini
+maintainer:          tittoassini@gmail.com
+copyright:           Copyright: (c) 2016 Pasqualino `Titto` Assini
 category: Data,Parsing,Serialization
 cabal-version: >=1.10
 build-type: Simple
-tested-with: GHC ==7.10.3 GHC ==8.0.2
+Tested-With: GHC == 7.10.3 GHC == 8.0.1 GHC == 8.0.2
 extra-source-files:
     stack.yaml
     stack801.yaml
     stack802.yaml
     README.md
+    CHANGELOG
 
 source-repository head
     type: git
     location: https://github.com/tittoassini/flat
 
 library
     exposed-modules:
+        Data.ByteString.Convert
         Data.Flat.Bits
         Data.Flat.Class
         Data.Flat.Decoder

other/Pretty.hs

@@ -7,24 +7,26 @@
 {-# LANGUAGE FlexibleInstances ,CPP #-}
 {-# LANGUAGE LambdaCase    ,ScopedTypeVariables    #-}
 module Test where
-import           Data.Binary.FloatCast
 import qualified Data.ByteString                as B
 import qualified Data.ByteString.Lazy as L
 import qualified Data.ByteString.Short.Internal as SBS
 import           Data.Coerce
 import           Data.Flat
-import           Data.Flat.Peek
 import           Data.Flat.Decoder
-import           Data.Flat.Pretty
 import           Data.Int
 import qualified Data.Sequence                  as S
 import qualified Data.Text                      as T
-import           Data.Word
 import           Prelude                        hiding (exponent, sign)
 import           System.Endian
 import           Text.Printf
 import Data.Flat.Bits
 import qualified Data.Map as M
+import Data.FloatCast
+-- import qualified Data.ByteString                as B
+-- import qualified Data.ByteString.Lazy           as L
+import           Data.Word
+import           Text.PrettyPrint.HughesPJClass
+-- import           Text.Printf
 
 instance Flat [Word16]
 instance Flat [Int16]
@@ -33,8 +35,11 @@ instance Flat [Word8]
 instance Flat [(Word64,Word16)]
 instance Flat [ABC]
 
+--u :: B.ByteString
+u = unflatWith (dBEBits8 3) [128+64+32+1::Word8]
+
 m1 = M.fromList [(False,True)]
-mmm = (size m1 0,bits m1, unflat $ flat m1 :: Decoded (M.Map Bool Bool))
+mmm = (size m1 0,bits m1, unflatRaw $ flat m1 :: Decoded (M.Map Bool Bool))
 
 -- deriving  instance {-# OVERLAPPABLE #-} Flat a => Flat [a]
 -- instance Flat a => Flat [a]
@@ -73,7 +78,7 @@ newtype A = A Bool
 newtype B = B Bool
 data C = C Bool
 
-kkk = [[127,1],[128,1,1],[129,1,1],[255,127,1],[128,128,1,1],[129,128,1,1],[255,255,1,1],[128,128,2,1],[129,128,2,1],[255,255,127,1],[128,128,128,1,1],[129,128,128,1,1]] == map (L.unpack . flat) [127::Word32,128,129,16383,16384,16385,32767,32768,32769,2097151,2097152,2097153]
+kkk = [[127,1],[128,1,1],[129,1,1],[255,127,1],[128,128,1,1],[129,128,1,1],[255,255,1,1],[128,128,2,1],[129,128,2,1],[255,255,127,1],[128,128,128,1,1],[129,128,128,1,1]] == map (B.unpack . flat) [127::Word32,128,129,16383,16384,16385,32767,32768,32769,2097151,2097152,2097153]
 
 -- qqq = size N4 0 + size N5 0
 
@@ -145,7 +150,7 @@ n1 = Node (Node (Leaf 11) (Leaf 22)) (Node (Leaf 33) (Leaf 44))
 -- [240,88,90,33,22,96,176,180,66,44,224,176,180,66,44,193,97,104,132,89]
 n2 = Node (Node n1 n1) (Node (Node n1 n1) n1)
 
-u =  pp $ WW 11 22 33 44 True
+u4 =  pp $ WW 11 22 33 44 True
 uu =  pp (11::Word32,22::Word64,33::Word64)
 --o = encodings $ postAligned $ T2 (11::Word32) (22::Word64)
 ww = unflat (flat (0::Word16)) :: Decoded Word16
@@ -201,7 +206,7 @@ y11 =show __GLASGOW_HASKELL__
 pp :: forall a . (Flat a, Show a) => a -> IO ()
 -- pp v = putStrLn (unwords [show v,"->",show (size :: Size a),show $ getSize (postAligned v),show $ encode v,"->",show $ L.unpack $ flat v])
 -- pp v = putStrLn (unwords [show v,"->",show (size :: Size a),show $ getSize v,show $ encode v,"->",show $ L.unpack $ flat v])
-pp v = putStrLn (unwords [show v,"->",show $ getSize v,show $ encode v,"->",show $ L.unpack $ flat v])
+pp v = putStrLn (unwords [show v,"->",show $ getSize v,show $ encode v,"->",show $ flat v])
 
 -- gg :: Flat a => a -> Vector Bool
 gg = bits . flat $ "abc"
@@ -213,7 +218,9 @@ bs = [32,32,32::Word8]
 
 -- px v = putStrLn (unwords [show v,"->",show $ encode v,"->",show $ L.unpack $ flatRaw (T2 v FillerEnd)])
 
-px v = let bs = L.unpack $ flat v in putStrLn (unwords [show v,"->",show $ encode v,"->",show bs,show $ sum bs])
+-- px :: Flat a => a -> IO ()
+px v = let bs = B.unpack $ flat v
+       in putStrLn (unwords [show v,"->",show $ encode v,"->",show bs,show $ sum bs])
 
 pf v = putStrLn (unwords [show v,"->",show $ flat v])
 
@@ -225,61 +232,61 @@ ll = flat [True,False,True]
 
 f3 = doubleToWord 1
 
-f2 =flat (1::Double)
+f2 =flat(1::Double)
 
-kk = flat [(55::Word64,18::Word16),(5599::Word64,1122::Word16)]
+kk = flat[(55::Word64,18::Word16),(5599::Word64,1122::Word16)]
 -- tt = encode [(55::Word64,18::Word16,True,AA)]
 e1 = encode (True,False) -- (AA,BB)
 e2 = encode [AA,BB]
 e3 = encode [True,False]
 e4 = encode $ DD True True
 -- e5 = encode $ ZZ (ZZ (ZZ (DD True True)))
 
-m = runGetStrict getChunksInfo $ B.pack [3,11,22,33,2,11,22,0]
+-- m = runGetStrict getChunksInfo $ B.pack [3,11,22,33,2,11,22,0]
 
-m2 = B.unpack <$> runGetStrict dByteString_ (B.pack [3,11,22,33,2,11,22,0])
-m3 = SBS.unpack <$> runGetStrict dShortByteString_ (B.pack [3,11,22,33,2,11,22,0])
-m4 = unflat (flat (T.pack "abc")) :: Decoded T.Text
+-- m2 = B.unpack <$> runGetStrict dByteString_ (B.pack [3,11,22,33,2,11,22,0])
+-- m3 = SBS.unpack <$> runGetStrict dShortByteString_ (B.pack [3,11,22,33,2,11,22,0])
+-- m4 = unflat (flat(T.pack "abc")) :: Decoded T.Text
 m5 = unflat (flat '经') :: Decoded Char
 
 {-# NOINLINE tup2 #-}
-tup2 a b = flat (a,b)
+tup2 a b = flat(a,b)
 --t = coerce (1::Word32) :: Float
--- b = L.unpack $ flat $ B.pack $ replicate 250 33
-b = L.unpack $ flat $ (11::Word8,SBS.toShort $ B.replicate 400 33) -- [1..255] B.pack bs
+-- b = L.unpack $ flat$ B.pack $ replicate 250 33
+-- b = L.unpack $ flat$ (11::Word8,SBS.toShort $ B.replicate 400 33) -- [1..255] B.pack bs
 
-w = L.unpack $ flat (2::Word16,2::Word32,2::Word64)
+-- w = L.unpack $ flat(2::Word16,2::Word32,2::Word64)
 
-l = L.unpack $ flat [1::Word16,1,1]
+-- l = L.unpack $ flat[1::Word16,1,1]
 
-j = L.unpack $ flat "abc"
+-- j = L.unpack $ flat "abc"
 
-js = L.unpack $ flat (S.fromList "abc")
+-- js = L.unpack $ flat(S.fromList "abc")
 
-jj = L.unpack $ flat $ (11::Word8,T.pack "\x1F600\&000aaa维维aaa")
+-- jj = L.unpack $ flat$ (11::Word8,T.pack "\x1F600\&000aaa维维aaa")
 
-jl = T.length (T.pack "\x1F600\&\x1F600\&")
+-- jl = T.length (T.pack "\x1F600\&\x1F600\&")
 
 q :: Decoded T.Text
-q = unflat $ flat (T.pack "D\226\FStz\GS3]\n8\149sV\243J\181\181\235\214&y\226\231\&2\239\212\174\DC1J'F\129hpsu\199\178")
+q = unflat $ flat(T.pack "D\226\FStz\GS3]\n8\149sV\243J\181\181\235\214&y\226\231\&2\239\212\174\DC1J'F\129hpsu\199\178")
 
 g = let v = (-1::Int16,255::Word8,False,-1::Int16,1::Word8,0::Word8,False)
     in (unflat (flat v) == Right v,bits v)
 
 g1 :: Decoded (Bool,Bool,Bool,Bool)
-g1 = unflat $ flat (True,False,True,True)
+g1 = unflat $ flat(True,False,True,True)
 
 g2 :: Decoded (Bool,Word8,Word8,Bool)
-g2 = unflat $ flat (False,1::Word8,1::Word8,False)
+g2 = unflat $ flat(False,1::Word8,1::Word8,False)
 
 g3 :: Decoded Word8
-g3 = unflat $ flat (0::Word8)
+g3 = unflat $ flat(0::Word8)
 
 g4 :: Decoded (Bool,Word8,Bool)
-g4 = unflat $ flat (False,0::Word8,False)
+g4 = unflat $ flat(False,0::Word8,False)
 
 g5 :: Decoded (Float,Double,Bool,Float,Double,Bool)
-g5 = unflat $ flat (8.11E-11::Float,8.11E-11::Double,True,8.11E-11::Float,8.11E-11::Double,True)
+g5 = unflat $ flat(8.11E-11::Float,8.11E-11::Double,True,8.11E-11::Float,8.11E-11::Double,True)
 
 f0 = bits ((False,255::Word8,False,255::Word8))
 f = bits (False,0::Word8,False)
@@ -296,20 +303,31 @@ d = prettyLBS $ flatRaw (3/0::Double)
 s = serRaw (True,False,True)
 
 z = bits $ (True,False,True)
-zz = L.unpack . flat $ (True,False,True)
+zz = flat (True,False,True)
 
 serRaw :: Flat a => a -> [Word8]
 --serRaw = L.unpack . flatRaw
 serRaw = asBytes . bits
 
 y :: Decoded Float
-y = unflat $ flat (-0.15625::Double)
+y = unflat $ flat(-0.15625::Double)
 
 -- i :: Decoded IEEE_754_binary32
--- i = unflat $ flat (-0.15625::Float)
+-- i = unflat $ flat(-0.15625::Float)
 
 --t = i == Right (IEEE_754_binary32 {sign = V1, exponent = Bits8 V0 V1 V1 V1 V1 V1 V0 V0, fraction = (Bits7 V0 V1 V0 V0 V0 V0 V0,Bits8 V0 V0 V0 V0 V0 V0 V0 V0,Bits8 V0 V0 V0 V0 V0 V0 V0 V0)})
 bb = 0b111
 
 c = (isInfinite (3/0),isNegativeZero (-0::Double),isNaN (0/0::Float),(0/0::Float)==(0/0::Float))
 
+prettyLBS :: L.ByteString -> String
+prettyLBS = render . prettyBL . L.unpack
+
+prettyBS :: B.ByteString -> String
+prettyBS = render . prettyBL . B.unpack
+
+prettyBL :: [Word8] -> Doc
+prettyBL = text . unwords . map prettyWord8
+
+prettyWord8 :: Word8 -> String
+prettyWord8 = printf "%08b"

other/Test.hs

@@ -0,0 +1,24 @@
+{-# LANGUAGE FlexibleInstances #-}
+-- |Convert to/from strict ByteStrings
+module Data.ByteString.Convert (AsByteString(..)) where
+
+import qualified Data.ByteString      as B
+import qualified Data.ByteString.Lazy as L
+import           Data.Word
+
+class AsByteString a where
+  toByteString :: a -> B.ByteString
+  fromByteString :: B.ByteString -> a
+
+instance AsByteString B.ByteString where
+  toByteString = id
+  fromByteString = id
+
+instance AsByteString L.ByteString where
+  toByteString = L.toStrict
+  fromByteString = L.fromStrict
+
+instance AsByteString [Word8] where
+  toByteString = B.pack
+  fromByteString = B.unpack
+

src/Data/ByteString/Convert.hs

@@ -4,15 +4,22 @@
 {-# LANGUAGE TypeSynonymInstances #-}
 
 -- |Utilities to represent and display bit sequences
-module Data.Flat.Bits (Bits, toBools, bits, paddedBits, asBytes) where
+module Data.Flat.Bits (
+    Bits,
+    toBools,
+    fromBools,
+    bits,
+    paddedBits,
+    asBytes,
+    asBits,
+    ) where
 
 import           Data.Bits                      hiding (Bits)
-import qualified Data.ByteString           as L
-import           Data.Flat.Decoder
+import qualified Data.ByteString                as B
 import           Data.Flat.Class
+import           Data.Flat.Decoder
 import           Data.Flat.Filler
 import           Data.Flat.Run
--- import           Data.Int
 import qualified Data.Vector.Unboxed            as V
 import           Data.Word
 import           Text.PrettyPrint.HughesPJClass
@@ -23,19 +30,27 @@ type Bits = V.Vector Bool
 toBools :: Bits -> [Bool]
 toBools = V.toList
 
+fromBools :: [Bool] -> Bits
+fromBools = V.fromList
+
 -- |The sequence of bits corresponding to the serialization of the passed value (without any final byte padding)
 bits :: forall a. Flat a => a -> Bits
 bits v = let lbs = flat v
              Right (PostAligned _ f) = unflatRaw lbs :: Decoded (PostAligned a)
-         in takeBits (8 * L.length lbs - fillerLength f) lbs
+         in takeBits (8 * B.length lbs - fillerLength f) lbs
 
 -- |The sequence of bits corresponding to the byte-padded serialization of the passed value
 paddedBits :: forall a. Flat a => a -> Bits
 paddedBits v = let lbs = flat v
-               in takeBits (8 * L.length lbs) lbs
+               in takeBits (8 * B.length lbs) lbs
+
+takeBits :: Int -> B.ByteString -> Bits
+takeBits numBits lbs  = V.generate (fromIntegral numBits) (\n -> let (bb,b) = n `divMod` 8 in testBit (B.index lbs (fromIntegral bb)) (7-b))
 
-takeBits :: Int -> L.ByteString -> V.Vector Bool
-takeBits numBits lbs  = V.generate (fromIntegral numBits) (\n -> let (bb,b) = n `divMod` 8 in testBit (L.index lbs (fromIntegral bb)) (7-b))
+-- | asBits (5::Word8)
+-- | > [False,False,False,False,False,True,False,True]
+asBits :: FiniteBits a => a -> Bits
+asBits w = let s = finiteBitSize w in V.generate s (testBit w . (s-1-))
 
 -- |Convert a sequence of bits to the corresponding list of bytes
 asBytes :: Bits -> [Word8]

src/Data/Flat/Bits.hs

@@ -29,6 +29,15 @@ module Data.Flat.Decoder (
     dInt32,
     dInt64,
     dInt,
+    dBE8,
+    dBE16,
+    dBE32,
+    dBE64,
+    dBEBits8,
+    dBEBits16,
+    dBEBits32,
+    dBEBits64,
+    dropBits,
     ) where
 
 import Data.Flat.Decoder.Prim