Bryan O'Sullivan avatar Bryan O'Sullivan committed 9f6ffc1

First stab at lazy text support

Comments (0)

Files changed (7)

 import Data.String (IsString(..))
 
 import qualified Data.Text.Fusion as S
+import qualified Data.Text.Fusion.Internal as S
 import Data.Text.Fusion (Stream(..), Step(..), stream, reverseStream, unstream)
-import Data.Text.Internal (Text(..), empty, text)
+
+import Data.Text.Internal (Text(..), empty, text, textP)
 import qualified Prelude as P
 import Data.Text.Unsafe (iter, iter_, unsafeHead, unsafeTail)
 import Data.Text.UnsafeChar (unsafeChr)
 --
 -- This function is subject to array fusion.
 pack :: String -> Text
-pack str = (unstream (stream_list str))
-    where
-      stream_list s0 = S.Stream next s0 (P.length s0) -- total guess
-          where
-            next []     = S.Done
-            next (x:xs) = S.Yield x xs
+pack = unstream . S.streamList
 {-# INLINE [1] pack #-}
--- TODO: Has to do validation! -- No, it doesn't, the
 
 -- | /O(n)/ Convert a Text into a String.
 -- Subject to array fusion.
 unpack :: Text -> String
-unpack txt = (unstream_list (stream txt))
-    where
-      unstream_list (S.Stream next s0 _len) = unfold s0
-          where
-            unfold !s = case next s of
-                          S.Done       -> []
-                          S.Skip s'    -> unfold s'
-                          S.Yield x s' -> x : unfold s'
+unpack = S.unstreamList . stream
 {-# INLINE [1] unpack #-}
 
 -- | /O(1)/ Convert a character into a Text.
     S.last (stream t) = last t
   #-}
 
--- | Construct a 'Text' without invisibly pinning its byte array in
--- memory if its length has dwindled to zero.
-textP :: A.Array Word16 -> Int -> Int -> Text
-textP arr off len | len == 0  = empty
-                  | otherwise = text arr off len
-{-# INLINE textP #-}
-
 -- | /O(1)/ Returns all characters after the head of a 'Text', which
 -- must be non-empty.  Subject to array fusion.
 tail :: Text -> Text

Data/Text/Fusion.hs

-{-# LANGUAGE ExistentialQuantification, BangPatterns, MagicHash #-}
+{-# LANGUAGE BangPatterns, MagicHash #-}
 
 -- |
 -- Module      : Data.Text.Fusion
 
     -- * Creation and elimination
     , stream
+    , streamList
     , unstream
     , reverseStream
     , empty
 import Data.Text.Internal (Text(..))
 import Data.Text.UnsafeChar (unsafeChr, unsafeWrite, unsafeWriteRev)
 import qualified Data.Text.Array as A
+import Data.Text.Fusion.Internal
 import qualified Data.Text.Internal as I
 import qualified Data.Text.Encoding.Utf16 as U16
 import qualified Prelude as P
 
 default(Int)
 
-infixl 2 :!:
-data PairS a b = !a :!: !b
-
--- | Allow a function over a stream to switch between two states.
-data Switch = S1 | S2
-
-data Stream a =
-    forall s. Stream
-    (s -> Step s a)             -- stepper function
-    !s                          -- current state
-    {-# UNPACK #-}!Int          -- length hint
-
--- The length hint in a Stream has two roles.  If its value is zero,
--- we trust it, and treat the stream as empty.  Otherwise, we treat it
--- as a hint: it should usually be accurate, so we use it when
--- unstreaming to decide what size array to allocate.  However, the
--- unstreaming functions must be able to cope with the hint being too
--- small or too large.
---
--- The size hint tries to track the UTF-16 code points in a stream,
--- but often counts the number of characters instead.  It can easily
--- undercount if, for instance, a transformed stream contains astral
--- plane characters (those above 0x10000).
-
-data Step s a = Done
-              | Skip !s
-              | Yield !a !s
-
 -- | /O(n)/ Convert a 'Text' into a 'Stream Char'.
 stream :: Text -> Stream Char
 stream (Text arr off len) = Stream next off len
 unstream :: Stream Char -> Text
 unstream (Stream next0 s0 len)
     | len == 0 = I.empty
-    | otherwise = Text (P.fst a) 0 (P.snd a)
+    | otherwise = I.textP (P.fst a) 0 (P.snd a)
     where
       a = runST (A.unsafeNew len >>= (\arr -> loop arr len s0 0))
       loop arr !top !s !i
 {-# INLINE [0] unstream #-}
 {-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-}
 
--- | The empty stream.
-empty :: Stream Char
-empty = Stream next () 0
-    where next _ = Done
-{-# INLINE [0] empty #-}
-
--- | /O(n)/ Determines if two streams are equal.
-eq :: Ord a => Stream a -> Stream a -> Bool
-eq (Stream next1 s1 _) (Stream next2 s2 _) = cmp (next1 s1) (next2 s2)
-    where
-      cmp Done Done = True
-      cmp Done _    = False
-      cmp _    Done = False
-      cmp (Skip s1')     (Skip s2')     = cmp (next1 s1') (next2 s2')
-      cmp (Skip s1')     x2             = cmp (next1 s1') x2
-      cmp x1             (Skip s2')     = cmp x1          (next2 s2')
-      cmp (Yield x1 s1') (Yield x2 s2') = x1 == x2 &&
-                                          cmp (next1 s1') (next2 s2')
-{-# SPECIALISE eq :: Stream Char -> Stream Char -> Bool #-}
-
 streamError :: String -> String -> a
 streamError func msg = P.error $ "Data.Text.Fusion." ++ func ++ ": " ++ msg
 
 reverse :: Stream Char -> Text
 reverse (Stream next s len0)
     | len0 == 0 = I.empty
-    | otherwise = Text arr off' len'
+    | otherwise = I.textP arr off' len'
   where
     len0' = max len0 4
     (arr, (off', len')) = A.run2 (A.unsafeNew len0' >>= loop s (len0'-1) len0')

Data/Text/Internal.hs

       Text(..)
     -- * Construction
     , text
+    , textP
     -- * Code that must be here for accessibility
     , empty
     -- * Debugging
 empty = Text A.empty 0 0
 {-# INLINE [1] empty #-}
 
+-- | Construct a 'Text' without invisibly pinning its byte array in
+-- memory if its length has dwindled to zero.
+textP :: A.Array Word16 -> Int -> Int -> Text
+textP arr off len | len == 0  = empty
+                  | otherwise = text arr off len
+{-# INLINE textP #-}
+
 -- | A useful 'show'-like function for debugging purposes.
 showText :: Text -> String
 showText (Text arr off len) =

Data/Text/Lazy/Internal.hs

+{-# LANGUAGE BangPatterns, DeriveDataTypeable #-}
+-- |
+-- Module      : Data.Text.Lazy.Internal
+-- Copyright   : (c) Bryan O'Sullivan 2009
+--
+-- License     : BSD-style
+-- Maintainer  : bos@serpentine.com
+-- Stability   : experimental
+-- Portability : GHC
+-- 
+-- A module containing semi-public 'Text' internals. This exposes the
+-- 'Text' representation and low level construction functions.
+-- Modules which extend the 'Text' system may need to use this module.
+-- Regular users should not.
+module Data.Text.Lazy.Internal
+    (
+      Text(..)
+    , chunk
+    , foldrChunks
+    , foldlChunks
+    -- * Data type invariant and abstraction functions
+    , invariant
+    , checkInvariant
+    , showStructure
+    -- * Chunk allocation sizes
+    , defaultChunkSize
+    , smallChunkSize
+    , chunkOverhead
+    ) where
+
+import qualified Data.Text.Internal as T
+import qualified Data.Text as T
+import Data.Typeable (Typeable)
+import Data.Word (Word16)
+import Foreign.Storable (sizeOf)
+
+data Text = Empty
+          | Chunk {-# UNPACK #-} !T.Text Text
+            deriving (Typeable)
+
+-- | The data type invariant: Every 'Text' is either 'Empty' or
+-- consists of non-null 'T.Text's.  All functions must preserve this,
+-- and the QC properties must check this.
+invariant :: Text -> Bool
+invariant Empty                       = True
+invariant (Chunk (T.Text _ _ len) cs) = len > 0 && invariant cs
+
+showStructure :: Text -> String
+showStructure Empty           = "Empty"
+showStructure (Chunk t Empty) = "Chunk " ++ show t ++ " Empty"
+showStructure (Chunk t ts)    =
+    "Chunk " ++ show t ++ " (" ++ showStructure ts ++ ")"
+
+-- | In a form that checks the invariant lazily.
+checkInvariant :: Text -> Text
+checkInvariant Empty = Empty
+checkInvariant (Chunk c@(T.Text _ _ len) cs)
+    | len > 0   = Chunk c (checkInvariant cs)
+    | otherwise = error $ "Data.Text.Lazy: invariant violation: "
+               ++ showStructure (Chunk c cs)
+
+-- | Smart constructor for 'Chunk'. Guarantees the data type invariant.
+chunk :: T.Text -> Text -> Text
+{-# INLINE chunk #-}
+chunk t@(T.Text _ _ len) ts | len == 0 = ts
+                            | otherwise = Chunk t ts
+
+-- | Consume the chunks of a lazy 'Text' with a natural right fold.
+foldrChunks :: (T.Text -> a -> a) -> a -> Text -> a
+foldrChunks f z = go
+  where go Empty        = z
+        go (Chunk c cs) = f c (go cs)
+{-# INLINE foldrChunks #-}
+
+-- | Consume the chunks of a lazy 'Text' with a strict, tail-recursive,
+-- accumulating left fold.
+foldlChunks :: (a -> T.Text -> a) -> a -> Text -> a
+foldlChunks f z = go z
+  where go !a Empty        = a
+        go !a (Chunk c cs) = go (f a c) cs
+{-# INLINE foldlChunks #-}
+
+-- | Currently set to 32k, less the memory management overhead.
+defaultChunkSize :: Int
+defaultChunkSize = 32 * k - chunkOverhead
+   where k = 1024 `div` sizeOf (undefined :: Word16)
+
+-- | Currently set to 4k, less the memory management overhead.
+smallChunkSize :: Int
+smallChunkSize = 4 * k - chunkOverhead
+   where k = 1024 `div` sizeOf (undefined :: Word16)
+
+-- | The memory management overhead. Currently this is tuned for GHC only.
+chunkOverhead :: Int
+chunkOverhead = 2 * sizeOf (undefined :: Int)

tests/Properties.hs

 import Data.Word
 import qualified Data.ByteString as B
 import qualified Data.Text as T
-import Data.Text (pack,unpack)
+import qualified Data.Text.Lazy as TL
 import qualified Data.Text.Encoding as E
 import Control.Exception
 import qualified Data.Text.Fusion as S
-import Data.Text.Fusion (stream, unstream)
+import qualified Data.Text.Lazy.Fusion as SL
 import qualified Data.List as L
 import System.IO.Unsafe
 import Prelude hiding (catch)
     let types = e :: SomeException
     in trace ("*** Exception: " ++ show e) return onException
 
-prop_pack_unpack s     = (unpack . pack) s == s
-prop_stream_unstream t = (unstream . stream) t == t
-prop_reverse_stream t  = (S.reverse . S.reverseStream) t == t
-prop_singleton c       = [c] == (unpack . T.singleton) c
+prop_T_pack_unpack       = (T.unpack . T.pack) `eq` id
+prop_TL_pack_unpack      = (TL.unpack . TL.pack) `eq` id
+prop_T_stream_unstream   = (S.unstream . S.stream) `eq` id
+prop_TL_stream_unstream  = (SL.unstream . SL.stream) `eq` id
+prop_T_reverse_stream t  = (S.reverse . S.reverseStream) t == t
+prop_T_singleton c       = [c] == (T.unpack . T.singleton) c
 
-prop_ascii t           = E.decodeASCII (E.encodeUtf8 a) == a
+prop_T_ascii t           = E.decodeASCII (E.encodeUtf8 a) == a
     where a            = T.map (\c -> chr (ord c `mod` 128)) t
-prop_utf8              = (E.decodeUtf8 . E.encodeUtf8) `eq` id
-prop_utf16LE           = (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id
-prop_utf16BE           = (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id
-prop_utf32LE           = (E.decodeUtf32LE . E.encodeUtf32LE) `eq` id
-prop_utf32BE           = (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id
+prop_T_utf8              = (E.decodeUtf8 . E.encodeUtf8) `eq` id
+prop_T_utf16LE           = (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id
+prop_T_utf16BE           = (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id
+prop_T_utf32LE           = (E.decodeUtf32LE . E.encodeUtf32LE) `eq` id
+prop_T_utf32BE           = (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id
 
 -- Do two functions give the same answer?
 eq :: (Eq a) => (t -> a) -> (t -> a) -> t -> Bool
 eqP a b s w  = eq "orig" (a s) (b t) &&
                eq "head" (a sa) (b ta) &&
                eq "tail" (a sb) (b tb)
-    where t             = pack s
+    where t             = T.pack s
           (sa,sb)       = splitAt m s
           (ta,tb)       = T.splitAt m t
           l             = length s
                 (null sb || a sb == b tb)
     where (sa,sb)       = splitAt m s
           (ta,tb)       = T.splitAt m t
-          t             = pack s
+          t             = T.pack s
           l             = length s
           m | l == 0    = n
             | otherwise = n `mod` l
           n             = fromIntegral w
           s             = notEmpty e
 
-prop_cons x          = (x:)     `eqP` (unpack . T.cons x)
-prop_snoc x          = (++ [x]) `eqP` (unpack . (flip T.snoc) x)
-prop_append s        = (s++)    `eqP` (unpack . T.append (pack s))
-prop_appendS s       = (s++)    `eqP` (unpack . unstream . S.append (stream (pack s)) . stream)
-prop_uncons s        = uncons   `eqP` (fmap (second unpack) . T.uncons)
+prop_T_cons x          = (x:)     `eqP` (T.unpack . T.cons x)
+prop_T_snoc x          = (++ [x]) `eqP` (T.unpack . (flip T.snoc) x)
+prop_T_append s        = (s++)    `eqP` (T.unpack . T.append (T.pack s))
+prop_T_appendS s       = (s++)    `eqP` (T.unpack . S.unstream . S.append (S.stream (T.pack s)) . S.stream)
+prop_T_uncons s        = uncons   `eqP` (fmap (second T.unpack) . T.uncons)
     where uncons (x:xs) = Just (x,xs)
           uncons _      = Nothing
           types         = s :: String
-prop_head            = head   `eqEP` T.head
-prop_last            = last   `eqEP` T.last
-prop_lastS           = last   `eqEP` (S.last . stream)
-prop_tail            = tail   `eqEP` (unpack . T.tail)
-prop_tailS           = tail   `eqEP` (unpack . unstream . S.tail . stream)
-prop_init            = init   `eqEP` (unpack . T.init)
-prop_initS           = init   `eqEP` (unpack . unstream . S.init . stream)
-prop_null            = null   `eqP`  T.null
-prop_length          = length `eqP`  T.length
-prop_map f           = map f  `eqP`  (unpack . T.map f)
-prop_intercalate c   = L.intercalate c `eq` (unpack . T.intercalate (pack c) . map pack)
-prop_intersperse c   = L.intersperse c `eqP` (unpack . T.intersperse c)
-prop_transpose       = L.transpose `eq` (map unpack . T.transpose . map pack)
-prop_reverse         = L.reverse `eqP` (unpack . T.reverse)
-prop_reverse_short n = L.reverse `eqP` (unpack . S.reverse . shorten n . stream)
+prop_T_head            = head   `eqEP` T.head
+prop_T_last            = last   `eqEP` T.last
+prop_T_lastS           = last   `eqEP` (S.last . S.stream)
+prop_T_tail            = tail   `eqEP` (T.unpack . T.tail)
+prop_T_tailS           = tail   `eqEP` (T.unpack . S.unstream . S.tail . S.stream)
+prop_T_init            = init   `eqEP` (T.unpack . T.init)
+prop_T_initS           = init   `eqEP` (T.unpack . S.unstream . S.init . S.stream)
+prop_T_null            = null   `eqP`  T.null
+prop_T_length          = length `eqP`  T.length
+prop_T_map f           = map f  `eqP`  (T.unpack . T.map f)
+prop_T_intercalate c   = L.intercalate c `eq` (T.unpack . T.intercalate (T.pack c) . map T.pack)
+prop_T_intersperse c   = L.intersperse c `eqP` (T.unpack . T.intersperse c)
+prop_T_transpose       = L.transpose `eq` (map T.unpack . T.transpose . map T.pack)
+prop_T_reverse         = L.reverse `eqP` (T.unpack . T.reverse)
+prop_T_reverse_short n = L.reverse `eqP` (T.unpack . S.reverse . shorten n . S.stream)
 
-prop_foldl f z       = L.foldl f z  `eqP`  (T.foldl f z)
+prop_T_foldl f z       = L.foldl f z  `eqP`  (T.foldl f z)
     where types      = f :: Char -> Char -> Char
-prop_foldl' f z      = L.foldl' f z `eqP`  T.foldl' f z
+prop_T_foldl' f z      = L.foldl' f z `eqP`  T.foldl' f z
     where types      = f :: Char -> Char -> Char
-prop_foldl1 f        = L.foldl1 f   `eqEP` T.foldl1 f
-prop_foldl1' f       = L.foldl1' f  `eqEP` T.foldl1' f
-prop_foldr f z       = L.foldr f z  `eqP`  T.foldr f z
+prop_T_foldl1 f        = L.foldl1 f   `eqEP` T.foldl1 f
+prop_T_foldl1' f       = L.foldl1' f  `eqEP` T.foldl1' f
+prop_T_foldr f z       = L.foldr f z  `eqP`  T.foldr f z
     where types      = f :: Char -> Char -> Char
-prop_foldr1 f        = L.foldr1 f   `eqEP` T.foldr1 f
+prop_T_foldr1 f        = L.foldr1 f   `eqEP` T.foldr1 f
 
-prop_concat          = L.concat      `eq`   (unpack . T.concat . map pack)
-prop_concatMap f     = L.concatMap f `eqP`  (unpack . T.concatMap (pack . f))
-prop_any p           = L.any p       `eqP`  T.any p
-prop_all p           = L.all p       `eqP`  T.all p
-prop_maximum         = L.maximum     `eqEP` T.maximum
-prop_minimum         = L.minimum     `eqEP` T.minimum
+prop_T_concat          = L.concat      `eq`   (T.unpack . T.concat . map T.pack)
+prop_T_concatMap f     = L.concatMap f `eqP`  (T.unpack . T.concatMap (T.pack . f))
+prop_T_any p           = L.any p       `eqP`  T.any p
+prop_T_all p           = L.all p       `eqP`  T.all p
+prop_T_maximum         = L.maximum     `eqEP` T.maximum
+prop_T_minimum         = L.minimum     `eqEP` T.minimum
 
-prop_scanl f z       = L.scanl f z   `eqP`  (unpack . T.scanl f z)
-prop_scanl1 f        = L.scanl1 f    `eqP`  (unpack . T.scanl1 f)
-prop_scanr f z       = L.scanr f z   `eqP`  (unpack . T.scanr f z)
-prop_scanr1 f        = L.scanr1 f    `eqP`  (unpack . T.scanr1 f)
+prop_T_scanl f z       = L.scanl f z   `eqP`  (T.unpack . T.scanl f z)
+prop_T_scanl1 f        = L.scanl1 f    `eqP`  (T.unpack . T.scanl1 f)
+prop_T_scanr f z       = L.scanr f z   `eqP`  (T.unpack . T.scanr f z)
+prop_T_scanr1 f        = L.scanr1 f    `eqP`  (T.unpack . T.scanr1 f)
 
-prop_mapAccumL f z   = L.mapAccumL f z `eqP` (second unpack . T.mapAccumL f z)
+prop_T_mapAccumL f z   = L.mapAccumL f z `eqP` (second T.unpack . T.mapAccumL f z)
     where types = f :: Int -> Char -> (Int,Char)
-prop_mapAccumR f z   = L.mapAccumR f z `eqP` (second unpack . T.mapAccumR f z)
+prop_T_mapAccumR f z   = L.mapAccumR f z `eqP` (second T.unpack . T.mapAccumR f z)
     where types = f :: Int -> Char -> (Int,Char)
 
-prop_replicate n     = L.replicate n `eq`   (unpack . T.replicate n)
-prop_unfoldr n       = L.unfoldr f   `eq`   (unpack . T.unfoldr f)
+prop_T_replicate n     = L.replicate n `eq`   (T.unpack . T.replicate n)
+prop_T_unfoldr n       = L.unfoldr f   `eq`   (T.unpack . T.unfoldr f)
     where f c | fromEnum c * 100 > n = Nothing
               | otherwise            = Just (c, succ c)
 
-prop_unfoldrN n m    = (L.take n . L.unfoldr f) `eq` (unpack . T.unfoldrN n f)
+prop_T_unfoldrN n m    = (L.take n . L.unfoldr f) `eq` (T.unpack . T.unfoldrN n f)
     where f c | fromEnum c * 100 > m = Nothing
               | otherwise            = Just (c, succ c)
 
-unpack2 = unpack *** unpack
+unpack2 = T.unpack *** T.unpack
 
-prop_take n          = L.take n      `eqP` (unpack . T.take n)
-prop_drop n          = L.drop n      `eqP` (unpack . T.drop n)
-prop_takeWhile p     = L.takeWhile p `eqP` (unpack . T.takeWhile p)
-prop_takeWhileS p    = L.takeWhile p `eqP` (unpack . unstream . S.takeWhile p . stream)
-prop_dropWhile p     = L.dropWhile p `eqP` (unpack . T.dropWhile p)
-prop_dropWhileS p    = L.dropWhile p `eqP` (unpack . unstream . S.dropWhile p . stream)
-prop_splitAt n       = L.splitAt n   `eqP` (unpack2 . T.splitAt n)
-prop_span p          = L.span p      `eqP` (unpack2 . T.span p)
-prop_break p         = L.break p     `eqP` (unpack2 . T.break p)
-prop_group           = L.group       `eqP` (map unpack . T.group)
-prop_groupBy p       = L.groupBy p   `eqP` (map unpack . T.groupBy p)
-prop_inits           = L.inits       `eqP` (map unpack . T.inits)
-prop_tails           = L.tails       `eqP` (map unpack . T.tails)
+prop_T_take n          = L.take n      `eqP` (T.unpack . T.take n)
+prop_T_drop n          = L.drop n      `eqP` (T.unpack . T.drop n)
+prop_T_takeWhile p     = L.takeWhile p `eqP` (T.unpack . T.takeWhile p)
+prop_T_takeWhileS p    = L.takeWhile p `eqP` (T.unpack . S.unstream . S.takeWhile p . S.stream)
+prop_T_dropWhile p     = L.dropWhile p `eqP` (T.unpack . T.dropWhile p)
+prop_T_dropWhileS p    = L.dropWhile p `eqP` (T.unpack . S.unstream . S.dropWhile p . S.stream)
+prop_T_splitAt n       = L.splitAt n   `eqP` (unpack2 . T.splitAt n)
+prop_T_span p          = L.span p      `eqP` (unpack2 . T.span p)
+prop_T_break p         = L.break p     `eqP` (unpack2 . T.break p)
+prop_T_group           = L.group       `eqP` (map T.unpack . T.group)
+prop_T_groupBy p       = L.groupBy p   `eqP` (map T.unpack . T.groupBy p)
+prop_T_inits           = L.inits       `eqP` (map T.unpack . T.inits)
+prop_T_tails           = L.tails       `eqP` (map T.unpack . T.tails)
 
-prop_split_i c       = id `eq` (T.intercalate (T.singleton c) . T.split c)
+prop_T_split_i c       = id `eq` (T.intercalate (T.singleton c) . T.split c)
 
-prop_splitWith p     = splitWith p `eqP` (map unpack . T.splitWith p)
+prop_T_splitWith p     = splitWith p `eqP` (map T.unpack . T.splitWith p)
 
 splitWith _ "" =  []
 splitWith p s  = if null s'
                  else l : splitWith p (tail s')
     where (l, s') = break p s
 
-prop_breakSubstring_isInfixOf s l
+prop_T_breakSubstring_isInfixOf s l
                      = T.isInfixOf s l ==
                        T.null s || (not . T.null . snd $ T.breakSubstring s l)
-prop_breakSubstringC c
+prop_T_breakSubstringC c
                      = L.break (==c) `eqP`
                        (unpack2 . T.breakSubstring (T.singleton c))
 
-prop_lines           = L.lines       `eqP` (map unpack . T.lines)
+prop_T_lines           = L.lines       `eqP` (map T.unpack . T.lines)
 {-
-prop_lines'          = lines'        `eqP` (map unpack . T.lines')
+prop_T_lines'          = lines'        `eqP` (map T.unpack . T.lines')
     where lines' "" =  []
           lines' s =  let (l, s') = break eol s
                       in  l : case s' of
                                 (_:s'') -> lines' s''
           eol c = c == '\r' || c == '\n'
 -}
-prop_words           = L.words       `eqP` (map unpack . T.words)
-prop_unlines         = L.unlines     `eq`  (unpack . T.unlines . map pack)
-prop_unwords         = L.unwords     `eq`  (unpack . T.unwords . map pack)
+prop_T_words           = L.words       `eqP` (map T.unpack . T.words)
+prop_T_unlines         = L.unlines     `eq`  (T.unpack . T.unlines . map T.pack)
+prop_T_unwords         = L.unwords     `eq`  (T.unpack . T.unwords . map T.pack)
 
-prop_isPrefixOf s    = L.isPrefixOf s`eqP` T.isPrefixOf (pack s)
-prop_isPrefixOfS s   = L.isPrefixOf s`eqP` (S.isPrefixOf (stream $ pack s) . stream)
-prop_isSuffixOf s    = L.isSuffixOf s`eqP` T.isSuffixOf (pack s)
-prop_isInfixOf s     = L.isInfixOf s `eqP` T.isInfixOf (pack s)
+prop_T_isPrefixOf s    = L.isPrefixOf s`eqP` T.isPrefixOf (T.pack s)
+prop_T_isPrefixOfS s   = L.isPrefixOf s`eqP` (S.isPrefixOf (S.stream $ T.pack s) . S.stream)
+prop_T_isSuffixOf s    = L.isSuffixOf s`eqP` T.isSuffixOf (T.pack s)
+prop_T_isInfixOf s     = L.isInfixOf s `eqP` T.isInfixOf (T.pack s)
 
-prop_elem c          = L.elem c      `eqP` T.elem c
-prop_filter p        = L.filter p    `eqP` (unpack . T.filter p)
-prop_find p          = L.find p      `eqP` T.find p
-prop_partition p     = L.partition p `eqP` (unpack2 . T.partition p)
+prop_T_elem c          = L.elem c      `eqP` T.elem c
+prop_T_filter p        = L.filter p    `eqP` (T.unpack . T.filter p)
+prop_T_find p          = L.find p      `eqP` T.find p
+prop_T_partition p     = L.partition p `eqP` (unpack2 . T.partition p)
 
-prop_index x s       = x < L.length s && x >= 0 ==>
-                       (L.!!) s x == T.index (pack s) x
-prop_findIndex p     = L.findIndex p `eqP` T.findIndex p
-prop_findIndices p   = L.findIndices p`eqP` T.findIndices p
-prop_elemIndex c     = L.elemIndex c `eqP` T.elemIndex c
-prop_elemIndices c   = L.elemIndices c`eqP` T.elemIndices c
-prop_count c         = (L.length . L.elemIndices c) `eqP` T.count c
-prop_zipWith c s     = L.zipWith c s `eqP` (unpack . T.zipWith c (pack s))
+prop_T_index x s       = x < L.length s && x >= 0 ==>
+                       (L.!!) s x == T.index (T.pack s) x
+prop_T_findIndex p     = L.findIndex p `eqP` T.findIndex p
+prop_T_findIndices p   = L.findIndices p`eqP` T.findIndices p
+prop_T_elemIndex c     = L.elemIndex c `eqP` T.elemIndex c
+prop_T_elemIndices c   = L.elemIndices c`eqP` T.elemIndices c
+prop_T_count c         = (L.length . L.elemIndices c) `eqP` T.count c
+prop_T_zipWith c s     = L.zipWith c s `eqP` (T.unpack . T.zipWith c (T.pack s))
 
 -- Make a stream appear shorter than it really is, to ensure that
 -- functions that consume inaccurately sized streams behave
 
 tests :: [(String, Int -> IO (Bool, Int))]
 tests = [
-  ("prop_pack_unpack", mytest prop_pack_unpack),
-  ("prop_stream_unstream", mytest prop_stream_unstream),
-  ("prop_reverse_stream", mytest prop_reverse_stream),
-  ("prop_singleton", mytest prop_singleton),
+  ("prop_T_pack_unpack", mytest prop_T_pack_unpack),
+  ("prop_TL_pack_unpack", mytest prop_TL_pack_unpack),
+  ("prop_T_stream_unstream", mytest prop_T_stream_unstream),
+  ("prop_TL_stream_unstream", mytest prop_TL_stream_unstream),
+  ("prop_T_reverse_stream", mytest prop_T_reverse_stream),
+  ("prop_T_singleton", mytest prop_T_singleton),
 
-  ("prop_ascii", mytest prop_ascii),
-  ("prop_utf8", mytest prop_utf8),
-  ("prop_utf16LE", mytest prop_utf16LE),
-  ("prop_utf16BE", mytest prop_utf16BE),
-  ("prop_utf32LE", mytest prop_utf32LE),
-  ("prop_utf32BE", mytest prop_utf32BE),
+  ("prop_T_ascii", mytest prop_T_ascii),
+  ("prop_T_utf8", mytest prop_T_utf8),
+  ("prop_T_utf16LE", mytest prop_T_utf16LE),
+  ("prop_T_utf16BE", mytest prop_T_utf16BE),
+  ("prop_T_utf32LE", mytest prop_T_utf32LE),
+  ("prop_T_utf32BE", mytest prop_T_utf32BE),
 
-  ("prop_cons", mytest prop_cons),
-  ("prop_snoc", mytest prop_snoc),
-  ("prop_append", mytest prop_append),
-  ("prop_appendS", mytest prop_appendS),
-  ("prop_uncons", mytest prop_uncons),
-  ("prop_head", mytest prop_head),
-  ("prop_last", mytest prop_last),
-  ("prop_lastS", mytest prop_lastS),
-  ("prop_tail", mytest prop_tail),
-  ("prop_tailS", mytest prop_tailS),
-  ("prop_init", mytest prop_init),
-  ("prop_initS", mytest prop_initS),
-  ("prop_null", mytest prop_null),
-  ("prop_length", mytest prop_length),
+  ("prop_T_cons", mytest prop_T_cons),
+  ("prop_T_snoc", mytest prop_T_snoc),
+  ("prop_T_append", mytest prop_T_append),
+  ("prop_T_appendS", mytest prop_T_appendS),
+  ("prop_T_uncons", mytest prop_T_uncons),
+  ("prop_T_head", mytest prop_T_head),
+  ("prop_T_last", mytest prop_T_last),
+  ("prop_T_lastS", mytest prop_T_lastS),
+  ("prop_T_tail", mytest prop_T_tail),
+  ("prop_T_tailS", mytest prop_T_tailS),
+  ("prop_T_init", mytest prop_T_init),
+  ("prop_T_initS", mytest prop_T_initS),
+  ("prop_T_null", mytest prop_T_null),
+  ("prop_T_length", mytest prop_T_length),
 
-  ("prop_map", mytest prop_map),
-  ("prop_intercalate", mytest prop_intercalate),
-  ("prop_intersperse", mytest prop_intersperse),
-  ("prop_transpose", mytest prop_transpose),
-  ("prop_reverse", mytest prop_reverse),
-  ("prop_reverse_short", mytest prop_reverse_short),
+  ("prop_T_map", mytest prop_T_map),
+  ("prop_T_intercalate", mytest prop_T_intercalate),
+  ("prop_T_intersperse", mytest prop_T_intersperse),
+  ("prop_T_transpose", mytest prop_T_transpose),
+  ("prop_T_reverse", mytest prop_T_reverse),
+  ("prop_T_reverse_short", mytest prop_T_reverse_short),
 
-  ("prop_foldl", mytest prop_foldl),
-  ("prop_foldl'", mytest prop_foldl'),
-  ("prop_foldl1", mytest prop_foldl1),
-  ("prop_foldl1'", mytest prop_foldl1'),
-  ("prop_foldr", mytest prop_foldr),
-  ("prop_foldr1", mytest prop_foldr1),
+  ("prop_T_foldl", mytest prop_T_foldl),
+  ("prop_T_foldl'", mytest prop_T_foldl'),
+  ("prop_T_foldl1", mytest prop_T_foldl1),
+  ("prop_T_foldl1'", mytest prop_T_foldl1'),
+  ("prop_T_foldr", mytest prop_T_foldr),
+  ("prop_T_foldr1", mytest prop_T_foldr1),
 
-  ("prop_concat", mytest prop_concat),
-  ("prop_concatMap", mytest prop_concatMap),
-  ("prop_any", mytest prop_any),
-  ("prop_all", mytest prop_all),
-  ("prop_maximum", mytest prop_maximum),
-  ("prop_minimum", mytest prop_minimum),
+  ("prop_T_concat", mytest prop_T_concat),
+  ("prop_T_concatMap", mytest prop_T_concatMap),
+  ("prop_T_any", mytest prop_T_any),
+  ("prop_T_all", mytest prop_T_all),
+  ("prop_T_maximum", mytest prop_T_maximum),
+  ("prop_T_minimum", mytest prop_T_minimum),
 
-  ("prop_scanl", mytest prop_scanl),
-  ("prop_scanl1", mytest prop_scanl1),
-  ("prop_scanr", mytest prop_scanr),
-  ("prop_scanr1", mytest prop_scanr1),
+  ("prop_T_scanl", mytest prop_T_scanl),
+  ("prop_T_scanl1", mytest prop_T_scanl1),
+  ("prop_T_scanr", mytest prop_T_scanr),
+  ("prop_T_scanr1", mytest prop_T_scanr1),
 
-  ("prop_mapAccumL", mytest prop_mapAccumL),
-  ("prop_mapAccumR", mytest prop_mapAccumR),
+  ("prop_T_mapAccumL", mytest prop_T_mapAccumL),
+  ("prop_T_mapAccumR", mytest prop_T_mapAccumR),
 
-  ("prop_replicate", mytest prop_replicate),
-  ("prop_unfoldr", mytest prop_unfoldr),
-  ("prop_unfoldrN", mytest prop_unfoldrN),
+  ("prop_T_replicate", mytest prop_T_replicate),
+  ("prop_T_unfoldr", mytest prop_T_unfoldr),
+  ("prop_T_unfoldrN", mytest prop_T_unfoldrN),
 
-  ("prop_take", mytest prop_take),
-  ("prop_drop", mytest prop_drop),
-  ("prop_takeWhile", mytest prop_takeWhile),
-  ("prop_takeWhileS", mytest prop_takeWhileS),
-  ("prop_dropWhile", mytest prop_dropWhile),
-  ("prop_dropWhileS", mytest prop_dropWhileS),
-  ("prop_splitAt", mytest prop_splitAt),
-  ("prop_span", mytest prop_span),
-  ("prop_break", mytest prop_break),
-  ("prop_group", mytest prop_group),
-  ("prop_groupBy", mytest prop_groupBy),
-  ("prop_inits", mytest prop_inits),
-  ("prop_tails", mytest prop_tails),
+  ("prop_T_take", mytest prop_T_take),
+  ("prop_T_drop", mytest prop_T_drop),
+  ("prop_T_takeWhile", mytest prop_T_takeWhile),
+  ("prop_T_takeWhileS", mytest prop_T_takeWhileS),
+  ("prop_T_dropWhile", mytest prop_T_dropWhile),
+  ("prop_T_dropWhileS", mytest prop_T_dropWhileS),
+  ("prop_T_splitAt", mytest prop_T_splitAt),
+  ("prop_T_span", mytest prop_T_span),
+  ("prop_T_break", mytest prop_T_break),
+  ("prop_T_group", mytest prop_T_group),
+  ("prop_T_groupBy", mytest prop_T_groupBy),
+  ("prop_T_inits", mytest prop_T_inits),
+  ("prop_T_tails", mytest prop_T_tails),
 
-  ("prop_split_i", mytest prop_split_i),
-  ("prop_splitWith", mytest prop_splitWith),
-  ("prop_breakSubstringC", mytest prop_breakSubstringC),
-  ("prop_breakSubstring_isInfixOf", mytest prop_breakSubstring_isInfixOf),
+  ("prop_T_split_i", mytest prop_T_split_i),
+  ("prop_T_splitWith", mytest prop_T_splitWith),
+  ("prop_T_breakSubstringC", mytest prop_T_breakSubstringC),
+  ("prop_T_breakSubstring_isInfixOf", mytest prop_T_breakSubstring_isInfixOf),
 
-  ("prop_lines", mytest prop_lines),
---("prop_lines'", mytest prop_lines'),
-  ("prop_words", mytest prop_words),
-  ("prop_unlines", mytest prop_unlines),
-  ("prop_unwords", mytest prop_unwords),
+  ("prop_T_lines", mytest prop_T_lines),
+--("prop_T_lines'", mytest prop_T_lines'),
+  ("prop_T_words", mytest prop_T_words),
+  ("prop_T_unlines", mytest prop_T_unlines),
+  ("prop_T_unwords", mytest prop_T_unwords),
 
-  ("prop_isPrefixOf", mytest prop_isPrefixOf),
-  ("prop_isPrefixOfS", mytest prop_isPrefixOfS),
-  ("prop_isSuffixOf", mytest prop_isSuffixOf),
-  ("prop_isInfixOf", mytest prop_isInfixOf),
+  ("prop_T_isPrefixOf", mytest prop_T_isPrefixOf),
+  ("prop_T_isPrefixOfS", mytest prop_T_isPrefixOfS),
+  ("prop_T_isSuffixOf", mytest prop_T_isSuffixOf),
+  ("prop_T_isInfixOf", mytest prop_T_isInfixOf),
 
-  ("prop_elem", mytest prop_elem),
-  ("prop_filter", mytest prop_filter),
-  ("prop_find", mytest prop_find),
-  ("prop_partition", mytest prop_partition),
+  ("prop_T_elem", mytest prop_T_elem),
+  ("prop_T_filter", mytest prop_T_filter),
+  ("prop_T_find", mytest prop_T_find),
+  ("prop_T_partition", mytest prop_T_partition),
 
-  ("prop_index", mytest prop_index),
-  ("prop_findIndex", mytest prop_findIndex),
-  ("prop_findIndices", mytest prop_findIndices),
-  ("prop_elemIndex", mytest prop_elemIndex),
-  ("prop_elemIndices", mytest prop_elemIndices),
-  ("prop_count", mytest prop_count),
-  ("prop_zipWith", mytest prop_zipWith)
+  ("prop_T_index", mytest prop_T_index),
+  ("prop_T_findIndex", mytest prop_T_findIndex),
+  ("prop_T_findIndices", mytest prop_T_findIndices),
+  ("prop_T_elemIndex", mytest prop_T_elemIndex),
+  ("prop_T_elemIndices", mytest prop_T_elemIndices),
+  ("prop_T_count", mytest prop_T_count),
+  ("prop_T_zipWith", mytest prop_T_zipWith)
   ]

tests/QuickCheckUtils.hs

 import Data.List
 import Data.Word
 import qualified Data.Text as T
+import qualified Data.Text.Lazy as TL
 import System.IO
 import System.Random
 import Test.QuickCheck
     arbitrary     = T.pack `fmap` arbitrary
     coarbitrary s = coarbitrary (T.unpack s)
 
+instance Arbitrary TL.Text where
+    arbitrary     = TL.pack `fmap` arbitrary
+    coarbitrary s = coarbitrary (TL.unpack s)
+
 newtype NotEmpty a = NotEmpty { notEmpty :: a }
     deriving (Eq, Ord, Show)
 
     arbitrary   = (fmap T.pack) `fmap` arbitrary
     coarbitrary = coarbitrary . notEmpty
 
+instance Arbitrary (NotEmpty TL.Text) where
+    arbitrary   = (fmap TL.pack) `fmap` arbitrary
+    coarbitrary = coarbitrary . notEmpty
+
 instance Arbitrary (NotEmpty B.ByteString) where
     arbitrary   = (fmap B.pack) `fmap` arbitrary
     coarbitrary = coarbitrary . notEmpty
     Data.Text.Encoding.Fusion
     Data.Text.Foreign
     Data.Text.Fusion
+    Data.Text.Lazy
+    Data.Text.Lazy.Fusion
   other-modules:
     Data.Text.Array
     Data.Text.Internal
+    Data.Text.Fusion.Internal
+    Data.Text.Lazy.Internal
     Data.Text.Unsafe
     Data.Text.UnsafeChar
     Data.Text.Encoding.Utf8
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.