Bryan O'Sullivan avatar Bryan O'Sullivan committed 3e52938

encodeUtf8_2: a C-based encoding function

Not surprisingly, this is a lot faster than encodeUtf8_1 and the
Builder-based rewrite under almost all circumstances.

It's slower on tiny inputs (20%), but roughly twice as fast as
encodeUtf8_1 on longer inputs.

Comments (0)

Files changed (3)

Data/Text/Encoding.hs

     , encodeUtf8
     , encodeUtf8_0
     , encodeUtf8_1
+    , encodeUtf8_2
     , encodeUtf16LE
     , encodeUtf16BE
     , encodeUtf32LE
 import Data.Text.Internal.Private (runText)
 import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)
 import Data.Word (Word8, Word32)
-import Foreign.C.Types (CSize)
+import Foreign.C.Types (CSize(..))
 import Foreign.ForeignPtr (touchForeignPtr, withForeignPtr)
 import Foreign.Marshal.Utils (with)
 import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)
 import Foreign.Storable (Storable, peek, poke)
-import GHC.Base (MutableByteArray#)
+import GHC.Base (ByteArray#, MutableByteArray#)
 import qualified Data.Text.Array as A
 import qualified Data.Text.Internal.Encoding.Fusion as E
 import qualified Data.Text.Internal.Encoding.Utf16 as U16
                     hot (n+2) (ptr `plusPtr` 4)
     hot n0 ptr0
 
+encodeUtf8_2 :: Text -> ByteString
+encodeUtf8_2 (Text arr off len)
+  | len == 0  = B.empty
+  | otherwise = unsafeDupablePerformIO $ do
+  fp <- mallocByteString (len*4)
+  withForeignPtr fp $ \ptr ->
+    with ptr $ \destPtr -> do
+      c_encode_utf8 destPtr (A.aBA arr) (fromIntegral off) (fromIntegral len)
+      newDest <- peek destPtr
+      return (PS fp 0 (newDest `minusPtr` ptr))
+
 -- | Decode text from little endian UTF-16 encoding.
 decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text
 decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)
 
 foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1
     :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO ()
+
+foreign import ccall unsafe "_hs_text_encode_utf8" c_encode_utf8
+    :: Ptr (Ptr Word8) -> ByteArray# -> CSize -> CSize -> IO ()

benchmarks/haskell/Benchmarks/Pure.hs

         , bgroup "encode"
             [ benchT   $ nf T.encodeUtf8 ta
             , benchT1  $ nf T.encodeUtf8_1 ta
+            , benchT2  $ nf T.encodeUtf8_2 ta
             , benchTL  $ nf TL.encodeUtf8 tla
             , benchBS  $ nf BS.pack sa
             , benchBSL $ nf BL.pack sa
     benchS   = bench ("String+" ++ kind)
     benchT   = bench ("Text+" ++ kind)
     benchT1  = bench ("Text1+" ++ kind)
+    benchT2  = bench ("Text2+" ++ kind)
     benchTL  = bench ("LazyText+" ++ kind)
     benchBS  = bench ("ByteString+" ++ kind)
     benchBSL = bench ("LazyByteString+" ++ kind)
     ret -= 1;
   return ret;
 }
+
+void
+_hs_text_encode_utf8(uint8_t **destp, const uint16_t *src, size_t srcoff,
+		     size_t srclen)
+{
+  const uint16_t *srcend;
+  uint8_t *dest = *destp;
+
+  src += srcoff;
+  srcend = src + srclen;
+
+  while (src < srcend) {
+    uint16_t w = *src++;
+
+    if (w <= 0x7F)
+      *dest++ = w;
+    else if (w <= 0x7FF) {
+      *dest++ = (w >> 6) | 0xC0;
+      *dest++ = (w & 0x3f) | 0x80;
+    }
+    else if (w < 0xD800 || w > 0xDBFF) {
+      *dest++ = (w >> 12) | 0xE0;
+      *dest++ = ((w >> 6) & 0x3F) | 0x80;
+      *dest++ = (w & 0x3F) | 0x80;
+    } else {
+      uint32_t c = ((((uint32_t) w) - 0xD800) << 10) +
+	(((uint32_t) *src++) - 0xDC00) + 0x10000;
+      *dest++ = (c >> 18) | 0xE0;
+      *dest++ = ((c >> 12) & 0x3F) | 0x80;
+      *dest++ = ((c >> 6) & 0x3F) | 0x80;
+      *dest++ = (c & 0x3F) | 0x80;
+    }
+  }
+
+  *destp = dest;
+}
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.