-- |
-- Module      : Basement.UTF8.Helper
-- License     : BSD-style
-- Maintainer  : Foundation
--
-- Some low level helpers to use UTF8
--
-- Most helpers are lowlevel and unsafe, don't use
-- directly.
{-# LANGUAGE BangPatterns               #-}
{-# LANGUAGE MagicHash                  #-}
{-# LANGUAGE NoImplicitPrelude          #-}
{-# LANGUAGE TypeFamilies               #-}
{-# LANGUAGE FlexibleContexts           #-}
{-# LANGUAGE CPP                        #-}
module Basement.UTF8.Helper
    where

import           Basement.Compat.Base
import           Basement.Compat.Primitive
import           Basement.Types.OffsetSize
import           Basement.UTF8.Types
import           Basement.Bits
import           GHC.Prim
import           GHC.Types
import           GHC.Word

-- mask an UTF8 continuation byte (stripping the leading 10 and returning 6 valid bits)
maskContinuation# :: Word# -> Word#
maskContinuation# :: Word# -> Word#
maskContinuation# Word#
v = Word# -> Word# -> Word#
and# Word#
v Word#
0x3f##
{-# INLINE maskContinuation# #-}

-- mask a UTF8 header for 2 bytes encoding (110xxxxx and 5 valid bits)
maskHeader2# :: Word# -> Word#
maskHeader2# :: Word# -> Word#
maskHeader2# Word#
h = Word# -> Word# -> Word#
and# Word#
h Word#
0x1f##
{-# INLINE maskHeader2# #-}

-- mask a UTF8 header for 3 bytes encoding (1110xxxx and 4 valid bits)
maskHeader3# :: Word# -> Word#
maskHeader3# :: Word# -> Word#
maskHeader3# Word#
h = Word# -> Word# -> Word#
and# Word#
h Word#
0xf##
{-# INLINE maskHeader3# #-}

-- mask a UTF8 header for 4 bytes encoding (11110xxx and 3 valid bits)
maskHeader4# :: Word# -> Word#
maskHeader4# :: Word# -> Word#
maskHeader4# Word#
h = Word# -> Word# -> Word#
and# Word#
h Word#
0x7##
{-# INLINE maskHeader4# #-}

or3# :: Word# -> Word# -> Word# -> Word#
or3# :: Word# -> Word# -> Word# -> Word#
or3# Word#
a Word#
b Word#
c = Word# -> Word# -> Word#
or# Word#
a (Word# -> Word# -> Word#
or# Word#
b Word#
c)
{-# INLINE or3# #-}

or4# :: Word# -> Word# -> Word# -> Word# -> Word#
or4# :: Word# -> Word# -> Word# -> Word# -> Word#
or4# Word#
a Word#
b Word#
c Word#
d = Word# -> Word# -> Word#
or# (Word# -> Word# -> Word#
or# Word#
a Word#
b) (Word# -> Word# -> Word#
or# Word#
c Word#
d)
{-# INLINE or4# #-}

toChar# :: Word# -> Char
toChar# :: Word# -> Char
toChar# Word#
w = Char# -> Char
C# (Int# -> Char#
chr# (Word# -> Int#
word2Int# Word#
w))
{-# INLINE toChar# #-}

toChar1 :: StepASCII -> Char
toChar1 :: StepASCII -> Char
toChar1 (StepASCII (W8# Word8#
w)) = Char# -> Char
C# (Word8# -> Char#
word8ToChar# Word8#
w)

toChar2 :: StepASCII -> Word8 -> Char
toChar2 :: StepASCII -> Word8 -> Char
toChar2 (StepASCII (W8# Word8#
b1)) (W8# Word8#
b2) =
    Word# -> Char
toChar# (Word# -> Word# -> Word#
or# (Word# -> Int# -> Word#
uncheckedShiftL# (Word# -> Word#
maskHeader2# Word#
w1) Int#
6#) (Word# -> Word#
maskContinuation# Word#
w2))
  where
    w1 :: Word#
w1 = Word8# -> Word#
word8ToWord# Word8#
b1
    w2 :: Word#
w2 = Word8# -> Word#
word8ToWord# Word8#
b2

toChar3 :: StepASCII -> Word8 -> Word8 -> Char
toChar3 :: StepASCII -> Word8 -> Word8 -> Char
toChar3 (StepASCII (W8# Word8#
b1)) (W8# Word8#
b2) (W8# Word8#
b3) =
    Word# -> Char
toChar# (Word# -> Word# -> Word# -> Word#
or3# (Word# -> Int# -> Word#
uncheckedShiftL# (Word# -> Word#
maskHeader3# Word#
w1) Int#
12#)
                  (Word# -> Int# -> Word#
uncheckedShiftL# (Word# -> Word#
maskContinuation# Word#
w2) Int#
6#)
                  (Word# -> Word#
maskContinuation# Word#
w3)
            )
  where
    w1 :: Word#
w1 = Word8# -> Word#
word8ToWord# Word8#
b1
    w2 :: Word#
w2 = Word8# -> Word#
word8ToWord# Word8#
b2
    w3 :: Word#
w3 = Word8# -> Word#
word8ToWord# Word8#
b3

toChar4 :: StepASCII -> Word8 -> Word8 -> Word8 -> Char
toChar4 :: StepASCII -> Word8 -> Word8 -> Word8 -> Char
toChar4 (StepASCII (W8# Word8#
b1)) (W8# Word8#
b2) (W8# Word8#
b3) (W8# Word8#
b4) =
    Word# -> Char
toChar# (Word# -> Word# -> Word# -> Word# -> Word#
or4# (Word# -> Int# -> Word#
uncheckedShiftL# (Word# -> Word#
maskHeader4# Word#
w1) Int#
18#)
                  (Word# -> Int# -> Word#
uncheckedShiftL# (Word# -> Word#
maskContinuation# Word#
w2) Int#
12#)
                  (Word# -> Int# -> Word#
uncheckedShiftL# (Word# -> Word#
maskContinuation# Word#
w3) Int#
6#)
                  (Word# -> Word#
maskContinuation# Word#
w4)
            )
  where
    w1 :: Word#
w1 = Word8# -> Word#
word8ToWord# Word8#
b1
    w2 :: Word#
w2 = Word8# -> Word#
word8ToWord# Word8#
b2
    w3 :: Word#
w3 = Word8# -> Word#
word8ToWord# Word8#
b3
    w4 :: Word#
w4 = Word8# -> Word#
word8ToWord# Word8#
b4

-- | Different way to encode a Character in UTF8 represented as an ADT
data UTF8Char =
      UTF8_1 {-# UNPACK #-} !Word8
    | UTF8_2 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8
    | UTF8_3 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8
    | UTF8_4 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8

-- | Transform a Unicode code point 'Char' into
--
-- note that we expect here a valid unicode code point in the *allowed* range.
-- bits will be lost if going above 0x10ffff
asUTF8Char :: Char -> UTF8Char
asUTF8Char :: Char -> UTF8Char
asUTF8Char !(C# Char#
c)
  | Int# -> Bool
bool# (Word# -> Word# -> Int#
ltWord# Word#
x Word#
0x80##   ) = UTF8Char
encode1
  | Int# -> Bool
bool# (Word# -> Word# -> Int#
ltWord# Word#
x Word#
0x800##  ) = UTF8Char
encode2
  | Int# -> Bool
bool# (Word# -> Word# -> Int#
ltWord# Word#
x Word#
0x10000##) = UTF8Char
encode3
  | Bool
otherwise                   = UTF8Char
encode4
    where
      !x :: Word#
x = Int# -> Word#
int2Word# (Char# -> Int#
ord# Char#
c)

      encode1 :: UTF8Char
encode1 = Word8 -> UTF8Char
UTF8_1 (Word8# -> Word8
W8# (Word# -> Word8#
wordToWord8# Word#
x))
      encode2 :: UTF8Char
encode2 =
          let !x1 :: Word8
x1 = Word8# -> Word8
W8# (Word# -> Word8#
wordToWord8# (Word# -> Word# -> Word#
or# (Word# -> Int# -> Word#
uncheckedShiftRL# Word#
x Int#
6#) Word#
0xc0##))
              !x2 :: Word8
x2 = Word# -> Word8
toContinuation Word#
x
           in Word8 -> Word8 -> UTF8Char
UTF8_2 Word8
x1 Word8
x2
      encode3 :: UTF8Char
encode3 =
          let !x1 :: Word8
x1 = Word8# -> Word8
W8# (Word# -> Word8#
wordToWord8# (Word# -> Word# -> Word#
or# (Word# -> Int# -> Word#
uncheckedShiftRL# Word#
x Int#
12#) Word#
0xe0##))
              !x2 :: Word8
x2 = Word# -> Word8
toContinuation (Word# -> Int# -> Word#
uncheckedShiftRL# Word#
x Int#
6#)
              !x3 :: Word8
x3 = Word# -> Word8
toContinuation Word#
x
           in Word8 -> Word8 -> Word8 -> UTF8Char
UTF8_3 Word8
x1 Word8
x2 Word8
x3
      encode4 :: UTF8Char
encode4 =
          let !x1 :: Word8
x1 = Word8# -> Word8
W8# (Word# -> Word8#
wordToWord8# (Word# -> Word# -> Word#
or# (Word# -> Int# -> Word#
uncheckedShiftRL# Word#
x Int#
18#) Word#
0xf0##))
              !x2 :: Word8
x2 = Word# -> Word8
toContinuation (Word# -> Int# -> Word#
uncheckedShiftRL# Word#
x Int#
12#)
              !x3 :: Word8
x3 = Word# -> Word8
toContinuation (Word# -> Int# -> Word#
uncheckedShiftRL# Word#
x Int#
6#)
              !x4 :: Word8
x4 = Word# -> Word8
toContinuation Word#
x
           in Word8 -> Word8 -> Word8 -> Word8 -> UTF8Char
UTF8_4 Word8
x1 Word8
x2 Word8
x3 Word8
x4

      toContinuation :: Word# -> Word8
      toContinuation :: Word# -> Word8
toContinuation Word#
w = Word8# -> Word8
W8# (Word# -> Word8#
wordToWord8# (Word# -> Word# -> Word#
or# (Word# -> Word# -> Word#
and# Word#
w Word#
0x3f##) Word#
0x80##))
      {-# INLINE toContinuation #-}

-- given the encoding of UTF8 Char, get the number of bytes of this sequence
numBytes :: UTF8Char -> CountOf Word8
numBytes :: UTF8Char -> CountOf Word8
numBytes UTF8_1{} = forall ty. Int -> CountOf ty
CountOf Int
1
numBytes UTF8_2{} = forall ty. Int -> CountOf ty
CountOf Int
2
numBytes UTF8_3{} = forall ty. Int -> CountOf ty
CountOf Int
3
numBytes UTF8_4{} = forall ty. Int -> CountOf ty
CountOf Int
4

-- given the leading byte of a utf8 sequence, get the number of bytes of this sequence
skipNextHeaderValue :: Word8 -> CountOf Word8
skipNextHeaderValue :: Word8 -> CountOf Word8
skipNextHeaderValue !Word8
x
    | Word8
x forall a. Ord a => a -> a -> Bool
< Word8
0xC0  = forall ty. Int -> CountOf ty
CountOf Int
1 -- 0b11000000
    | Word8
x forall a. Ord a => a -> a -> Bool
< Word8
0xE0  = forall ty. Int -> CountOf ty
CountOf Int
2 -- 0b11100000
    | Word8
x forall a. Ord a => a -> a -> Bool
< Word8
0xF0  = forall ty. Int -> CountOf ty
CountOf Int
3 -- 0b11110000
    | Bool
otherwise = forall ty. Int -> CountOf ty
CountOf Int
4
{-# INLINE skipNextHeaderValue #-}

headerIsAscii :: StepASCII -> Bool
headerIsAscii :: StepASCII -> Bool
headerIsAscii (StepASCII Word8
x) = Word8
x forall a. Ord a => a -> a -> Bool
< Word8
0x80

charToBytes :: Int -> CountOf Word8
charToBytes :: Int -> CountOf Word8
charToBytes Int
c
    | Int
c forall a. Ord a => a -> a -> Bool
< Int
0x80     = forall ty. Int -> CountOf ty
CountOf Int
1
    | Int
c forall a. Ord a => a -> a -> Bool
< Int
0x800    = forall ty. Int -> CountOf ty
CountOf Int
2
    | Int
c forall a. Ord a => a -> a -> Bool
< Int
0x10000  = forall ty. Int -> CountOf ty
CountOf Int
3
    | Int
c forall a. Ord a => a -> a -> Bool
< Int
0x110000 = forall ty. Int -> CountOf ty
CountOf Int
4
    | Bool
otherwise    = forall a. HasCallStack => [Char] -> a
error ([Char]
"invalid code point: " forall a. Monoid a => a -> a -> a
`mappend` forall a. Show a => a -> [Char]
show Int
c)

-- | Encode a Char into a CharUTF8
encodeCharUTF8 :: Char -> CharUTF8
encodeCharUTF8 :: Char -> CharUTF8
encodeCharUTF8 !(C# Char#
c)
    | Int# -> Bool
bool# (Word# -> Word# -> Int#
ltWord# Word#
x Word#
0x80##   ) = Word32 -> CharUTF8
CharUTF8 (Word32# -> Word32
W32# (Word# -> Word32#
wordToWord32# Word#
x))
    | Int# -> Bool
bool# (Word# -> Word# -> Int#
ltWord# Word#
x Word#
0x800##  ) = Word32 -> CharUTF8
CharUTF8 (Word32# -> Word32
W32# (Word# -> Word32#
wordToWord32# Word#
encode2))
    | Int# -> Bool
bool# (Word# -> Word# -> Int#
ltWord# Word#
x Word#
0x10000##) = Word32 -> CharUTF8
CharUTF8 (Word32# -> Word32
W32# (Word# -> Word32#
wordToWord32# Word#
encode3))
    | Bool
otherwise                   = Word32 -> CharUTF8
CharUTF8 (Word32# -> Word32
W32# (Word# -> Word32#
wordToWord32# Word#
encode4))
  where
    !x :: Word#
x = Int# -> Word#
int2Word# (Char# -> Int#
ord# Char#
c)

    -- clearing mask, clearing all the bits that need to be clear as per the UTF8 encoding
    mask2 :: Word#
mask2 = Word#
0x0000bfdf## -- 1 continuation , 5 bits header
    mask3 :: Word#
mask3 = Word#
0x00bfbfef## -- 2 continuations, 4 bits header
    mask4 :: Word#
mask4 = Word#
0xbfbfbff7## -- 3 continuations, 3 bits header

    -- setting mask, settings all the bits that need to be set per the UTF8 encoding
    set2 :: Word#
set2  = Word#
0x000080c0## -- 10xxxxxx     110xxxxx
    set3 :: Word#
set3  = Word#
0x008080e0## -- 10xxxxxx * 2 1110xxxx
    set4 :: Word#
set4  = Word#
0x808080f0## -- 10xxxxxx * 3 11111xxx

    encode2 :: Word#
encode2 = Word# -> Word# -> Word#
and# Word#
mask2 (Word# -> Word# -> Word# -> Word#
or3# Word#
set2
                               (Word# -> Int# -> Word#
uncheckedShiftRL# Word#
x Int#
6#) -- 5 bits to 1st byte
                               (Word# -> Int# -> Word#
uncheckedShiftL# Word#
x Int#
8# ) -- move lowest bits to the 2nd byte
                         )
    encode3 :: Word#
encode3 = Word# -> Word# -> Word#
and# Word#
mask3 (Word# -> Word# -> Word# -> Word# -> Word#
or4# Word#
set3
                               (Word# -> Int# -> Word#
uncheckedShiftRL# Word#
x Int#
12#) -- 4 bits to 1st byte
                               (Word# -> Word# -> Word#
and# Word#
0x3f00## (Word# -> Int# -> Word#
uncheckedShiftL# Word#
x Int#
2#)) -- 6 bits to the 2nd byte
                               (Word# -> Int# -> Word#
uncheckedShiftL# Word#
x Int#
16# ) -- move lowest bits to the 3rd byte
                         )
    encode4 :: Word#
encode4 = Word# -> Word# -> Word#
and# Word#
mask4 (Word# -> Word# -> Word# -> Word# -> Word#
or4# Word#
set4
                               (Word# -> Int# -> Word#
uncheckedShiftRL# Word#
x Int#
18#) -- 3 bits to 1st byte
                               (Word# -> Word# -> Word#
or# (Word# -> Word# -> Word#
and# Word#
0x3f00## (Word# -> Int# -> Word#
uncheckedShiftRL# Word#
x Int#
4#))   -- 6 bits to the 2nd byte
                                    (Word# -> Word# -> Word#
and# Word#
0x3f0000## (Word# -> Int# -> Word#
uncheckedShiftL# Word#
x Int#
10#)) -- 6 bits to the 3nd byte
                               )
                               (Word# -> Int# -> Word#
uncheckedShiftL# Word#
x Int#
24# ) -- move lowest bits to the 4rd byte
                         )

-- | decode a CharUTF8 into a Char
--
-- If the value inside a CharUTF8 is not properly encoded, this will result in violation
-- of the Char invariants
decodeCharUTF8 :: CharUTF8 -> Char
decodeCharUTF8 :: CharUTF8 -> Char
decodeCharUTF8 c :: CharUTF8
c@(CharUTF8 !(W32# Word32#
w_))
    | CharUTF8 -> Bool
isCharUTF8Case1 CharUTF8
c = Word# -> Char
toChar# Word#
w
    | CharUTF8 -> Bool
isCharUTF8Case2 CharUTF8
c = Char
encode2
    | CharUTF8 -> Bool
isCharUTF8Case3 CharUTF8
c = Char
encode3
    | Bool
otherwise         = Char
encode4
  where
    w :: Word#
w = Word32# -> Word#
word32ToWord# Word32#
w_
    encode2 :: Char
encode2 =
        Word# -> Char
toChar# (Word# -> Word# -> Word#
or# (Word# -> Int# -> Word#
uncheckedShiftL# (Word# -> Word#
maskHeader2# Word#
w) Int#
6#)
                     (Word# -> Word#
maskContinuation# (Word# -> Int# -> Word#
uncheckedShiftRL# Word#
w Int#
8#))
                )
    encode3 :: Char
encode3 =
        Word# -> Char
toChar# (Word# -> Word# -> Word# -> Word#
or3# (Word# -> Int# -> Word#
uncheckedShiftL# (Word# -> Word#
maskHeader3# Word#
w) Int#
12#)
                      (Word# -> Int# -> Word#
uncheckedShiftRL# (Word# -> Word# -> Word#
and# Word#
0x3f00## Word#
w) Int#
8#)
                      (Word# -> Word#
maskContinuation# (Word# -> Int# -> Word#
uncheckedShiftRL# Word#
w Int#
16#))
                )
    encode4 :: Char
encode4 =
        Word# -> Char
toChar# (Word# -> Word# -> Word# -> Word# -> Word#
or4# (Word# -> Int# -> Word#
uncheckedShiftL#  (Word# -> Word#
maskHeader4# Word#
w) Int#
18#)
                      (Word# -> Int# -> Word#
uncheckedShiftRL# (Word# -> Word# -> Word#
and# Word#
0x3f00## Word#
w) Int#
10#)
                      (Word# -> Int# -> Word#
uncheckedShiftL#  (Word# -> Word# -> Word#
and# Word#
0x3f0000## Word#
w) Int#
4#)
                      (Word# -> Word#
maskContinuation# (Word# -> Int# -> Word#
uncheckedShiftRL# Word#
w Int#
24#))
                )

    -- clearing mask, removing all UTF8 metadata and keeping only signal (content)
    --maskContent2 = 0x00003f1f## -- 1 continuation , 5 bits header
    --maskContent3 = 0x003f3f0f## -- 2 continuations, 4 bits header
    --maskContent4 = 0x3f3f3f07## -- 3 continuations, 3 bits header

isCharUTF8Case1 :: CharUTF8 -> Bool
isCharUTF8Case1 :: CharUTF8 -> Bool
isCharUTF8Case1 (CharUTF8 !Word32
w) = (Word32
w forall bits. BitOps bits => bits -> bits -> bits
.&. Word32
0x80) forall a. Eq a => a -> a -> Bool
== Word32
0
{-# INLINE isCharUTF8Case1 #-}

isCharUTF8Case2 :: CharUTF8 -> Bool
isCharUTF8Case2 :: CharUTF8 -> Bool
isCharUTF8Case2 (CharUTF8 !Word32
w) = (Word32
w forall bits. BitOps bits => bits -> bits -> bits
.&. Word32
0x20) forall a. Eq a => a -> a -> Bool
== Word32
0
{-# INLINE isCharUTF8Case2 #-}

isCharUTF8Case3 :: CharUTF8 -> Bool
isCharUTF8Case3 :: CharUTF8 -> Bool
isCharUTF8Case3 (CharUTF8 !Word32
w) = (Word32
w forall bits. BitOps bits => bits -> bits -> bits
.&. Word32
0x10) forall a. Eq a => a -> a -> Bool
== Word32
0
{-# INLINE isCharUTF8Case3 #-}

isCharUTF8Case4 :: CharUTF8 -> Bool
isCharUTF8Case4 :: CharUTF8 -> Bool
isCharUTF8Case4 (CharUTF8 !Word32
w) = (Word32
w forall bits. BitOps bits => bits -> bits -> bits
.&. Word32
0x08) forall a. Eq a => a -> a -> Bool
== Word32
0
{-# INLINE isCharUTF8Case4 #-}