Remove CodePoints parsers and use CodeUnits parsers (#59)

* Remove CodePoints parser

* Implement, test, and document `anyCodePoint`

Implementation was designed by @hdgarrood

* Add tests for lone surrogate pairs

* Test the parsed UTF-16 string becomes invalid due to splitting surrogate pair

Author: JordanMartinez

src/Text/Parsing/StringParser/CodePoints.purs

@@ -6,6 +6,7 @@
 module Text.Parsing.StringParser.CodeUnits
   ( eof
   , anyChar
+  , anyCodePoint
   , anyDigit
   , string
   , satisfy
@@ -30,6 +31,7 @@ import Data.Char (toCharCode)
 import Data.Either (Either(..))
 import Data.Foldable (class Foldable, foldMap, elem, notElem)
 import Data.Maybe (Maybe(..))
+import Data.String as SCP
 import Data.String.CodeUnits (charAt, singleton)
 import Data.String.CodeUnits as SCU
 import Data.String.Pattern (Pattern(..))
@@ -45,13 +47,22 @@ eof = Parser \s ->
     { str, pos } | pos < SCU.length str -> Left { pos, error: ParseError "Expected EOF" }
     _ -> Right { result: unit, suffix: s }
 
--- | Match any character.
+-- | Match any character. This is limited by `Char` to any code points
+-- | that are below `0xFFFF`. If you need to use higher code points
+-- | (e.g. emoji), see `anyCodePoint` and `string`.
 anyChar :: Parser Char
 anyChar = Parser \{ str, pos } ->
   case charAt pos str of
     Just chr -> Right { result: chr, suffix: { str, pos: pos + 1 } }
     Nothing -> Left { pos, error: ParseError "Unexpected EOF" }
 
+-- | Match any code point, including those above `0xFFFF`
+anyCodePoint :: Parser SCP.CodePoint
+anyCodePoint = Parser \rec@{ str, pos } ->
+   case SCP.codePointAt 0 (SCU.drop pos str) of
+     Just cp -> Right { result: cp, suffix: { str, pos: pos + SCU.length (SCP.singleton cp) } }
+     Nothing -> Left { pos, error: ParseError "Unexpected EOF" }
+
 -- | Match any digit.
 anyDigit :: Parser Char
 anyDigit = try do

src/Text/Parsing/StringParser/CodeUnits.purs

@@ -9,15 +9,16 @@ import Data.List (List(Nil), (:))
 import Data.List.Lazy (take, repeat)
 import Data.List.NonEmpty (NonEmptyList(..))
 import Data.NonEmpty ((:|))
+import Data.String.CodePoints as SCP
 import Data.String.CodeUnits (singleton)
 import Data.String.Common as SC
 import Data.Unfoldable (replicate)
 import Effect (Effect)
 import Test.Assert (assert', assert)
 import Text.Parsing.StringParser (Parser, runParser, try)
+import Text.Parsing.StringParser.CodeUnits (anyChar, anyCodePoint, anyDigit, eof, regex, string)
 import Text.Parsing.StringParser.Combinators (many1, endBy1, sepBy1, optionMaybe, many, manyTill, many1Till, chainl, fix, between)
 import Text.Parsing.StringParser.Expr (Assoc(..), Operator(..), buildExprParser)
-import Text.Parsing.StringParser.CodeUnits (anyDigit, eof, string, anyChar, regex)
 
 parens :: forall a. Parser a -> Parser a
 parens = between (string "(") (string ")")
@@ -97,3 +98,22 @@ testCodeUnits = do
   assert $ canParse (many1Till (string "a") (string "and")) $ (fold <<< take 10000 $ repeat "a") <> "and"
   -- check correct order
   assert $ expectResult (NonEmptyList ('a' :| 'b':'c':Nil)) (many1Till anyChar (string "d")) "abcd"
+  -- check anyCodePoint
+  let anyCodePointStr = map SCP.singleton anyCodePoint
+  let anyCharStr = map singleton anyChar
+  assert $ expectResult (NonEmptyList ("🍔" :| "🍺":Nil)) (many1 $ anyCodePointStr) "🍔🍺"
+  assert $ expectResult "🍔" (anyChar *> anyCodePointStr <* anyChar) "a🍔a"
+  assert $ expectResult ({a: "🍔", b: "🍺"}) ({a:_, b:_} <$> (anyCodePointStr <* void anyChar) <*> anyCodePointStr) "🍔a🍺"
+  assert $ expectResult ({a: "a", b: "b", c:"c"}) ({a:_, b:_, c:_} <$> anyCodePointStr <*> anyCodePointStr <*> anyCodePointStr) "abc"
+  -- check string
+  assert $ expectResult "🍔🍺" (string "🍔🍺") "🍔🍺"
+  assert $ expectResult (NonEmptyList ("🍔🍺" :| "🍔🍺":"🍔🍺":Nil)) (many1 $ string "🍔🍺") "🍔🍺🍔🍺🍔🍺"
+  assert $ expectResult (NonEmptyList ("a🍔🍺":|"a🍔🍺":"a🍔🍺":Nil)) (many1 $ string "a🍔🍺") "a🍔🍺a🍔🍺a🍔🍺"
+  assert $ expectResult (NonEmptyList ("🍔a🍺":|"🍔a🍺":"🍔a🍺":Nil)) (many1 $ string "🍔a🍺") "🍔a🍺🍔a🍺🍔a🍺"
+  assert $ expectResult (NonEmptyList ("🍔🍺a" :| "🍔🍺a":"🍔🍺a":Nil)) (many1 $ string "🍔🍺a") "🍔🍺a🍔🍺a🍔🍺a"
+  assert $ expectResult (NonEmptyList ("a" :| "a":"a":Nil)) (many1 $ string "a") "aaa"
+  assert $ expectResult (NonEmptyList ("abc" :| "abc":"abc":Nil)) (many1 $ string "abc") "abcabcabc"
+  assert $ expectResult (NonEmptyList ("abc" :| "abc":"abc":Nil)) (many1 $ string "abc") "abcabcabc"
+  assert $ expectResult (NonEmptyList ("abc�def" :| Nil)) (many1 $ string "abc�def") "abc�def"
+
+  assert $ expectResult "🍔\xd83c" (string "🍔\xd83c") "🍔🍺"

test/CodePoints.purs

@@ -9,7 +9,7 @@ import Data.List.Types (NonEmptyList)
 import Effect (Effect)
 import Effect.Console (log, logShow)
 import Text.Parsing.StringParser (Parser, fail, runParser, unParser)
-import Text.Parsing.StringParser.CodePoints (anyChar, char, eof, regex, skipSpaces, string)
+import Text.Parsing.StringParser.CodeUnits (anyChar, char, eof, regex, skipSpaces, string)
 import Text.Parsing.StringParser.Combinators (between, endBy1, lookAhead, many, many1, sepBy1, (<?>))
 
 -- Serves only to make this file runnable

test/CodeUnits.purs

@@ -4,13 +4,9 @@ import Prelude
 
 import Effect (Effect)
 import Effect.Console (log)
-import Test.CodePoints (testCodePoints)
 import Test.CodeUnits (testCodeUnits)
 
 main :: Effect Unit
 main = do
-  log "Testing CodePoint parsing\n"
-  testCodePoints
-
   log "\n\nTesting CodeUnit parsing\n"
   testCodeUnits