git.iko.soy/rowtype-yoga/purescript-parsing-dataview
on master help sign in
New parser: anyCodePointUTF8
commit50a5cc94db5dc45426adceeca04097dfdeefad70 authorJames Brock <jamesbrock@gmail.com> dateFri 2 Dec 2022, 21:41 (4 years ago) parent1e6a6ce
5 files changed+82−3
compare with master
unifiedsplit
CHANGELOG.md+2−1 bower.json+2 spago.dhall+2 src/Parsing/DataView.Basic.purs+47 test/Main.purs+29−2
CHANGELOG.md+2−1
@@ -2,7 +2,8 @@
22
33## Unreleased
44
5−New combinator: `anyTill`.
5+- New combinator: `anyTill`
6+- New parser: `anyCodePointUTF8`
67
78## v3.1.0
89
bower.json+2
@@ -17,10 +17,12 @@
1717 "purescript-arraybuffer": "^v13.1.1",
1818 "purescript-arraybuffer-types": "^v3.0.2",
1919 "purescript-effect": "^v4.0.0",
20+ "purescript-enums": "^v6.0.1",
2021 "purescript-float32": "^v2.0.0",
2122 "purescript-maybe": "^v6.0.0",
2223 "purescript-parsing": "^v10.2.0",
2324 "purescript-prelude": "^v6.0.1",
25+ "purescript-strings": "^v6.0.1",
2426 "purescript-tailrec": "^v6.1.0",
2527 "purescript-transformers": "^v6.0.0",
2628 "purescript-tuples": "^v7.0.0",
spago.dhall+2
@@ -5,10 +5,12 @@
55 [ "arraybuffer"
66 , "arraybuffer-types"
77 , "effect"
8+ , "enums"
89 , "float32"
910 , "maybe"
1011 , "parsing"
1112 , "prelude"
13+ , "strings"
1214 , "tailrec"
1315 , "transformers"
1416 , "tuples"
src/Parsing/DataView.Basic.pursnew file+47
@@ -0,0 +1,47 @@
1+-- | Basic `DataView` parsers derived from primitive `DataView` parsers.
2+-- |
3+-- | All of these parsers will consume when they succeed.
4+-- |
5+-- | All of these parsers will not consume and will automatically
6+-- | backtrack when they fail.
7+module Parsing.DataView.Basic
8+ ( anyCodePointUTF8
9+ ) where
10+
11+import Prelude
12+
13+import Data.ArrayBuffer.Types (DataView)
14+import Data.Enum (toEnum)
15+import Data.String (CodePoint)
16+import Data.UInt (toInt)
17+import Effect.Class (class MonadEffect)
18+import Parsing (ParserT, fail, liftMaybe)
19+import Parsing.Combinators (tryRethrow)
20+import Parsing.DataView (anyUint8)
21+
22+-- | Decode one UTF-8 code point, *1—4* bytes.
23+anyCodePointUTF8
24+ :: forall m
25+ . MonadEffect m
26+ => ParserT DataView m CodePoint
27+anyCodePointUTF8 = tryRethrow do
28+ -- https://en.wikipedia.org/wiki/UTF-8#Encoding
29+ c0 <- toInt <$> anyUint8
30+ if c0 < 128 then do
31+ liftMaybe (\_ -> "Invalid CodePoint " <> show c0) (toEnum c0)
32+ else if c0 < 224 then do
33+ c1 <- toInt <$> anyUint8
34+ let codepoint = (64 * (c0 `mod` 32)) + (c1 `mod` 64)
35+ liftMaybe (\_ -> "Invalid CodePoint " <> show codepoint) (toEnum codepoint)
36+ else if c0 < 240 then do
37+ c1 <- toInt <$> anyUint8
38+ c2 <- toInt <$> anyUint8
39+ let codepoint = (4096 * (c0 `mod` 16)) + (64 * (c1 `mod` 64)) + (c2 `mod` 64)
40+ liftMaybe (\_ -> "Invalid CodePoint " <> show codepoint) (toEnum codepoint)
41+ else if c0 < 248 then do
42+ c1 <- toInt <$> anyUint8
43+ c2 <- toInt <$> anyUint8
44+ c3 <- toInt <$> anyUint8
45+ let codepoint = (262144 * (c0 `mod` 8)) + (4096 * (c1 `mod` 64)) + (64 * (c2 `mod` 64)) + (c3 `mod` 64)
46+ liftMaybe (\_ -> "Invalid CodePoint " <> show codepoint) (toEnum codepoint)
47+ else fail "Invalid first code point"
test/Main.purs+29−2
@@ -8,6 +8,7 @@ import Data.ArrayBuffer.ArrayBuffer as AB
88import Data.ArrayBuffer.Cast (fromUint8Array, toUint8Array)
99import Data.ArrayBuffer.DataView as DataView
1010import Data.Either (Either(..))
11+import Data.Enum (fromEnum)
1112import Data.List (fromFoldable)
1213import Data.List as List
1314import Data.Maybe (Maybe(..))
@@ -20,8 +21,9 @@ import Effect.Console (logShow)
2021import Effect.Exception (catchException, message)
2122import Parsing (ParserT, Position(..), fail, liftExceptT, parseErrorPosition, runParserT)
2223import Parsing.Combinators (many, manyTill)
23−import Parsing.DataView (anyInt8, anyTill, satisfyInt8)
24+import Parsing.DataView (anyInt8, anyTill, eof, satisfyInt8)
2425import Parsing.DataView as DV
26+import Parsing.DataView.Basic (anyCodePointUTF8)
2527import Test.Assert (assert', assertEqual')
2628import Web.Encoding.TextDecoder as TextDecoder
2729import Web.Encoding.TextEncoder as TextEncoder
@@ -181,4 +183,29 @@ main = do
181183 assertEqual' "anyTill"
182184 { expected: Right 2
183185 , actual: DataView.byteLength <$> result
184− }
186+ }
187+
188+ do
189+ -- Test one code point from each byte group
190+ -- a 97
191+ -- λ 955
192+ -- 月 0x6708
193+ -- 👓 0x1F453
194+ let teststring = "aλ月👓a"
195+ textEncoder <- TextEncoder.new
196+ let testarray = TextEncoder.encode teststring textEncoder
197+ testview <- fromUint8Array testarray
198+
199+ result <- (map <<< map <<< map) fromEnum $ runParserT testview do
200+ c0 <- anyCodePointUTF8
201+ c1 <- anyCodePointUTF8
202+ c2 <- anyCodePointUTF8
203+ c3 <- anyCodePointUTF8
204+ c4 <- anyCodePointUTF8
205+ eof
206+ pure [ c0, c1, c2, c3, c4 ]
207+
208+ assertEqual' "anyCodePointUTF8"
209+ { expected: Right [ 97, 955, 0x6708, 0x1F453, 97 ]
210+ , actual: result
211+ }