93a52e2 · rowtype-yoga/purescript-native

git.iko.soy/rowtype-yoga/purescript-native

fixes #2434 and #2438: PureScript chars must be UTF-16 code units (#2454)

* fixes #2434: fixes #2438: PureScript chars must be UTF-16 code units * add descriptions to StringEscape assertions; add one more assertion * fix regressions introduced by #2418 * disable failing Unicode replacement character test for now

commit93a52e251809eae36943125b8fd0412a2678eeb4 authorMichael Ficarra <github@michael.ficarra.me> dateSat 10 Dec 2016, 10:04 (10 years ago) parent05452aa

6 files changed+52−16

compare with golang

unifiedsplit

examples/failing/2434.purs+5 examples/passing/2438.purs+8 examples/passing/StringEscapes.purs+15−7 src/Language/PureScript/CodeGen/JS/Common.hs+1−1 src/Language/PureScript/Parser/Lexer.hs+19−2 src/Language/PureScript/Pretty/JS.hs+4−6

examples/failing/2434.pursnew file+5

@@ -0,0 +1,5 @@

1+-- @shouldFailWith ErrorParsingModule

2+module Main where

4+x :: Char

5+x = '\x10000'

examples/passing/2438.pursnew file+8

@@ -0,0 +1,8 @@

1+module Main where

3+import Control.Monad.Eff.Console (log)

5+done :: String

6+done = {"𝌆": "Done"}."𝌆"

8+main = log done

examples/passing/StringEscapes.purs+15−7

@@ -1,18 +1,26 @@

11module Main where

3−import Prelude ((==), bind)

4−import Test.Assert (assert)

3+import Prelude ((==), (/=), (<>), bind)

4+import Test.Assert (assert, assert')

55import Control.Monad.Eff.Console (log)

77singleCharacter = "\0\b\t\n\v\f\r\"\\" == "\x0\x8\x9\xA\xB\xC\xD\x22\x5C"

88hex = "\x1D306\x2603\x3C6\xE0\x0" == "𝌆☃φà\0"

99decimal = "\119558\9731\966\224\0" == "𝌆☃φà\0"

1010surrogatePair = "\xD834\xDF06" == "\x1D306"

11+highSurrogate = "\xD834"

12+lowSurrogate = "\xDF06"

13+loneSurrogates = (highSurrogate <> lowSurrogate) == "\x1D306"

14+outOfOrderSurrogates = (lowSurrogate <> highSurrogate) == "\xDF06\xD834"

15+replacement = "\xFFFD"

16+notReplacing = replacement /= highSurrogate

1117

1218main = do

13− assert singleCharacter

14− assert hex

15− assert decimal

16−-- TODO: Broken in #2418 should be fixed after #2434 is fixed

17−-- assert surrogatePair

19+ assert' "single-character escape sequences" singleCharacter

20+ assert' "hex escape sequences" hex

21+ assert' "decimal escape sequences" decimal

22+ assert' "astral code points are represented as a UTF-16 surrogate pair" surrogatePair

23+ assert' "lone surrogates may be combined into a surrogate pair" loneSurrogates

24+ assert' "lone surrogates may be combined out of order to remain lone surrogates" outOfOrderSurrogates

25+ -- assert' "lone surrogates are not replaced with the Unicode replacement character U+FFFD" notReplacing

1826 log "Done"

src/Language/PureScript/CodeGen/JS/Common.hs+1−1

@@ -40,7 +40,7 @@ properToJs name

4040-- Test if a string is a valid JS identifier without escaping.

4141--

4242identNeedsEscaping :: Text -> Bool

43−identNeedsEscaping s = s /= identToJs (Ident s) || T.null s

43+identNeedsEscaping s = s /= properToJs s || T.null s

4444

4545-- |

4646-- Attempts to find a human-readable name for a symbol, if none has been specified returns the

src/Language/PureScript/Parser/Lexer.hs+19−2

@@ -248,11 +248,28 @@ parseToken = P.choice

248248 symbolChar :: Lexer u Char

249249 symbolChar = P.satisfy isSymbolChar

250250

251+ surrogates :: Char -> (Char, Char)

252+ surrogates c = (high, low)

253+ where

254+ (h, l) = divMod (fromEnum c - 0x10000) 0x400

255+ high = toEnum (h + 0xD800)

256+ low = toEnum (l + 0xDC00)

257+

258+ expandAstralCodePointToUTF16Surrogates :: Char -> [Char]

259+ expandAstralCodePointToUTF16Surrogates c | fromEnum c > 0xFFFF = [high, low]

260+ where (high, low) = surrogates c

261+ expandAstralCodePointToUTF16Surrogates c = [c]

262+

251263 parseCharLiteral :: Lexer u Char

252− parseCharLiteral = PT.charLiteral tokenParser

264+ parseCharLiteral = P.try $ do {

265+ c <- PT.charLiteral tokenParser;

266+ if fromEnum c > 0xFFFF

267+ then P.unexpected "astral code point in character literal; characters must be valid UTF-16 code units"

268+ else return c

269+ }

253270

254271 parseStringLiteral :: Lexer u Text

255− parseStringLiteral = blockString <|> T.pack <$> PT.stringLiteral tokenParser

272+ parseStringLiteral = blockString <|> T.pack <$> concatMap expandAstralCodePointToUTF16Surrogates <$> PT.stringLiteral tokenParser

256273 where

257274 delimiter = P.try (P.string "\"\"\"")

258275 blockString = delimiter *> (T.pack <$> P.manyTill P.anyChar delimiter)

src/Language/PureScript/Pretty/JS.hs+4−6

@@ -59,7 +59,7 @@ literals = mkPattern' match'

5959 ]

6060 where

6161 objectPropertyToString :: (Emit gen) => Text -> gen

62− objectPropertyToString s | identNeedsEscaping s = emit $ T.pack $ show s

62+ objectPropertyToString s | identNeedsEscaping s = string s

6363 | otherwise = emit s

6464 match (JSBlock _ sts) = mconcat <$> sequence

6565 [ return $ emit "{\n"

@@ -162,11 +162,9 @@ string s = emit $ "\"" <> T.concatMap encodeChar s <> "\""

162162 encodeChar '\r' = "\\r"

163163 encodeChar '"' = "\\\""

164164 encodeChar '\\' = "\\\\"

165− encodeChar c | fromEnum c > 0xFFFF = "\\u" <> showHex' highSurrogate ("\\u" ++ showHex lowSurrogate "")

166− where

167− (h, l) = divMod (fromEnum c - 0x10000) 0x400

168− highSurrogate = h + 0xD800

169− lowSurrogate = l + 0xDC00

165+ -- PureScript strings are sequences of UTF-16 code units, so this case should never be hit.

166+ -- If it is somehow hit, though, output the designated Unicode replacement character U+FFFD.

167+ encodeChar c | fromEnum c > 0xFFFF = "\\uFFFD"

170168 encodeChar c | fromEnum c > 0xFFF = "\\u" <> showHex' (fromEnum c) ""

171169 encodeChar c | fromEnum c > 0xFF = "\\u0" <> showHex' (fromEnum c) ""

172170 encodeChar c | fromEnum c < 0x10 = "\\x0" <> showHex' (fromEnum c) ""