3ad0db6 · elk · git.iko.soy

git.iko.soy/elk

on master help sign in

Keep escape characters, switch from \x0040 to \u0040

3ad0db6Evan Czaplicki10 years ago

src/AST/Literal.hs+5−4

compare with master

unifiedsplit

@@ -3,8 +3,9 @@

3module AST.Literal where

5import qualified Data.Map as Map

6+import Data.Monoid ((<>))

6import Data.Text (Text)

7import Data.Text (Text)

7−import Data.Text.Lazy.Builder (Builder, fromString)

8+import Data.Text.Lazy.Builder (Builder, fromText)

8import Data.Text.Lazy.Builder.Int (decimal)

9import Data.Text.Lazy.Builder.Int (decimal)

9import Data.Text.Lazy.Builder.RealFloat (realFloat)

10import Data.Text.Lazy.Builder.RealFloat (realFloat)

@@ -14,7 +15,7 @@ import Data.Text.Lazy.Builder.RealFloat (realFloat)

16data Literal

17data Literal

17− = Chr Char

18+ = Chr Text

18 | Str Text

19 | Str Text

19 | IntNum Int

20 | IntNum Int

20 | FloatNum Double

21 | FloatNum Double

@@ -25,8 +26,8 @@ data Literal

25toBuilder :: Literal -> Builder

26toBuilder :: Literal -> Builder

26toBuilder literal =

27toBuilder literal =

27 case literal of

28 case literal of

28− Chr c -> fromString (show c)

29+ Chr c -> fromText ("'" <> c <> "'")

29− Str s -> fromString (show s)

30+ Str s -> fromText ("\"" <> s <> "\"")

30 IntNum n -> decimal n

31 IntNum n -> decimal n

31 FloatNum n -> realFloat n

32 FloatNum n -> realFloat n

32 Boolean bool -> if bool then "True" else "False"

33 Boolean bool -> if bool then "True" else "False"

src/Generate/JavaScript/BuiltIn.hs+2−3

@@ -10,7 +10,6 @@ module Generate.JavaScript.BuiltIn

10 )

11 where

13−import qualified Data.Text as Text

14import Data.Text (Text)

13import Data.Text (Text)

16import qualified AST.Module.Name as ModuleName

15import qualified AST.Module.Name as ModuleName

@@ -30,9 +29,9 @@ utils func args =

30-- LITERALS

29-- LITERALS

33−character :: Char -> JS.Expr

32+character :: Text -> JS.Expr

34character char =

33character char =

35− utils "chr" [ JS.String (Text.singleton char) ]

34+ utils "chr" [ JS.String char ]

src/Generate/JavaScript/Expression.hs+1−1

@@ -498,7 +498,7 @@ testToExpr test =

498 JS.String tag

499

500 DT.Literal (L.Chr char) ->

501− JS.String (Text.singleton char)

501+ JS.String char

502

503 DT.Literal lit ->

504 Literal.literal lit

src/Parse/Primitives.hs+57−109

@@ -340,16 +340,8 @@ varPrim isGoodFirstChar =

340 (# newOffset, newLength, newCol #) =

341 varPrimHelp array (offset + size) (length - size) (col + 1)

342

343− !newSize =

344− newOffset - offset

345−

346− makeArray =

347− do mutableArray <- Text.new newSize

348− Text.copyI mutableArray 0 array offset newSize

349− return mutableArray

350−

351 copiedText =

343 copiedText =

352− Text.Text (Text.run makeArray) 0 newSize

344+ copyText array offset (newOffset - offset)

353 in

345 in

354 if Set.member copiedText keywords then

346 if Set.member copiedText keywords then

355 eerr (loneTheory row newCol Variable)

347 eerr (loneTheory row newCol Variable)

@@ -360,6 +352,18 @@ varPrim isGoodFirstChar =

360 eerr (loneTheory row col Variable)

352 eerr (loneTheory row col Variable)

361

353

362

354

355+{-# INLINE copyText #-}

356+copyText :: Text.Array -> Int -> Int -> Text.Text

357+copyText array offset size =

358+ let

359+ makeArray =

360+ do mutableArray <- Text.new size

361+ Text.copyI mutableArray 0 array offset size

362+ return mutableArray

363+ in

364+ Text.Text (Text.run makeArray) 0 size

365+

366+

363{-# INLINE peek #-}

367{-# INLINE peek #-}

364peek :: Text.Array -> Int -> Text.Iter

368peek :: Text.Array -> Int -> Text.Iter

365peek array offset =

369peek array offset =

@@ -428,22 +432,13 @@ infixOp =

428 cerr err

432 cerr err

429

433

430 Right ( newOffset, newLength, newCol ) ->

434 Right ( newOffset, newLength, newCol ) ->

431− let

435+ case copyText array offset (newOffset - offset) of

432− !newSize =

433− newOffset - offset

434−

435− makeArray =

436− do mutableArray <- Text.new newSize

437− Text.copyI mutableArray 0 array offset newSize

438− return mutableArray

439− in

440− case Text.Text (Text.run makeArray) 0 newSize of

441− "=" -> cerr (ParseError row col Dot)

436+ "=" -> cerr (ParseError row col Dot)

442− "|" -> cerr (ParseError row col Pipe)

437+ "|" -> cerr (ParseError row col Pipe)

443− ":" -> cerr (ParseError row col Arrow)

438+ ":" -> cerr (ParseError row col Arrow)

444− "." -> cerr (ParseError row col Equals)

439+ "." -> cerr (ParseError row col Equals)

445− "->" -> cerr (ParseError row col HasType)

440+ "->" -> cerr (ParseError row col HasType)

446− op -> cok op (State array newOffset newLength indent row newCol)

441+ op -> cok op (State array newOffset newLength indent row newCol)

447

442

448

443

449infixOpHelp :: Text.Array -> Int -> Int -> Int -> Int -> Either ParseError (Int, Int, Int)

444infixOpHelp :: Text.Array -> Int -> Int -> Int -> Int -> Either ParseError (Int, Int, Int)

@@ -645,19 +640,9 @@ docComment =

645 cerr err

640 cerr err

646

641

647 Right (newOffset, newLength, newRow, newCol) ->

642 Right (newOffset, newLength, newRow, newCol) ->

648− let

643+ cok

649− !newSize =

650− newOffset - offset

644+ (copyText array offset (newOffset - offset))

651−

652− makeArray =

653− do mutableArray <- Text.new newSize

654− Text.copyI mutableArray 0 array offset newSize

655− return mutableArray

656−

657− comment =

658− Text.Text (Text.run makeArray) 0 newSize

659− in

660− cok comment (State array newOffset newLength indent newRow newCol)

645+ (State array newOffset newLength indent newRow newCol)

661

646

662

647

663

648

@@ -739,13 +724,8 @@ singleString array offset length row col initialOffset builder =

739 Left err ->

724 Left err ->

740 Left err

725 Left err

741

726

742− Right ( size, char ) ->

727+ Right size ->

743− let

744− !newOffset = offset + size

745− chunk = Text.Text array initialOffset (offset - initialOffset)

746− newBuilder = builder <> LB.fromText chunk <> LB.singleton char

747− in

748− singleString array newOffset (length - size) row (col + size) newOffset newBuilder

728+ singleString array (offset + size) (length - size) row (col + size) initialOffset builder

749

729

750 else if word < 0xD800 || word > 0xDBFF then

730 else if word < 0xD800 || word > 0xDBFF then

751 singleString array (offset + 1) (length - 1) row (col + 1) initialOffset builder

731 singleString array (offset + 1) (length - 1) row (col + 1) initialOffset builder

@@ -754,58 +734,34 @@ singleString array offset length row col initialOffset builder =

754 singleString array (offset + 2) (length - 2) row (col + 1) initialOffset builder

734 singleString array (offset + 2) (length - 2) row (col + 1) initialOffset builder

755

735

756

736

757−eatEscape :: Text.Array -> Int -> Int -> Int -> Int -> Problem -> Either ParseError ( Int, Char )

737+eatEscape :: Text.Array -> Int -> Int -> Int -> Int -> Problem -> Either ParseError Int

758eatEscape array offset length row col problem =

738eatEscape array offset length row col problem =

759 if length == 0 then

739 if length == 0 then

760 Left (ParseError row col problem)

740 Left (ParseError row col problem)

761

741

762 else

742 else

763 case Text.unsafeIndex array offset of

743 case Text.unsafeIndex array offset of

764− 0x0061 {- a -} -> Right ( 2, '\a' )

744+ 0x0061 {- a -} -> Right 2

765− 0x0062 {- b -} -> Right ( 2, '\b' )

745+ 0x0062 {- b -} -> Right 2

766− 0x0066 {- f -} -> Right ( 2, '\f' )

746+ 0x0066 {- f -} -> Right 2

767− 0x006E {- n -} -> Right ( 2, '\n' )

747+ 0x006E {- n -} -> Right 2

768− 0x0072 {- r -} -> Right ( 2, '\r' )

748+ 0x0072 {- r -} -> Right 2

769− 0x0074 {- t -} -> Right ( 2, '\t' )

749+ 0x0074 {- t -} -> Right 2

770− 0x0076 {- v -} -> Right ( 2, '\v' )

750+ 0x0076 {- v -} -> Right 2

771− 0x0022 {- " -} -> Right ( 2, '\"' )

751+ 0x0022 {- " -} -> Right 2

772− 0x005C {- \ -} -> Right ( 2, '\\' )

752+ 0x005C {- \ -} -> Right 2

773− 0x0027 {- ' -} -> Right ( 2, '\'' )

753+ 0x0027 {- ' -} -> Right 2

774− 0x0078 {- x -} ->

754+ 0x0075 {- u -} | length >= 5 && fourHex array offset -> Right 6

775− case eatHex array (offset + 1) (length - 1) 0 of

776− Nothing ->

777− Left (ParseError row col BadEscape)

778−

779− Just (newOffset, code) ->

780− if code <= 0x10FFFF then

781− Right ( 1 + (newOffset - offset), toEnum (fromInteger code) )

782− else

783− Left (ParseError row col BadEscape)

784−

785 _ ->

755 _ ->

786 Left (ParseError row col BadEscape)

756 Left (ParseError row col BadEscape)

787

757

788

758

789−eatHex :: Text.Array -> Int -> Int -> Integer -> Maybe ( Int, Integer )

759+fourHex :: Text.Array -> Int -> Bool

790−eatHex array offset length n =

760+fourHex array offset =

791− if length < 3 then

761+ isHex (Text.unsafeIndex array (offset + 1))

792− Nothing

762+ && isHex (Text.unsafeIndex array (offset + 2))

793−

763+ && isHex (Text.unsafeIndex array (offset + 3))

794− else

795− let

796− !word = Text.unsafeIndex array offset

764+ && isHex (Text.unsafeIndex array (offset + 4))

797− in

798− if 0x0030 <= word && word <= 0x0039 then

799− eatHex array (offset + 1) (length - 1) (16 * n + fromIntegral (word - 0x0030))

800−

801− else if 0x0061 <= word && word <= 0x0066 then

802− eatHex array (offset + 1) (length - 1) (16 * n + 10 + fromIntegral (word - 0x0061))

803−

804− else if 0x0041 <= word && word <= 0x0046 then

805− eatHex array (offset + 1) (length - 1) (16 * n + 10 + fromIntegral (word - 0x0041))

806−

807− else

808− Just (offset, n)

809

765

810

766

811

767

@@ -854,13 +810,8 @@ multiString array offset length row col initialOffset builder =

854 Left err ->

810 Left err ->

855 Left err

811 Left err

856

812

857− Right ( size, char ) ->

813+ Right size ->

858− let

859− !newOffset = offset + size

860− chunk = Text.Text array initialOffset (offset - initialOffset)

861− newBuilder = builder <> LB.fromText chunk <> LB.singleton char

862− in

863− multiString array newOffset (length - size) row (col + size) newOffset newBuilder

814+ multiString array (offset + size) (length - size) row (col + size) initialOffset builder

864

815

865 else if word < 0xD800 || word > 0xDBFF then

816 else if word < 0xD800 || word > 0xDBFF then

866 multiString array (offset + 1) (length - 1) row (col + 1) initialOffset builder

817 multiString array (offset + 1) (length - 1) row (col + 1) initialOffset builder

@@ -873,7 +824,7 @@ multiString array offset length row col initialOffset builder =

873-- CHARACTER

824-- CHARACTER

874

825

875

826

876−character :: Parser Char

827+character :: Parser Text

877character =

828character =

878 Parser $ \(State array offset length indent row col) cok cerr _ eerr ->

829 Parser $ \(State array offset length indent row col) cok cerr _ eerr ->

879 if length == 0 then

830 if length == 0 then

@@ -891,40 +842,37 @@ character =

891 Left err ->

842 Left err ->

892 cerr err

843 cerr err

893

844

894− Right ( size, char ) ->

845+ Right (endCol, size) ->

895 let

846 let

896 !newOffset = offset + size + 1

847 !newOffset = offset + size + 1

897 !newLength = length - size - 1

848 !newLength = length - size - 1

898 in

849 in

899− if newLength == 0 then

850+ if newLength == 0 || Text.unsafeIndex array newOffset /= 0x0027 {- ' -} then

900 cerr (ParseError row col BadChar)

851 cerr (ParseError row col BadChar)

901

852

902− else if Text.unsafeIndex array newOffset == 0x0027 {- ' -} then

903− cok char (State array (newOffset + 1) (newLength - 1) indent row (col + size + 2))

904−

905 else

853 else

906− cerr (ParseError row col BadChar)

854+ cok

855+ (copyText array (offset + 1) size)

856+ (State array (newOffset + 1) (newLength - 1) indent row endCol)

907

857

908

858

909−characterHelp :: Text.Array -> Int -> Int -> Int -> Int -> Either ParseError ( Int, Char )

859+characterHelp :: Text.Array -> Int -> Int -> Int -> Int -> Either ParseError ( Int, Int )

910characterHelp array offset length row col =

860characterHelp array offset length row col =

911 let

861 let

912 !word = Text.unsafeIndex array offset

862 !word = Text.unsafeIndex array offset

913 in

863 in

914− if word == 0x0027 {- ' -} then

864+ if word == 0x0027 {- ' -} || word == 0x000A {- \n -} then

915 Left (ParseError row col BadChar)

865 Left (ParseError row col BadChar)

916

866

917 else if word == 0x005C {- \ -} then

867 else if word == 0x005C {- \ -} then

918− eatEscape array (offset + 1) (length - 1) row (col + 1) BadChar

868+ do n <- eatEscape array (offset + 1) (length - 1) row (col + 1) BadChar

919−

869+ return (col + n + 2, n)

920− else if word == 0x000A {- \n -} then

921− Left (ParseError row col BadChar)

922

870

923 else if word < 0xD800 || word > 0xDBFF then

871 else if word < 0xD800 || word > 0xDBFF then

924− Right (1, unsafeChr word)

872+ Right (col + 3, 1)

925

873

926 else if length > 2 then

874 else if length > 2 then

927− Right (2, chr2 word (Text.unsafeIndex array offset))

875+ Right (col + 3, 2)

928

876

929 else

877 else

930 Left (ParseError row col BadChar)

878 Left (ParseError row col BadChar)