From 4a2d5bda532925d06407866f4b303c7eff94f977 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 7 Mar 2018 14:26:15 -0500 Subject: [PATCH 1/7] necessary changes for floating-point munging --- semantic.cabal | 1 + src/Data/Syntax/Literal.hs | 40 +++++++++++++++++++++++++-- src/Language/Go/Assignment.hs | 2 +- src/Language/JSON/Assignment.hs | 2 +- src/Language/PHP/Assignment.hs | 2 +- src/Language/Python/Assignment.hs | 6 +++- src/Language/Ruby/Assignment.hs | 3 +- src/Language/TypeScript/Assignment.hs | 2 +- 8 files changed, 49 insertions(+), 9 deletions(-) diff --git a/semantic.cabal b/semantic.cabal index 0f6682c2d..4b8268a97 100644 --- a/semantic.cabal +++ b/semantic.cabal @@ -163,6 +163,7 @@ library , pointed , recursion-schemes , semigroups + , scientific , split , stm-chans , template-haskell diff --git a/src/Data/Syntax/Literal.hs b/src/Data/Syntax/Literal.hs index f743e13e9..aab6558e0 100644 --- a/src/Data/Syntax/Literal.hs +++ b/src/Data/Syntax/Literal.hs @@ -2,9 +2,15 @@ module Data.Syntax.Literal where import Data.Abstract.Evaluatable -import Data.ByteString.Char8 (readInteger) +import Data.ByteString.Char8 (readInteger, unpack) +import qualified Data.ByteString.Char8 as B +import Data.Monoid (Endo (..), appEndo) +import Data.Scientific (Scientific) +import Data.Word (Word8) import Diffing.Algorithm +import Prelude hiding (Float, fail) import Prologue hiding (Set) +import Text.Read (readMaybe) -- Boolean @@ -45,17 +51,45 @@ instance Evaluatable Data.Syntax.Literal.Integer where -- TODO: Consider a Numeric datatype with FloatingPoint/Integral/etc constructors. -- | A literal float of unspecified width. -newtype Float a = Float { floatContent :: ByteString } +data Float a = Float { floatContent :: Scientific + , floatOriginal :: ByteString + } deriving (Diffable, Eq, Foldable, Functor, GAlign, Generic1, Mergeable, Ord, Show, Traversable, FreeVariables1) instance Eq1 Data.Syntax.Literal.Float where liftEq = genericLiftEq instance Ord1 Data.Syntax.Literal.Float where liftCompare = genericLiftCompare instance Show1 Data.Syntax.Literal.Float where liftShowsPrec = genericLiftShowsPrec +-- | Ensures that numbers of the form '.52' are parsed correctly. You usually want this. +padWithLeadingZero :: ByteString -> ByteString +padWithLeadingZero b + | fmap fst (B.uncons b) == Just '.' = B.cons '0' b + | otherwise = b + +padWithTrailingZero :: ByteString -> ByteString +padWithTrailingZero b + | fmap snd (B.unsnoc b) == Just '.' = B.snoc b '0' + | otherwise = b + +-- | Removes underscores in numeric literals. Python 3 and Ruby support this, whereas Python 2, JS, and Go do not. +removeUnderscores :: ByteString -> ByteString +removeUnderscores = B.filter (/= '_') + +-- | Strip suffixes from floating-point literals so as to handle Python's +-- TODO: tree-sitter-python needs some love so that it parses j-suffixed floats as complexen +dropAlphaSuffix :: ByteString -> ByteString +dropAlphaSuffix = B.takeWhile (\x -> x `notElem` ("lLjJiI" :: [Char])) + +buildFloat :: MonadFail m => [ByteString -> ByteString] -> ByteString -> m (Float a) +buildFloat preds val = + let munger = appEndo (foldMap Endo preds) + in case readMaybe (unpack (munger val)) of + Nothing -> fail ("Invalid floating-point value: " <> show val) + Just s -> pure (Float s val) + -- TODO: Implement Eval instance for Float instance Evaluatable Data.Syntax.Literal.Float - -- Rational literals e.g. `2/3r` newtype Rational a = Rational ByteString deriving (Diffable, Eq, Foldable, Functor, GAlign, Generic1, Mergeable, Ord, Show, Traversable, FreeVariables1) diff --git a/src/Language/Go/Assignment.hs b/src/Language/Go/Assignment.hs index da08c6b53..1b2bdddda 100644 --- a/src/Language/Go/Assignment.hs +++ b/src/Language/Go/Assignment.hs @@ -225,7 +225,7 @@ fieldIdentifier :: Assignment fieldIdentifier = makeTerm <$> symbol FieldIdentifier <*> (Syntax.Identifier <$> source) floatLiteral :: Assignment -floatLiteral = makeTerm <$> symbol FloatLiteral <*> (Literal.Float <$> source) +floatLiteral = makeTerm <$> symbol FloatLiteral <*> (source >>= Literal.buildFloat [Literal.padWithLeadingZero, Literal.dropAlphaSuffix]) identifier :: Assignment identifier = makeTerm <$> (symbol Identifier <|> symbol Identifier') <*> (Syntax.Identifier <$> source) diff --git a/src/Language/JSON/Assignment.hs b/src/Language/JSON/Assignment.hs index 591c88da2..b8db963e2 100644 --- a/src/Language/JSON/Assignment.hs +++ b/src/Language/JSON/Assignment.hs @@ -50,7 +50,7 @@ array :: Assignment array = makeTerm <$> symbol Array <*> children (Literal.Array <$> many jsonValue) number :: Assignment -number = makeTerm <$> symbol Number <*> (Literal.Float <$> source) +number = makeTerm <$> symbol Number <*> (source >>= Literal.buildFloat [Literal.padWithLeadingZero]) string :: Assignment string = makeTerm <$> symbol String <*> (Literal.TextElement <$> source) diff --git a/src/Language/PHP/Assignment.hs b/src/Language/PHP/Assignment.hs index 6805f85cf..ed6641864 100644 --- a/src/Language/PHP/Assignment.hs +++ b/src/Language/PHP/Assignment.hs @@ -470,7 +470,7 @@ literal :: Assignment literal = integer <|> float <|> string float :: Assignment -float = makeTerm <$> symbol Float <*> (Literal.Float <$> source) +float = makeTerm <$> symbol Float <*> (source >>= Literal.buildFloat [Literal.padWithLeadingZero]) integer :: Assignment integer = makeTerm <$> symbol Integer <*> (Literal.Integer <$> source) diff --git a/src/Language/Python/Assignment.hs b/src/Language/Python/Assignment.hs index 4eeabbeb0..3c71292db 100644 --- a/src/Language/Python/Assignment.hs +++ b/src/Language/Python/Assignment.hs @@ -17,6 +17,7 @@ import GHC.Stack import Language.Python.Grammar as Grammar import Language.Python.Syntax as Python.Syntax import qualified Assigning.Assignment as Assignment +import qualified Data.ByteString as B import qualified Data.Syntax as Syntax import qualified Data.Syntax.Comment as Comment import qualified Data.Syntax.Declaration as Declaration @@ -364,7 +365,10 @@ concatenatedString :: Assignment concatenatedString = makeTerm <$> symbol ConcatenatedString <*> children (manyTerm string) float :: Assignment -float = makeTerm <$> symbol Float <*> (Literal.Float <$> source) +float = makeTerm <$> symbol Float <*> (source >>= Literal.buildFloat [ Literal.padWithLeadingZero + , Literal.padWithTrailingZero + , Literal.dropAlphaSuffix + , Literal.removeUnderscores]) integer :: Assignment integer = makeTerm <$> symbol Integer <*> (Literal.Integer <$> source) diff --git a/src/Language/Ruby/Assignment.hs b/src/Language/Ruby/Assignment.hs index d1bf292d2..8242d1175 100644 --- a/src/Language/Ruby/Assignment.hs +++ b/src/Language/Ruby/Assignment.hs @@ -12,6 +12,7 @@ import Data.Record import Data.List (elem) import Data.Syntax (contextualize, postContextualize, emptyTerm, parseError, handleError, infixContext, makeTerm, makeTerm', makeTerm'', makeTerm1) import qualified Assigning.Assignment as Assignment +import qualified Data.ByteString as B import qualified Data.Syntax as Syntax import qualified Data.Syntax.Comment as Comment import qualified Data.Syntax.Declaration as Declaration @@ -166,7 +167,7 @@ literal = <|> makeTerm <$> token Grammar.False <*> pure Literal.false <|> makeTerm <$> token Grammar.Nil <*> pure Literal.Null <|> makeTerm <$> symbol Grammar.Integer <*> (Literal.Integer <$> source) - <|> makeTerm <$> symbol Grammar.Float <*> (Literal.Float <$> source) + <|> makeTerm <$> symbol Grammar.Float <*> (source >>= Literal.buildFloat [Literal.padWithLeadingZero, Literal.removeUnderscores]) <|> makeTerm <$> symbol Grammar.Rational <*> (Literal.Rational <$> source) <|> makeTerm <$> symbol Grammar.Complex <*> (Literal.Complex <$> source) -- TODO: Do we want to represent the difference between .. and ... diff --git a/src/Language/TypeScript/Assignment.hs b/src/Language/TypeScript/Assignment.hs index a06a603d1..db92c075a 100644 --- a/src/Language/TypeScript/Assignment.hs +++ b/src/Language/TypeScript/Assignment.hs @@ -325,7 +325,7 @@ importAlias' :: Assignment importAlias' = makeTerm <$> symbol Grammar.ImportAlias <*> children (TypeScript.Syntax.ImportAlias <$> term identifier <*> term (identifier <|> nestedIdentifier)) number :: Assignment -number = makeTerm <$> symbol Grammar.Number <*> (Literal.Float <$> source) +number = makeTerm <$> symbol Grammar.Number <*> (source >>= Literal.buildFloat [Literal.padWithLeadingZero]) string :: Assignment string = makeTerm <$> symbol Grammar.String <*> (Literal.TextElement <$> source) From 86697b46d1131929aad4cd93e28df339dd186440 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 7 Mar 2018 14:26:31 -0500 Subject: [PATCH 2/7] changes to the diffs because the hashes of the values changed --- test/fixtures/python/float.diffA-B.txt | 8 ++++---- test/fixtures/python/float.diffB-A.txt | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/fixtures/python/float.diffA-B.txt b/test/fixtures/python/float.diffA-B.txt index ac2863db4..db81ffe15 100644 --- a/test/fixtures/python/float.diffA-B.txt +++ b/test/fixtures/python/float.diffA-B.txt @@ -6,14 +6,14 @@ ->(Float) } {+(Float)+} {+(Float)+} -{+(Float)+} -{+(Float)+} { (Float) ->(Float) } {+(Float)+} {+(Float)+} -{ (Float) -->(Float) } +{+(Float)+} +{+(Float)+} +{+(Float)+} +{-(Float)-} {-(Float)-} {-(Float)-} {-(Float)-} diff --git a/test/fixtures/python/float.diffB-A.txt b/test/fixtures/python/float.diffB-A.txt index 48cb37153..82f4b4fa2 100644 --- a/test/fixtures/python/float.diffB-A.txt +++ b/test/fixtures/python/float.diffB-A.txt @@ -5,14 +5,14 @@ { (Float) ->(Float) } {+(Float)+} -{+(Float)+} -{+(Float)+} -{+(Float)+} -{+(Float)+} { (Float) ->(Float) } {+(Float)+} {+(Float)+} +{+(Float)+} +{+(Float)+} +{+(Float)+} +{+(Float)+} {-(Float)-} {-(Float)-} {-(Float)-} From ad4e7e83f4c2eda565dfa2337c760a34a17cfbac Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 7 Mar 2018 14:38:02 -0500 Subject: [PATCH 3/7] Ensure that floats can be evaluated. --- src/Control/Abstract/Value.hs | 6 ++++++ src/Data/Abstract/Type.hs | 1 + src/Data/Abstract/Value.hs | 12 +++++++++++- src/Data/Syntax/Literal.hs | 5 ++--- src/Language/Python/Assignment.hs | 1 - src/Language/Ruby/Assignment.hs | 1 - 6 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/Control/Abstract/Value.hs b/src/Control/Abstract/Value.hs index 8120eb47c..da3ef8881 100644 --- a/src/Control/Abstract/Value.hs +++ b/src/Control/Abstract/Value.hs @@ -10,6 +10,7 @@ import Data.Abstract.Environment import Data.Abstract.FreeVariables import Data.Abstract.Value as Value import Data.Abstract.Type as Type +import Data.Scientific (Scientific) import Prologue import Prelude hiding (fail) @@ -29,6 +30,9 @@ class (MonadEvaluator t v m) => MonadValue t v m where -- | Construct an abstract string value. string :: ByteString -> m v + -- | Construct a floating-point value. + float :: Scientific -> m v + -- | Eliminate boolean values. TODO: s/boolean/truthy ifthenelse :: v -> m v -> m v -> m v @@ -51,6 +55,7 @@ instance ( FreeVariables t integer = pure . inj . Integer boolean = pure . inj . Boolean string = pure . inj . Value.String + float = pure . inj . Value.Float ifthenelse cond if' else' | Just (Boolean b) <- prj cond = if b then if' else else' @@ -83,6 +88,7 @@ instance (Alternative m, MonadEvaluator t Type m, MonadFresh m) => MonadValue t integer _ = pure Int boolean _ = pure Bool string _ = pure Type.String + float _ = pure Type.Float ifthenelse cond if' else' = unify cond Bool *> (if' <|> else') diff --git a/src/Data/Abstract/Type.hs b/src/Data/Abstract/Type.hs index c0b352bce..9709d087a 100644 --- a/src/Data/Abstract/Type.hs +++ b/src/Data/Abstract/Type.hs @@ -13,6 +13,7 @@ data Type | Bool -- ^ Primitive boolean type. | String -- ^ Primitive string type. | Unit -- ^ The unit type. + | Float -- ^ Floating-point type. | Type :-> Type -- ^ Binary function types. | Var TName -- ^ A type variable. | Product [Type] -- ^ N-ary products. diff --git a/src/Data/Abstract/Value.hs b/src/Data/Abstract/Value.hs index 9f798baf2..62a406bad 100644 --- a/src/Data/Abstract/Value.hs +++ b/src/Data/Abstract/Value.hs @@ -8,8 +8,9 @@ import Data.Abstract.FreeVariables import Data.Abstract.Live import qualified Data.Abstract.Type as Type import qualified Data.Set as Set +import Data.Scientific (Scientific) import Prologue -import Prelude hiding (Integer, String, fail) +import Prelude hiding (Float, Integer, String, fail) import qualified Prelude type ValueConstructors location @@ -17,6 +18,7 @@ type ValueConstructors location , Interface location , Unit , Boolean + , Float , Integer , String ] @@ -75,6 +77,14 @@ instance Eq1 String where liftEq = genericLiftEq instance Ord1 String where liftCompare = genericLiftCompare instance Show1 String where liftShowsPrec = genericLiftShowsPrec +-- | Float values. +newtype Float term = Float Scientific + deriving (Eq, Generic1, Ord, Show) + +instance Eq1 Float where liftEq = genericLiftEq +instance Ord1 Float where liftCompare = genericLiftCompare +instance Show1 Float where liftShowsPrec = genericLiftShowsPrec + -- | The environment for an abstract value type. type EnvironmentFor v = Environment (LocationFor v) v diff --git a/src/Data/Syntax/Literal.hs b/src/Data/Syntax/Literal.hs index aab6558e0..2ab2b9991 100644 --- a/src/Data/Syntax/Literal.hs +++ b/src/Data/Syntax/Literal.hs @@ -6,7 +6,6 @@ import Data.ByteString.Char8 (readInteger, unpack) import qualified Data.ByteString.Char8 as B import Data.Monoid (Endo (..), appEndo) import Data.Scientific (Scientific) -import Data.Word (Word8) import Diffing.Algorithm import Prelude hiding (Float, fail) import Prologue hiding (Set) @@ -87,8 +86,8 @@ buildFloat preds val = Nothing -> fail ("Invalid floating-point value: " <> show val) Just s -> pure (Float s val) --- TODO: Implement Eval instance for Float -instance Evaluatable Data.Syntax.Literal.Float +instance Evaluatable Data.Syntax.Literal.Float where + eval (Float s _) = float s -- Rational literals e.g. `2/3r` newtype Rational a = Rational ByteString diff --git a/src/Language/Python/Assignment.hs b/src/Language/Python/Assignment.hs index 3c71292db..a22905507 100644 --- a/src/Language/Python/Assignment.hs +++ b/src/Language/Python/Assignment.hs @@ -17,7 +17,6 @@ import GHC.Stack import Language.Python.Grammar as Grammar import Language.Python.Syntax as Python.Syntax import qualified Assigning.Assignment as Assignment -import qualified Data.ByteString as B import qualified Data.Syntax as Syntax import qualified Data.Syntax.Comment as Comment import qualified Data.Syntax.Declaration as Declaration diff --git a/src/Language/Ruby/Assignment.hs b/src/Language/Ruby/Assignment.hs index 8242d1175..1cf03032a 100644 --- a/src/Language/Ruby/Assignment.hs +++ b/src/Language/Ruby/Assignment.hs @@ -12,7 +12,6 @@ import Data.Record import Data.List (elem) import Data.Syntax (contextualize, postContextualize, emptyTerm, parseError, handleError, infixContext, makeTerm, makeTerm', makeTerm'', makeTerm1) import qualified Assigning.Assignment as Assignment -import qualified Data.ByteString as B import qualified Data.Syntax as Syntax import qualified Data.Syntax.Comment as Comment import qualified Data.Syntax.Declaration as Declaration From 6c4777a8543774b4745c71cc6c4116e02aa717d1 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 7 Mar 2018 14:49:50 -0500 Subject: [PATCH 4/7] add some comments --- src/Data/Syntax/Literal.hs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Data/Syntax/Literal.hs b/src/Data/Syntax/Literal.hs index 2ab2b9991..99ba59d48 100644 --- a/src/Data/Syntax/Literal.hs +++ b/src/Data/Syntax/Literal.hs @@ -59,12 +59,13 @@ instance Eq1 Data.Syntax.Literal.Float where liftEq = genericLiftEq instance Ord1 Data.Syntax.Literal.Float where liftCompare = genericLiftCompare instance Show1 Data.Syntax.Literal.Float where liftShowsPrec = genericLiftShowsPrec --- | Ensures that numbers of the form '.52' are parsed correctly. You usually want this. +-- | Ensures that numbers of the form '.52' are parsed correctly. Most languages need this. padWithLeadingZero :: ByteString -> ByteString padWithLeadingZero b | fmap fst (B.uncons b) == Just '.' = B.cons '0' b | otherwise = b +-- | As @padWithLeadingZero@, but on the end. Not all languages need this. padWithTrailingZero :: ByteString -> ByteString padWithTrailingZero b | fmap snd (B.unsnoc b) == Just '.' = B.snoc b '0' @@ -79,6 +80,11 @@ removeUnderscores = B.filter (/= '_') dropAlphaSuffix :: ByteString -> ByteString dropAlphaSuffix = B.takeWhile (\x -> x `notElem` ("lLjJiI" :: [Char])) +-- | This is the shared function that processes a bytestring representation of a float +-- into a Float-wrapped @Scientific@. It takes as its arguments a list of functions, which +-- will be some combination of the above 'ByteString -> ByteString' functions. This is meant +-- to be called from an @Assignment@, hence the @MonadFail@ constraint. Caveat: the list is +-- order-dependent; the rightmost function will be applied first. buildFloat :: MonadFail m => [ByteString -> ByteString] -> ByteString -> m (Float a) buildFloat preds val = let munger = appEndo (foldMap Endo preds) From 957897fe8c1b4096612018e06561468abb6c3124 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 7 Mar 2018 14:55:51 -0500 Subject: [PATCH 5/7] strictify --- src/Data/Syntax/Literal.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Data/Syntax/Literal.hs b/src/Data/Syntax/Literal.hs index 99ba59d48..0ed4f68f7 100644 --- a/src/Data/Syntax/Literal.hs +++ b/src/Data/Syntax/Literal.hs @@ -50,8 +50,8 @@ instance Evaluatable Data.Syntax.Literal.Integer where -- TODO: Consider a Numeric datatype with FloatingPoint/Integral/etc constructors. -- | A literal float of unspecified width. -data Float a = Float { floatContent :: Scientific - , floatOriginal :: ByteString +data Float a = Float { floatContent :: !Scientific + , floatOriginal :: !ByteString } deriving (Diffable, Eq, Foldable, Functor, GAlign, Generic1, Mergeable, Ord, Show, Traversable, FreeVariables1) From 3ba4ddef61b0b38f9d9b59ee7d34831c0c7d4f54 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 7 Mar 2018 18:46:46 -0500 Subject: [PATCH 6/7] convert to a Scientific in the evaluation stage --- src/Data/Syntax/Literal.hs | 24 +++++++++++++----------- src/Language/Go/Assignment.hs | 2 +- src/Language/JSON/Assignment.hs | 2 +- src/Language/PHP/Assignment.hs | 2 +- src/Language/Python/Assignment.hs | 9 +++++---- src/Language/Ruby/Assignment.hs | 2 +- src/Language/TypeScript/Assignment.hs | 2 +- 7 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/Data/Syntax/Literal.hs b/src/Data/Syntax/Literal.hs index 0ed4f68f7..b3dfd5d35 100644 --- a/src/Data/Syntax/Literal.hs +++ b/src/Data/Syntax/Literal.hs @@ -1,4 +1,4 @@ -{-# LANGUAGE DataKinds, DeriveAnyClass, DeriveGeneric, MultiParamTypeClasses #-} +{-# LANGUAGE DataKinds, DeriveAnyClass, DeriveGeneric, MultiParamTypeClasses, TypeApplications #-} module Data.Syntax.Literal where import Data.Abstract.Evaluatable @@ -50,9 +50,7 @@ instance Evaluatable Data.Syntax.Literal.Integer where -- TODO: Consider a Numeric datatype with FloatingPoint/Integral/etc constructors. -- | A literal float of unspecified width. -data Float a = Float { floatContent :: !Scientific - , floatOriginal :: !ByteString - } +newtype Float a = Float { floatContent :: ByteString } deriving (Diffable, Eq, Foldable, Functor, GAlign, Generic1, Mergeable, Ord, Show, Traversable, FreeVariables1) instance Eq1 Data.Syntax.Literal.Float where liftEq = genericLiftEq @@ -80,20 +78,24 @@ removeUnderscores = B.filter (/= '_') dropAlphaSuffix :: ByteString -> ByteString dropAlphaSuffix = B.takeWhile (\x -> x `notElem` ("lLjJiI" :: [Char])) --- | This is the shared function that processes a bytestring representation of a float --- into a Float-wrapped @Scientific@. It takes as its arguments a list of functions, which +-- | This is the shared function that munges a bytestring representation of a float +-- so that it can be parsed to a @Scientific@ later. It takes as its arguments a list of functions, which -- will be some combination of the above 'ByteString -> ByteString' functions. This is meant -- to be called from an @Assignment@, hence the @MonadFail@ constraint. Caveat: the list is -- order-dependent; the rightmost function will be applied first. -buildFloat :: MonadFail m => [ByteString -> ByteString] -> ByteString -> m (Float a) -buildFloat preds val = +normalizeFloatString :: MonadFail m => [ByteString -> ByteString] -> ByteString -> m (Float a) +normalizeFloatString preds val = let munger = appEndo (foldMap Endo preds) - in case readMaybe (unpack (munger val)) of + in case readMaybe @Scientific (unpack (munger val)) of Nothing -> fail ("Invalid floating-point value: " <> show val) - Just s -> pure (Float s val) + Just _ -> pure (Float val) instance Evaluatable Data.Syntax.Literal.Float where - eval (Float s _) = float s + eval (Float s) = do + sci <- case readMaybe (unpack s) of + Just s -> pure s + Nothing -> fail ("Bug: non-normalized float string: " <> show s) + float sci -- Rational literals e.g. `2/3r` newtype Rational a = Rational ByteString diff --git a/src/Language/Go/Assignment.hs b/src/Language/Go/Assignment.hs index f3fef8d2e..6e8fad1cd 100644 --- a/src/Language/Go/Assignment.hs +++ b/src/Language/Go/Assignment.hs @@ -225,7 +225,7 @@ fieldIdentifier :: Assignment fieldIdentifier = makeTerm <$> symbol FieldIdentifier <*> (Syntax.Identifier <$> source) floatLiteral :: Assignment -floatLiteral = makeTerm <$> symbol FloatLiteral <*> (source >>= Literal.buildFloat [Literal.padWithLeadingZero, Literal.dropAlphaSuffix]) +floatLiteral = makeTerm <$> symbol FloatLiteral <*> (source >>= Literal.normalizeFloatString [Literal.padWithLeadingZero, Literal.dropAlphaSuffix]) identifier :: Assignment identifier = makeTerm <$> (symbol Identifier <|> symbol Identifier') <*> (Syntax.Identifier <$> source) diff --git a/src/Language/JSON/Assignment.hs b/src/Language/JSON/Assignment.hs index b8db963e2..2359670d5 100644 --- a/src/Language/JSON/Assignment.hs +++ b/src/Language/JSON/Assignment.hs @@ -50,7 +50,7 @@ array :: Assignment array = makeTerm <$> symbol Array <*> children (Literal.Array <$> many jsonValue) number :: Assignment -number = makeTerm <$> symbol Number <*> (source >>= Literal.buildFloat [Literal.padWithLeadingZero]) +number = makeTerm <$> symbol Number <*> (source >>= Literal.normalizeFloatString [Literal.padWithLeadingZero]) string :: Assignment string = makeTerm <$> symbol String <*> (Literal.TextElement <$> source) diff --git a/src/Language/PHP/Assignment.hs b/src/Language/PHP/Assignment.hs index ed6641864..c0667606a 100644 --- a/src/Language/PHP/Assignment.hs +++ b/src/Language/PHP/Assignment.hs @@ -470,7 +470,7 @@ literal :: Assignment literal = integer <|> float <|> string float :: Assignment -float = makeTerm <$> symbol Float <*> (source >>= Literal.buildFloat [Literal.padWithLeadingZero]) +float = makeTerm <$> symbol Float <*> (source >>= Literal.normalizeFloatString [Literal.padWithLeadingZero]) integer :: Assignment integer = makeTerm <$> symbol Integer <*> (Literal.Integer <$> source) diff --git a/src/Language/Python/Assignment.hs b/src/Language/Python/Assignment.hs index a22905507..f5649ad2e 100644 --- a/src/Language/Python/Assignment.hs +++ b/src/Language/Python/Assignment.hs @@ -364,10 +364,11 @@ concatenatedString :: Assignment concatenatedString = makeTerm <$> symbol ConcatenatedString <*> children (manyTerm string) float :: Assignment -float = makeTerm <$> symbol Float <*> (source >>= Literal.buildFloat [ Literal.padWithLeadingZero - , Literal.padWithTrailingZero - , Literal.dropAlphaSuffix - , Literal.removeUnderscores]) +float = makeTerm <$> symbol Float <*> (source >>= Literal.normalizeFloatString [ Literal.padWithLeadingZero + , Literal.padWithTrailingZero + , Literal.dropAlphaSuffix + , Literal.removeUnderscores + ]) integer :: Assignment integer = makeTerm <$> symbol Integer <*> (Literal.Integer <$> source) diff --git a/src/Language/Ruby/Assignment.hs b/src/Language/Ruby/Assignment.hs index 1cf03032a..f49f52379 100644 --- a/src/Language/Ruby/Assignment.hs +++ b/src/Language/Ruby/Assignment.hs @@ -166,7 +166,7 @@ literal = <|> makeTerm <$> token Grammar.False <*> pure Literal.false <|> makeTerm <$> token Grammar.Nil <*> pure Literal.Null <|> makeTerm <$> symbol Grammar.Integer <*> (Literal.Integer <$> source) - <|> makeTerm <$> symbol Grammar.Float <*> (source >>= Literal.buildFloat [Literal.padWithLeadingZero, Literal.removeUnderscores]) + <|> makeTerm <$> symbol Grammar.Float <*> (source >>= Literal.normalizeFloatString [Literal.padWithLeadingZero, Literal.removeUnderscores]) <|> makeTerm <$> symbol Grammar.Rational <*> (Literal.Rational <$> source) <|> makeTerm <$> symbol Grammar.Complex <*> (Literal.Complex <$> source) -- TODO: Do we want to represent the difference between .. and ... diff --git a/src/Language/TypeScript/Assignment.hs b/src/Language/TypeScript/Assignment.hs index db92c075a..6a7c06888 100644 --- a/src/Language/TypeScript/Assignment.hs +++ b/src/Language/TypeScript/Assignment.hs @@ -325,7 +325,7 @@ importAlias' :: Assignment importAlias' = makeTerm <$> symbol Grammar.ImportAlias <*> children (TypeScript.Syntax.ImportAlias <$> term identifier <*> term (identifier <|> nestedIdentifier)) number :: Assignment -number = makeTerm <$> symbol Grammar.Number <*> (source >>= Literal.buildFloat [Literal.padWithLeadingZero]) +number = makeTerm <$> symbol Grammar.Number <*> (source >>= Literal.normalizeFloatString [Literal.padWithLeadingZero]) string :: Assignment string = makeTerm <$> symbol Grammar.String <*> (Literal.TextElement <$> source) From b67afc5231e595c7846f2384f10173f2e781ab3d Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 7 Mar 2018 18:47:26 -0500 Subject: [PATCH 7/7] Revert "changes to the diffs because the hashes of the values changed" This reverts commit 349dd95d77e8e2f1812ac444e765abca72af4683. --- test/fixtures/python/float.diffA-B.txt | 8 ++++---- test/fixtures/python/float.diffB-A.txt | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/fixtures/python/float.diffA-B.txt b/test/fixtures/python/float.diffA-B.txt index db81ffe15..ac2863db4 100644 --- a/test/fixtures/python/float.diffA-B.txt +++ b/test/fixtures/python/float.diffA-B.txt @@ -6,14 +6,14 @@ ->(Float) } {+(Float)+} {+(Float)+} +{+(Float)+} +{+(Float)+} { (Float) ->(Float) } {+(Float)+} {+(Float)+} -{+(Float)+} -{+(Float)+} -{+(Float)+} -{-(Float)-} +{ (Float) +->(Float) } {-(Float)-} {-(Float)-} {-(Float)-} diff --git a/test/fixtures/python/float.diffB-A.txt b/test/fixtures/python/float.diffB-A.txt index 82f4b4fa2..48cb37153 100644 --- a/test/fixtures/python/float.diffB-A.txt +++ b/test/fixtures/python/float.diffB-A.txt @@ -5,14 +5,14 @@ { (Float) ->(Float) } {+(Float)+} +{+(Float)+} +{+(Float)+} +{+(Float)+} +{+(Float)+} { (Float) ->(Float) } {+(Float)+} {+(Float)+} -{+(Float)+} -{+(Float)+} -{+(Float)+} -{+(Float)+} {-(Float)-} {-(Float)-} {-(Float)-}