1
1
mirror of https://github.com/github/semantic.git synced 2024-11-24 17:04:47 +03:00

🔥 the SyntaxTerm, SyntaxTermF, & SyntaxDiff type synonyms.

This commit is contained in:
Rob Rix 2017-09-13 19:14:01 -04:00
parent d612b22fca
commit 9de429be45
16 changed files with 54 additions and 56 deletions

View File

@ -12,7 +12,6 @@ import Data.JSON.Fields
import Data.Mergeable
import Data.Record
import Patch
import Syntax
import Term
import Text.Show
@ -52,9 +51,6 @@ merge :: (ann, ann) -> syntax (Diff syntax ann) -> Diff syntax ann
merge = (Diff .) . (Merge .) . In
type SyntaxDiff fields = Diff Syntax (Record fields)
diffSum :: (Foldable syntax, Functor syntax) => (forall a. Patch a -> Int) -> Diff syntax ann -> Int
diffSum patchCost = cata $ \ diff -> case diff of
Patch patch -> patchCost patch + sum (sum <$> patch)

View File

@ -26,8 +26,8 @@ import Term
-- | Diff two terms recursively, given functions characterizing the diffing.
diffTerms :: HasField fields Category
=> Both (SyntaxTerm fields) -- ^ A pair of terms representing the old and new state, respectively.
-> SyntaxDiff fields
=> Both (Term Syntax (Record fields)) -- ^ A pair of terms representing the old and new state, respectively.
-> Diff Syntax (Record fields)
diffTerms = decoratingWith getLabel (diffTermsWith algorithmWithTerms comparableByCategory)
-- | Diff two terms by decorating with feature vectors computed using the supplied labelling algebra, and stripping the feature vectors from the resulting diff.
@ -64,9 +64,9 @@ getLabel (In h t) = (Info.category h, case t of
-- | Construct an algorithm to diff a pair of terms.
algorithmWithTerms :: SyntaxTerm fields
-> SyntaxTerm fields
-> Algorithm (Term Syntax) (SyntaxDiff fields) (SyntaxDiff fields)
algorithmWithTerms :: Term Syntax (Record fields)
-> Term Syntax (Record fields)
-> Algorithm (Term Syntax) (Diff Syntax (Record fields)) (Diff Syntax (Record fields))
algorithmWithTerms t1 t2 = case (unwrap t1, unwrap t2) of
(Indexed a, Indexed b) ->
annotate . Indexed <$> byRWS a b

View File

@ -53,13 +53,13 @@ toTuple child | S.Fixed [key,value] <- unwrap child = [termIn (extract child) (S
toTuple child | S.Leaf c <- unwrap child = [termIn (extract child) (S.Comment c)]
toTuple child = pure child
toPublicFieldDefinition :: HasField fields Category => [SyntaxTerm fields] -> Maybe (S.Syntax (SyntaxTerm fields))
toPublicFieldDefinition :: HasField fields Category => [Term S.Syntax (Record fields)] -> Maybe (S.Syntax (Term S.Syntax (Record fields)))
toPublicFieldDefinition children = case break (\x -> category (extract x) == Identifier) children of
(prev, [identifier, assignment]) -> Just $ S.VarAssignment (prev ++ [identifier]) assignment
(_, [_]) -> Just $ S.VarDecl children
_ -> Nothing
toInterface :: HasField fields Category => [SyntaxTerm fields] -> Maybe (S.Syntax (SyntaxTerm fields))
toInterface :: HasField fields Category => [Term S.Syntax (Record fields)] -> Maybe (S.Syntax (Term S.Syntax (Record fields)))
toInterface (id : rest) = case break (\x -> category (extract x) == Other "object_type") rest of
(clauses, [body]) -> Just $ S.Interface id clauses (toList (unwrap body))
_ -> Nothing

View File

@ -5,6 +5,7 @@ import Control.Comonad
import Control.Comonad.Cofree
import Data.Foldable (toList)
import Data.Maybe
import Data.Record
import Data.Source
import Data.Text
import Info
@ -14,8 +15,8 @@ import Term
termAssignment
:: Source -- ^ The source of the term.
-> Category -- ^ The category for the term.
-> [ SyntaxTerm DefaultFields ] -- ^ The child nodes of the term.
-> Maybe (S.Syntax (SyntaxTerm DefaultFields)) -- ^ The resulting term, in Maybe.
-> [ Term S.Syntax (Record DefaultFields) ] -- ^ The child nodes of the term.
-> Maybe (S.Syntax (Term S.Syntax (Record DefaultFields))) -- ^ The resulting term, in Maybe.
termAssignment source category children = case (category, children) of
(Module, [moduleName]) -> Just $ S.Module moduleName []
(Import, [importName]) -> Just $ S.Import importName []

View File

@ -3,6 +3,7 @@ module Language.Ruby where
import Data.Foldable (toList)
import Data.List (partition)
import Data.Record
import Data.Semigroup
import Data.Source
import Data.Text (Text)
@ -14,8 +15,8 @@ import Term
termAssignment
:: Source -- ^ The source of the term.
-> Category -- ^ The category for the term.
-> [ SyntaxTerm DefaultFields ] -- ^ The child nodes of the term.
-> Maybe (S.Syntax (SyntaxTerm DefaultFields)) -- ^ The resulting term, in Maybe.
-> [ Term S.Syntax (Record DefaultFields) ] -- ^ The child nodes of the term.
-> Maybe (S.Syntax (Term S.Syntax (Record DefaultFields))) -- ^ The resulting term, in Maybe.
termAssignment _ category children
= case (category, children) of
(ArgumentPair, [ k, v ] ) -> Just $ S.Pair k v

View File

@ -4,6 +4,7 @@ module Language.TypeScript where
import Control.Comonad (extract)
import Control.Comonad.Cofree (unwrap)
import Data.Foldable (toList)
import Data.Record
import Data.Source
import Data.Text (Text)
import Info
@ -14,8 +15,8 @@ import Term
termAssignment
:: Source -- ^ The source of the term.
-> Category -- ^ The category for the term.
-> [ SyntaxTerm DefaultFields ] -- ^ The child nodes of the term.
-> Maybe (S.Syntax (SyntaxTerm DefaultFields)) -- ^ The resulting term, in Maybe.
-> [ Term S.Syntax (Record DefaultFields) ] -- ^ The child nodes of the term.
-> Maybe (S.Syntax (Term S.Syntax (Record DefaultFields))) -- ^ The resulting term, in Maybe.
termAssignment _ category children =
case (category, children) of
(Assignment, [ identifier, value ]) -> Just $ S.Assignment identifier value

View File

@ -45,14 +45,14 @@ data Parser term where
-> Assignment ast grammar (Term (Union fs) (Record Location)) -- ^ An assignment from AST onto 'Term's.
-> Parser (Term (Union fs) (Record Location)) -- ^ A parser producing 'Term's.
-- | A tree-sitter parser.
TreeSitterParser :: Ptr TS.Language -> Parser (SyntaxTerm DefaultFields)
TreeSitterParser :: Ptr TS.Language -> Parser (Term Syntax (Record DefaultFields))
-- | A parser for 'Markdown' using cmark.
MarkdownParser :: Parser (Term (TermF [] CMarkGFM.NodeType) (Node Markdown.Grammar))
-- | A parser which will parse any input 'Source' into a top-level 'Term' whose children are leaves consisting of the 'Source's lines.
LineByLineParser :: Parser (SyntaxTerm DefaultFields)
LineByLineParser :: Parser (Term Syntax (Record DefaultFields))
-- | Return a 'Language'-specific 'Parser', if one exists, falling back to the 'LineByLineParser'.
parserForLanguage :: Maybe Language -> Parser (SyntaxTerm DefaultFields)
parserForLanguage :: Maybe Language -> Parser (Term Syntax (Record DefaultFields))
parserForLanguage Nothing = LineByLineParser
parserForLanguage (Just language) = case language of
Go -> TreeSitterParser tree_sitter_go
@ -77,6 +77,6 @@ markdownParser = AssignmentParser MarkdownParser Markdown.assignment
-- | A fallback parser that treats a file simply as rows of strings.
lineByLineParser :: Source -> SyntaxTerm DefaultFields
lineByLineParser :: Source -> Term Syntax (Record DefaultFields)
lineByLineParser source = termIn (totalRange source :. Program :. totalSpan source :. Nil) (Indexed (zipWith toLine [1..] (sourceLineRanges source)))
where toLine line range = termIn (range :. Program :. Span (Pos line 1) (Pos line (end range)) :. Nil) (Leaf (toText (slice range source)))

View File

@ -24,9 +24,10 @@ import Data.Foldable (asum)
import Data.JSON.Fields
import qualified Data.Map as Map
import Data.Output
import Data.Record
import Data.Syntax.Algebra (RAlgebra)
import Data.Text (Text)
import Diff (SyntaxDiff)
import Diff
import Info (DefaultFields)
import Renderer.JSON as R
import Renderer.Patch as R
@ -45,8 +46,8 @@ data DiffRenderer output where
JSONDiffRenderer :: DiffRenderer (Map.Map Text Value)
-- | Render to a 'ByteString' formatted as nested s-expressions with patches indicated.
SExpressionDiffRenderer :: DiffRenderer ByteString
-- | “Render” by returning the computed 'SyntaxDiff'. This renderer is not surfaced in the command-line interface, and is intended strictly for tests. Further, as it cannot render à la carte terms, it should be regarded as a (very) short-term hack until such time as we have a better idea for TOCSpec.hs.
IdentityDiffRenderer :: DiffRenderer (Maybe (SyntaxDiff (Maybe Declaration ': DefaultFields)))
-- | “Render” by returning the computed 'Diff'. This renderer is not surfaced in the command-line interface, and is intended strictly for tests. Further, as it cannot render à la carte terms, it should be regarded as a (very) short-term hack until such time as we have a better idea for TOCSpec.hs.
IdentityDiffRenderer :: DiffRenderer (Maybe (Diff Syntax (Record (Maybe Declaration ': DefaultFields))))
deriving instance Eq (DiffRenderer output)
deriving instance Show (DiffRenderer output)
@ -59,8 +60,8 @@ data TermRenderer output where
JSONTermRenderer :: TermRenderer [Value]
-- | Render to a 'ByteString' formatted as nested s-expressions.
SExpressionTermRenderer :: TermRenderer ByteString
-- | “Render” by returning the computed 'SyntaxTerm'. This renderer is not surfaced in the command-line interface, and is intended strictly for tests. Further, as it cannot render à la carte terms, it should be regarded as a (very) short-term hack until such time as we have a better idea for SemanticSpec.hs.
IdentityTermRenderer :: TermRenderer (Maybe (SyntaxTerm DefaultFields))
-- | “Render” by returning the computed 'Term'. This renderer is not surfaced in the command-line interface, and is intended strictly for tests. Further, as it cannot render à la carte terms, it should be regarded as a (very) short-term hack until such time as we have a better idea for SemanticSpec.hs.
IdentityTermRenderer :: TermRenderer (Maybe (Term Syntax (Record DefaultFields)))
deriving instance Eq (TermRenderer output)
deriving instance Show (TermRenderer output)

View File

@ -99,7 +99,7 @@ declaration (In annotation _) = annotation <$ (getField annotation :: Maybe Decl
-- | Compute 'Declaration's for methods and functions in 'Syntax'.
syntaxDeclarationAlgebra :: HasField fields Range => Blob -> RAlgebra (SyntaxTermF fields) (SyntaxTerm fields) (Maybe Declaration)
syntaxDeclarationAlgebra :: HasField fields Range => Blob -> RAlgebra (TermF S.Syntax (Record fields)) (Term S.Syntax (Record fields)) (Maybe Declaration)
syntaxDeclarationAlgebra Blob{..} (In a r) = case r of
S.Function (identifier, _) _ _ -> Just $ FunctionDeclaration (getSource identifier)
S.Method _ (identifier, _) Nothing _ _ -> Just $ MethodDeclaration (getSource identifier)

View File

@ -3,8 +3,6 @@ module Term
( Term(..)
, termIn
, TermF(..)
, SyntaxTerm
, SyntaxTermF
, termSize
, extract
, unwrap
@ -24,7 +22,6 @@ import Data.Functor.Foldable
import Data.JSON.Fields
import Data.Record
import Data.Semigroup ((<>))
import Syntax
import Text.Show
-- | A Term with an abstract syntax tree and an annotation.
@ -33,9 +30,6 @@ newtype Term syntax ann = Term { unTerm :: TermF syntax ann (Term syntax ann) }
data TermF syntax ann recur = In { termAnnotation :: ann, termOut :: syntax recur }
deriving (Eq, Foldable, Functor, Show, Traversable)
-- | A Term with a Syntax leaf and a record of fields.
type SyntaxTerm fields = Term Syntax (Record fields)
type SyntaxTermF fields = TermF Syntax (Record fields)
-- | Return the node count of a term.
termSize :: (Foldable f, Functor f) => Term f annotation -> Int

View File

@ -35,7 +35,7 @@ import qualified TreeSitter.TypeScript as TS
import Info
-- | Returns a TreeSitter parser for the given language and TreeSitter grammar.
treeSitterParser :: Ptr TS.Language -> Blob -> IO (SyntaxTerm DefaultFields)
treeSitterParser :: Ptr TS.Language -> Blob -> IO (Term S.Syntax (Record DefaultFields))
treeSitterParser language blob = bracket TS.ts_document_new TS.ts_document_free $ \ document -> do
TS.ts_document_set_language document language
unsafeUseAsCStringLen (sourceBytes (blobSource blob)) $ \ (sourceBytes, len) -> do
@ -71,13 +71,13 @@ anaM g = a where a = pure . embed <=< traverse a <=< g
-- | Return a parser for a tree sitter language & document.
documentToTerm :: Ptr TS.Language -> Ptr TS.Document -> Blob -> IO (SyntaxTerm DefaultFields)
documentToTerm :: Ptr TS.Language -> Ptr TS.Document -> Blob -> IO (Term S.Syntax (Record DefaultFields))
documentToTerm language document Blob{..} = do
root <- alloca (\ rootPtr -> do
TS.ts_document_root_node_p document rootPtr
peek rootPtr)
toTerm root
where toTerm :: TS.Node -> IO (SyntaxTerm DefaultFields)
where toTerm :: TS.Node -> IO (Term S.Syntax (Record DefaultFields))
toTerm node@TS.Node{..} = do
name <- peekCString nodeType
@ -96,7 +96,7 @@ documentToTerm language document Blob{..} = do
copyNamed = TS.ts_node_copy_named_child_nodes document
copyAll = TS.ts_node_copy_child_nodes document
isNonEmpty :: HasField fields Category => SyntaxTerm fields -> Bool
isNonEmpty :: HasField fields Category => Term S.Syntax (Record fields) -> Bool
isNonEmpty = (/= Empty) . category . extract
nodeRange :: TS.Node -> Range
@ -106,19 +106,19 @@ nodeSpan :: TS.Node -> Span
nodeSpan TS.Node{..} = nodeStartPoint `seq` nodeEndPoint `seq` Span (pointPos nodeStartPoint) (pointPos nodeEndPoint)
where pointPos TS.TSPoint{..} = pointRow `seq` pointColumn `seq` Pos (1 + fromIntegral pointRow) (1 + fromIntegral pointColumn)
assignTerm :: Ptr TS.Language -> Source -> Record DefaultFields -> [ SyntaxTerm DefaultFields ] -> IO [ SyntaxTerm DefaultFields ] -> IO (SyntaxTerm DefaultFields)
assignTerm :: Ptr TS.Language -> Source -> Record DefaultFields -> [ Term S.Syntax (Record DefaultFields) ] -> IO [ Term S.Syntax (Record DefaultFields) ] -> IO (Term S.Syntax (Record DefaultFields))
assignTerm language source annotation children allChildren =
case assignTermByLanguage source (category annotation) children of
Just a -> pure (termIn annotation a)
_ -> defaultTermAssignment source annotation children allChildren
where assignTermByLanguage :: Source -> Category -> [ SyntaxTerm DefaultFields ] -> Maybe (S.Syntax (SyntaxTerm DefaultFields))
where assignTermByLanguage :: Source -> Category -> [ Term S.Syntax (Record DefaultFields) ] -> Maybe (S.Syntax (Term S.Syntax (Record DefaultFields)))
assignTermByLanguage = case languageForTSLanguage language of
Just Language.Go -> Go.termAssignment
Just Ruby -> Ruby.termAssignment
Just TypeScript -> TS.termAssignment
_ -> \ _ _ _ -> Nothing
defaultTermAssignment :: Source -> Record DefaultFields -> [ SyntaxTerm DefaultFields ] -> IO [ SyntaxTerm DefaultFields ] -> IO (SyntaxTerm DefaultFields)
defaultTermAssignment :: Source -> Record DefaultFields -> [ Term S.Syntax (Record DefaultFields) ] -> IO [ Term S.Syntax (Record DefaultFields) ] -> IO (Term S.Syntax (Record DefaultFields))
defaultTermAssignment source annotation children allChildren
| category annotation `elem` operatorCategories = Term . In annotation . S.Operator <$> allChildren
| otherwise = case (category annotation, children) of

View File

@ -20,19 +20,19 @@ spec = parallel $ do
let positively = succ . abs
describe "pqGramDecorator" $ do
prop "produces grams with stems of the specified length" $
\ (term, p, q) -> pqGramDecorator (rhead . termAnnotation) (positively p) (positively q) (term :: SyntaxTerm '[Category]) `shouldSatisfy` all ((== positively p) . length . stem . rhead)
\ (term, p, q) -> pqGramDecorator (rhead . termAnnotation) (positively p) (positively q) (term :: Term Syntax (Record '[Category])) `shouldSatisfy` all ((== positively p) . length . stem . rhead)
prop "produces grams with bases of the specified width" $
\ (term, p, q) -> pqGramDecorator (rhead . termAnnotation) (positively p) (positively q) (term :: SyntaxTerm '[Category]) `shouldSatisfy` all ((== positively q) . length . base . rhead)
\ (term, p, q) -> pqGramDecorator (rhead . termAnnotation) (positively p) (positively q) (term :: Term Syntax (Record '[Category])) `shouldSatisfy` all ((== positively q) . length . base . rhead)
describe "featureVectorDecorator" $ do
prop "produces a vector of the specified dimension" $
\ (term, p, q, d) -> featureVectorDecorator (rhead . termAnnotation) (positively p) (positively q) (positively d) (term :: SyntaxTerm '[Category]) `shouldSatisfy` all ((== (0, abs d)) . bounds . rhead)
\ (term, p, q, d) -> featureVectorDecorator (rhead . termAnnotation) (positively p) (positively q) (positively d) (term :: Term Syntax (Record '[Category])) `shouldSatisfy` all ((== (0, abs d)) . bounds . rhead)
describe "rws" $ do
prop "produces correct diffs" $
\ (as, bs) -> let tas = decorate <$> (as :: [SyntaxTerm '[Category]])
tbs = decorate <$> (bs :: [SyntaxTerm '[Category]])
\ (as, bs) -> let tas = decorate <$> (as :: [Term Syntax (Record '[Category])])
tbs = decorate <$> (bs :: [Term Syntax (Record '[Category])])
root = termIn (Program :. Nil) . Indexed
diff = merge ((Program :. Nil, Program :. Nil)) (Indexed (stripDiff . diffThese <$> rws editDistance canCompare tas tbs)) in
(beforeTerm diff, afterTerm diff) `shouldBe` (Just (root (stripTerm <$> tas)), Just (root (stripTerm <$> tbs)))
@ -43,7 +43,7 @@ spec = parallel $ do
where canCompare a b = termAnnotation a == termAnnotation b
decorate :: SyntaxTerm '[Category] -> SyntaxTerm '[FeatureVector, Category]
decorate :: Term Syntax (Record '[Category]) -> Term Syntax (Record '[FeatureVector, Category])
decorate = defaultFeatureVectorDecorator (category . termAnnotation)
diffThese = these deleting inserting replacing

View File

@ -4,10 +4,12 @@ module DiffSpec where
import Category
import Data.Functor.Both
import Data.Functor.Listable ()
import Data.Record
import RWS
import Diff
import Info
import Interpreter
import Syntax
import Term
import Test.Hspec
import Test.Hspec.LeanCheck
@ -16,19 +18,19 @@ spec :: Spec
spec = parallel $ do
let decorate = defaultFeatureVectorDecorator (category . termAnnotation)
prop "equality is reflexive" $
\ a -> let diff = a :: SyntaxDiff '[Category] in
\ a -> let diff = a :: Term Syntax (Record '[Category]) in
diff `shouldBe` diff
prop "equal terms produce identity diffs" $
\ a -> let term = decorate (a :: SyntaxTerm '[Category]) in
\ a -> let term = decorate (a :: Term Syntax (Record '[Category])) in
diffCost (diffTerms (pure term)) `shouldBe` 0
describe "beforeTerm" $ do
prop "recovers the before term" $
\ a b -> let diff = diffTerms (both a b :: Both (SyntaxTerm '[Category])) in
\ a b -> let diff = diffTerms (both a b :: Both (Term Syntax (Record '[Category]))) in
beforeTerm diff `shouldBe` Just a
describe "afterTerm" $ do
prop "recovers the after term" $
\ a b -> let diff = diffTerms (both a b :: Both (SyntaxTerm '[Category])) in
\ a b -> let diff = diffTerms (both a b :: Both (Term Syntax (Record '[Category]))) in
afterTerm diff `shouldBe` Just b

View File

@ -23,15 +23,15 @@ spec = parallel $ do
diffTerms (both termA termB) `shouldBe` replacing termA termB
prop "produces correct diffs" $
\ a b -> let diff = diffTerms (unListableF <$> both a b :: Both (SyntaxTerm '[Category])) in
\ a b -> let diff = diffTerms (unListableF <$> both a b :: Both (Term Syntax (Record '[Category]))) in
(beforeTerm diff, afterTerm diff) `shouldBe` (Just (unListableF a), Just (unListableF b))
prop "constructs zero-cost diffs of equal terms" $
\ a -> let term = (unListableF a :: SyntaxTerm '[Category])
\ a -> let term = (unListableF a :: Term Syntax (Record '[Category]))
diff = diffTerms (pure term) in
diffCost diff `shouldBe` 0
it "produces unbiased insertions within branches" $
let term s = Term ((StringLiteral :. Nil) `In` Indexed [ Term ((StringLiteral :. Nil) `In` Leaf s) ]) :: SyntaxTerm '[Category]
let term s = Term ((StringLiteral :. Nil) `In` Indexed [ Term ((StringLiteral :. Nil) `In` Leaf s) ]) :: Term Syntax (Record '[Category])
root = termIn (Program :. Nil) . Indexed in
diffTerms (both (root [ term "b" ]) (root [ term "a", term "b" ])) `shouldBe` merge ((Program :. Nil, Program :. Nil)) (Indexed [ inserting (term "a"), cata (\ (In a r) -> merge (a, a) r) (term "b") ])

View File

@ -160,8 +160,8 @@ spec = parallel $ do
toOutput output `shouldBe` ("{\"changes\":{\"test/fixtures/toc/markdown/headings.A.md -> test/fixtures/toc/markdown/headings.B.md\":[{\"span\":{\"start\":[5,1],\"end\":[7,10]},\"category\":\"Heading 2\",\"term\":\"Two\",\"changeType\":\"added\"},{\"span\":{\"start\":[9,1],\"end\":[10,4]},\"category\":\"Heading 1\",\"term\":\"Final\",\"changeType\":\"added\"}]},\"errors\":{}}\n" :: ByteString)
type Diff' = SyntaxDiff (Maybe Declaration ': DefaultFields)
type Term' = SyntaxTerm (Maybe Declaration ': DefaultFields)
type Diff' = Diff Syntax (Record (Maybe Declaration ': DefaultFields))
type Term' = Term Syntax (Record (Maybe Declaration ': DefaultFields))
numTocSummaries :: Diff' -> Int
numTocSummaries diff = length $ filter isValidSummary (diffTOC diff)

View File

@ -3,6 +3,8 @@ module TermSpec where
import Category
import Data.Functor.Listable
import Data.Record
import Syntax
import Term
import Test.Hspec (Spec, describe, parallel)
import Test.Hspec.Expectations.Pretty
@ -12,4 +14,4 @@ spec :: Spec
spec = parallel $ do
describe "Term" $ do
prop "equality is reflexive" $
\ a -> unListableF a `shouldBe` (unListableF a :: SyntaxTerm '[Category])
\ a -> unListableF a `shouldBe` (unListableF a :: Term Syntax (Record '[Category]))