Merge pull request #1479 from github/import-structure

Give Imports structure and expose them in tags/toc output

src/Analysis/Declaration.hs

@@ -29,6 +29,7 @@ import qualified Language.Markdown.Syntax as Markdown
 data Declaration
   = MethodDeclaration   { declarationIdentifier :: T.Text, declarationText :: T.Text, declarationLanguage :: Maybe Language, declarationReceiver :: Maybe T.Text }
   | ClassDeclaration    { declarationIdentifier :: T.Text, declarationText :: T.Text, declarationLanguage :: Maybe Language }
+  | ImportDeclaration   { declarationIdentifier :: T.Text, declarationText :: T.Text, declarationLanguage :: Maybe Language }
   | FunctionDeclaration { declarationIdentifier :: T.Text, declarationText :: T.Text, declarationLanguage :: Maybe Language }
   | HeadingDeclaration  { declarationIdentifier :: T.Text, declarationText :: T.Text, declarationLanguage :: Maybe Language, declarationLevel :: Int }
   | ErrorDeclaration    { declarationIdentifier :: T.Text, declarationText :: T.Text, declarationLanguage :: Maybe Language }
@@ -116,6 +117,12 @@ instance CustomHasDeclaration Declaration.Class where
     = Just $ ClassDeclaration (getSource identifierAnn) (getClassSource blob (In ann decl)) blobLanguage
     where getSource = toText . flip Source.slice blobSource . getField
 
+instance CustomHasDeclaration Declaration.Import where
+  customToDeclaration blob@Blob{..} ann decl@(Declaration.Import (Term (In fromAnn _), _) _ _)
+    -- Imports
+    = Just $ ImportDeclaration (getSource fromAnn) (getImportSource blob (In ann decl)) blobLanguage
+    where getSource = toText . flip Source.slice blobSource . getField
+
 -- | Produce a 'Declaration' for 'Union's using the 'HasDeclaration' instance & therefore using a 'CustomHasDeclaration' instance when one exists & the type is listed in 'DeclarationStrategy'.
 instance Apply HasDeclaration fs => CustomHasDeclaration (Union fs) where
   customToDeclaration blob ann = apply (Proxy :: Proxy HasDeclaration) (toDeclaration blob ann)
@@ -139,6 +146,7 @@ class HasDeclarationWithStrategy (strategy :: Strategy) syntax where
 type family DeclarationStrategy syntax where
   DeclarationStrategy Declaration.Class = 'Custom
   DeclarationStrategy Declaration.Function = 'Custom
+  DeclarationStrategy Declaration.Import = 'Custom
   DeclarationStrategy Declaration.Method = 'Custom
   DeclarationStrategy Markdown.Heading = 'Custom
   DeclarationStrategy Syntax.Error = 'Custom
@@ -175,3 +183,10 @@ getClassSource Blob{..} (In a r)
         bodyRange = getField <$> case r of
           Declaration.Class _ _ _ (Term (In a' _), _) -> Just a'
     in maybe mempty (T.stripEnd . toText . flip Source.slice blobSource . subtractRange declRange) bodyRange
+
+getImportSource :: (HasField fields Range) => Blob -> TermF Declaration.Import (Record fields) (Term syntax (Record fields), a) -> T.Text
+getImportSource Blob{..} (In a r)
+  = let declRange = getField a
+        bodyRange = getField <$> case r of
+          Declaration.Import (Term (In a' _), _) _ _ -> Just a'
+    in maybe mempty (T.stripEnd . toText . flip Source.slice blobSource . subtractRange declRange) bodyRange

src/Data/Syntax.hs

@@ -45,6 +45,12 @@ makeTerm a = makeTerm' a . inj
 makeTerm' :: (HasCallStack, Semigroup a, Foldable f) => a -> f (Term f a) -> Term f a
 makeTerm' a f = termIn (sconcat (a :| (termAnnotation <$> toList f))) f
 
+-- | Lift syntax and an annotation into a term, injecting the syntax into a union & ensuring the annotation encompasses all children. Removes extra structure if term is a list of a single item.
+makeTerm'' :: (HasCallStack, f :< fs, Semigroup a, Apply Foldable fs, Foldable f) => a -> f (Term (Union fs) a) -> Term (Union fs) a
+makeTerm'' a children = case toList children of
+  [x] -> x
+  _ -> makeTerm' a (inj children)
+
 -- | Lift non-empty syntax into a term, injecting the syntax into a union & appending all subterms’.annotations to make the new term’s annotation.
 makeTerm1 :: (HasCallStack, f :< fs, Semigroup a, Apply Foldable fs) => f (Term (Union fs) a) -> Term (Union fs) a
 makeTerm1 = makeTerm1' . inj

src/Data/Syntax/Declaration.hs

@@ -251,7 +251,7 @@ instance Show1 Comprehension where liftShowsPrec = genericLiftShowsPrec
 instance (MonadFail m) => Eval t v m Comprehension
 
 -- | Import declarations.
-data Import a = Import { importContent :: ![a] }
+data Import a = Import { importFrom :: !a, importAlias :: !a, importSymbols :: ![a] }
   deriving (Diffable, Eq, Foldable, Functor, GAlign, Generic1, Mergeable, Ord, Show, Traversable, FreeVariables1)
 
 instance Eq1 Import where liftEq = genericLiftEq
@@ -261,6 +261,18 @@ instance Show1 Import where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for Import
 instance (MonadFail m) => Eval t v m Import
 
+-- | An imported symbol
+data ImportSymbol a = ImportSymbol { importSymbolName :: !a, importSymbolAlias :: !a }
+  deriving (Diffable, Eq, Foldable, Functor, GAlign, Generic1, Mergeable, Ord, Show, Traversable, FreeVariables1)
+
+instance Eq1 ImportSymbol where liftEq = genericLiftEq
+instance Ord1 ImportSymbol where liftCompare = genericLiftCompare
+instance Show1 ImportSymbol where liftShowsPrec = genericLiftShowsPrec
+
+-- TODO: Implement Eval instance for ImportSymbol
+instance (MonadFail m) => Eval t v m ImportSymbol
+
+
 -- | A declared type (e.g. `a []int` in Go).
 data Type a = Type { typeName :: !a, typeKind :: !a }
   deriving (Diffable, Eq, Foldable, Functor, GAlign, Generic1, Mergeable, Ord, Show, Traversable, FreeVariables1)

src/Language/Go/Assignment.hs

@@ -11,7 +11,7 @@ import qualified Assigning.Assignment as Assignment
 import Data.Functor (void)
 import Data.List.NonEmpty (some1)
 import Data.Record
-import Data.Syntax (contextualize, emptyTerm, parseError, handleError, infixContext, makeTerm, makeTerm', makeTerm1)
+import Data.Syntax (contextualize, emptyTerm, parseError, handleError, infixContext, makeTerm, makeTerm', makeTerm'', makeTerm1)
 import qualified Data.Syntax as Syntax
 import qualified Data.Syntax.Comment as Comment
 import qualified Data.Syntax.Declaration as Declaration
@@ -152,8 +152,6 @@ expressionChoices =
   , incStatement
   , identifier
   , importDeclaration
-  , importSpec
-  , importSpecList
   , indexExpression
   , interpretedStringLiteral
   , intLiteral
@@ -208,16 +206,10 @@ types =
          ]
 
 identifiers :: Assignment
-identifiers = mk <$> location <*> manyTerm identifier
+identifiers = makeTerm'' <$> location <*> manyTerm identifier
-  where
-    mk _ [a] = a
-    mk loc children = makeTerm loc children
 
 expressions :: Assignment
-expressions = mk <$> location <*> manyTerm expression
+expressions = makeTerm'' <$> location <*> manyTerm expression
-  where
-    mk _ [a] = a
-    mk loc children = makeTerm loc children
 
 
 -- Literals
@@ -389,13 +381,12 @@ functionDeclaration =  makeTerm <$> (symbol FunctionDeclaration <|> symbol FuncL
     returnParameters = makeTerm <$> symbol ParameterList <*> children (manyTerm expression)
 
 importDeclaration :: Assignment
-importDeclaration = makeTerm <$> symbol ImportDeclaration <*> children (Declaration.Import <$> manyTerm expression)
+importDeclaration = makeTerm'' <$> symbol ImportDeclaration <*> children (manyTerm (importSpec <|> importSpecList))
-
+  where
-importSpec :: Assignment
+    importSpec = makeTerm <$> symbol ImportSpec <*> children (namedImport <|> plainImport)
-importSpec = symbol ImportSpec *> children expressions
+    namedImport = flip Declaration.Import <$> expression <*> expression <*> pure []
-
+    plainImport = Declaration.Import <$> expression <*> emptyTerm <*> pure []
-importSpecList :: Assignment
+    importSpecList = makeTerm <$> symbol ImportSpecList <*> children (manyTerm (importSpec <|> comment))
-importSpecList = symbol ImportSpecList *> children expressions
 
 indexExpression :: Assignment
 indexExpression = makeTerm <$> symbol IndexExpression <*> children (Expression.Subscript <$> expression <*> manyTerm expression)

src/Language/Python/Assignment.hs

@@ -12,7 +12,7 @@ import Data.Functor (void)
 import Data.List.NonEmpty (some1)
 import Data.Maybe (fromMaybe)
 import Data.Record
-import Data.Syntax (contextualize, emptyTerm, handleError, infixContext, makeTerm, makeTerm', makeTerm1, parseError, postContextualize)
+import Data.Syntax (contextualize, emptyTerm, handleError, infixContext, makeTerm, makeTerm', makeTerm'', makeTerm1, parseError, postContextualize)
 import qualified Data.Syntax as Syntax
 import qualified Data.Syntax.Comment as Comment
 import qualified Data.Syntax.Declaration as Declaration
@@ -374,10 +374,12 @@ comment :: Assignment
 comment = makeTerm <$> symbol Comment <*> (Comment.Comment <$> source)
 
 import' :: Assignment
-import' =  makeTerm <$> symbol ImportStatement <*> children (Declaration.Import <$> manyTerm expression)
+import' =  makeTerm'' <$> symbol ImportStatement <*> children (manyTerm (aliasedImport <|> plainImport))
-       <|> makeTerm <$> symbol ImportFromStatement <*> children (Declaration.Import <$> manyTerm expression)
+       <|> makeTerm <$> symbol ImportFromStatement <*> children (Declaration.Import <$> expression <*> emptyTerm <*> (pure <$> (wildCard <|> expressions)))
-       <|> makeTerm <$> symbol AliasedImport <*> children (flip Statement.Let <$> term expression <*> term expression <*> emptyTerm)
+  where
-       <|> makeTerm <$> symbol WildcardImport <*> (Syntax.Identifier <$> source)
+    aliasedImport = makeTerm <$> symbol AliasedImport <*> children (Declaration.Import <$> expression <*> expression <*> pure [])
+    plainImport = makeTerm <$> symbol DottedName <*> children (Declaration.Import <$> expression <*> emptyTerm <*> pure [])
+    wildCard = makeTerm <$> symbol WildcardImport <*> (Syntax.Identifier <$> source)
 
 assertStatement :: Assignment
 assertStatement = makeTerm <$> symbol AssertStatement <*> children (Expression.Call <$> pure [] <*> (makeTerm <$> symbol AnonAssert <*> (Syntax.Identifier <$> source)) <*> manyTerm expression <*> emptyTerm)

src/Language/Ruby/Assignment.hs

@@ -9,10 +9,12 @@ module Language.Ruby.Assignment
 import Assigning.Assignment hiding (Assignment, Error)
 import qualified Assigning.Assignment as Assignment
 import Data.Maybe (fromMaybe)
+import Control.Monad (guard)
 import Data.Record
 import Data.Functor (void)
 import Data.List.NonEmpty (some1)
-import Data.Syntax (contextualize, postContextualize, emptyTerm, parseError, handleError, infixContext, makeTerm, makeTerm', makeTerm1)
+import Data.List (elem)
+import Data.Syntax (contextualize, postContextualize, emptyTerm, parseError, handleError, infixContext, makeTerm, makeTerm', makeTerm'', makeTerm1)
 import qualified Data.Syntax as Syntax
 import qualified Data.Syntax.Comment as Comment
 import qualified Data.Syntax.Declaration as Declaration
@@ -29,6 +31,7 @@ type Syntax = '[
     Comment.Comment
   , Declaration.Class
   , Declaration.Function
+  , Declaration.Import
   , Declaration.Method
   , Declaration.Module
   , Expression.Arithmetic
@@ -136,14 +139,10 @@ expressionChoices =
     mk s construct = makeTerm <$> symbol s <*> children ((construct .) . fromMaybe <$> emptyTerm <*> optional (symbol ArgumentList *> children expressions))
 
 expressions :: Assignment
-expressions = mk <$> location <*> many expression
+expressions = makeTerm'' <$> location <*> many expression
-  where mk _ [a] = a
-        mk loc children = makeTerm loc children
 
 parenthesized_expressions :: Assignment
-parenthesized_expressions = mk <$> symbol ParenthesizedStatements <*> children (many expression)
+parenthesized_expressions = makeTerm'' <$> symbol ParenthesizedStatements <*> children (many expression)
-  where mk _ [a] = a
-        mk loc children = makeTerm loc children
 
 identifier :: Assignment
 identifier =
@@ -300,9 +299,15 @@ pair :: Assignment
 pair =   makeTerm <$> symbol Pair <*> children (Literal.KeyValue <$> expression <*> (expression <|> emptyTerm))
 
 methodCall :: Assignment
-methodCall = makeTerm <$> symbol MethodCall <*> children (Expression.Call <$> pure [] <*> expression <*> args <*> (block <|> emptyTerm))
+methodCall = makeTerm' <$> symbol MethodCall <*> children (require <|> regularCall)
   where
+    regularCall = inj <$> (Expression.Call <$> pure [] <*> expression <*> args <*> (block <|> emptyTerm))
+    require = inj <$> (symbol Identifier *> do
+      s <- source
+      guard (elem s ["autoload", "load", "require", "require_relative"])
+      Declaration.Import <$> args' <*> emptyTerm <*> pure [])
     args = (symbol ArgumentList <|> symbol ArgumentListWithParens) *> children (many expression) <|> pure []
+    args' = makeTerm'' <$> (symbol ArgumentList <|> symbol ArgumentListWithParens) <*> children (many expression) <|> emptyTerm
 
 call :: Assignment
 call = makeTerm <$> symbol Call <*> children (Expression.MemberAccess <$> expression <*> (expression <|> args))

src/Language/TypeScript/Assignment.hs

@@ -10,7 +10,7 @@ import Assigning.Assignment hiding (Assignment, Error)
 import qualified Assigning.Assignment as Assignment
 import Data.Maybe (fromMaybe, catMaybes)
 import Data.Record
-import Data.Syntax (emptyTerm, handleError, parseError, infixContext, makeTerm, makeTerm', makeTerm1, contextualize, postContextualize)
+import Data.Syntax (emptyTerm, handleError, parseError, infixContext, makeTerm, makeTerm', makeTerm'', makeTerm1, contextualize, postContextualize)
 import qualified Data.Syntax as Syntax
 import qualified Data.Syntax.Comment as Comment
 import qualified Data.Syntax.Declaration as Declaration
@@ -166,6 +166,7 @@ type Syntax = '[
   , TypeScript.Syntax.Update
   , TypeScript.Syntax.ComputedPropertyName
   , TypeScript.Syntax.Decorator
+  , Declaration.ImportSymbol
   , []
   ]
 
@@ -543,10 +544,8 @@ statementBlock :: Assignment
 statementBlock = makeTerm <$> symbol StatementBlock <*> children (manyTerm statement)
 
 classBodyStatements :: Assignment
-classBodyStatements = mk <$> symbol ClassBody <*> children (contextualize' <$> Assignment.manyThrough comment (postContextualize' <$> (concat <$> many ((\as b -> as ++ [b]) <$> manyTerm decorator <*> term (methodDefinition <|> publicFieldDefinition <|> methodSignature <|> indexSignature <|> abstractMethodSignature))) <*> many comment))
+classBodyStatements = makeTerm'' <$> symbol ClassBody <*> children (contextualize' <$> Assignment.manyThrough comment (postContextualize' <$> (concat <$> many ((\as b -> as ++ [b]) <$> manyTerm decorator <*> term (methodDefinition <|> publicFieldDefinition <|> methodSignature <|> indexSignature <|> abstractMethodSignature))) <*> many comment))
   where
-    mk _ [a] = a
-    mk loc children = makeTerm loc children
     contextualize' (cs, formalParams) = case nonEmpty cs of
       Just cs -> toList cs ++ formalParams
       Nothing -> formalParams
@@ -616,19 +615,29 @@ statementIdentifier :: Assignment
 statementIdentifier = makeTerm <$> symbol StatementIdentifier <*> (Syntax.Identifier <$> source)
 
 importStatement :: Assignment
-importStatement = makeTerm <$> symbol Grammar.ImportStatement <*> children (Declaration.Import <$> (((\a b -> [a, b]) <$> term importClause <*> term fromClause) <|> (pure <$> term (importRequireClause <|> string))))
+importStatement =  makeImport <$> symbol Grammar.ImportStatement <*> children ((,) <$> importClause' <*> term string)
+               <|> makeTerm <$> symbol Grammar.ImportStatement <*> children requireClause'
+               <|> makeTerm <$> symbol Grammar.ImportStatement <*> children (declarationImport <$> emptyTerm <*> pure [] <*> term string)
+  where
+    makeImport loc ([clause], from) = makeTerm loc (clause from)
+    makeImport loc (clauses, from) = makeTerm loc $ fmap (\c -> makeTerm loc (c from)) clauses
+    importClause' = symbol Grammar.ImportClause *> children (
+      namedImports'
+      <|> (pure <$> namespace')
+      <|> ((\a b -> [a, b]) <$> identifier' <*> namespace')
+      <|> ((:) <$> identifier' <*> namedImports')
+      <|> (pure <$> identifier')
+      )
+    requireClause' = symbol Grammar.ImportRequireClause *> children (declarationImport <$> term identifier <*> pure [] <*> term string)
+    identifier' = (declarationImport <$> emptyTerm <*> (pure <$> term identifier))
+    namespace' = (declarationImport <$> term namespaceImport <*> pure [])
 
-importClause :: Assignment
+    namedImports' = symbol Grammar.NamedImports *> children (many (declarationImport <$> emptyTerm <*> (pure <$> term importSymbol)))
-importClause = makeTerm <$> symbol Grammar.ImportClause <*> children (TypeScript.Syntax.ImportClause <$> (((\a b -> [a, b]) <$> term identifier <*> term (namespaceImport <|> namedImports)) <|> (pure <$> term (namespaceImport <|> namedImports <|> identifier))))
+    importSymbol = makeTerm <$> symbol Grammar.ImportSpecifier <*> children (Declaration.ImportSymbol <$> term identifier <*> (term identifier <|> emptyTerm))
+    namespaceImport = symbol Grammar.NamespaceImport *> children (term identifier)
 
-namedImports :: Assignment
-namedImports = makeTerm <$> symbol Grammar.NamedImports <*> children (TypeScript.Syntax.NamedImports <$> manyTerm importExportSpecifier)
 
-namespaceImport :: Assignment
+    declarationImport alias symbols from = Declaration.Import from alias symbols
-namespaceImport = makeTerm <$> symbol Grammar.NamespaceImport <*> children (TypeScript.Syntax.NamespaceImport <$> term identifier)
-
-importRequireClause :: Assignment
-importRequireClause = makeTerm <$> symbol Grammar.ImportRequireClause <*> children (TypeScript.Syntax.ImportRequireClause <$> term identifier <*> term string)
 
 debuggerStatement :: Assignment
 debuggerStatement = makeTerm <$> symbol Grammar.DebuggerStatement <*> (TypeScript.Syntax.Debugger <$ source)

src/Rendering/Renderer.hs

@@ -15,6 +15,8 @@ module Rendering.Renderer
 , renderDOTDiff
 , renderDOTTerm
 , Summaries(..)
+, TagFields(..)
+, defaultTagFields
 ) where
 
 import Data.Aeson (Value)
@@ -49,7 +51,7 @@ data TermRenderer output where
   -- | Render to a 'ByteString' formatted as nested s-expressions.
   SExpressionTermRenderer :: TermRenderer ByteString
   -- | Render to a list of tags.
-  TagsTermRenderer :: TermRenderer [Value]
+  TagsTermRenderer :: TagFields -> TermRenderer [Value]
   -- | Render to a 'ByteString' formatted as a DOT description of the term.
   DOTTermRenderer :: TermRenderer ByteString
 

src/Rendering/TOC.hs

@@ -166,7 +166,12 @@ renderToCDiff blobs = uncurry Summaries . bimap toMap toMap . List.partition isV
         summaryKey = T.pack $ pathKeyForBlobPair blobs
 
 diffTOC :: (HasField fields (Maybe Declaration), HasField fields Span, Foldable f, Functor f) => Diff f (Record fields) (Record fields) -> [TOCSummary]
-diffTOC = mapMaybe entrySummary . dedupe . tableOfContentsBy declaration
+diffTOC = mapMaybe entrySummary . dedupe . filter removeImports . tableOfContentsBy declaration
+  where
+    removeImports :: HasField fields (Maybe Declaration) => Entry (Record fields) -> Bool
+    removeImports entry = case getDeclaration (entryPayload entry) of
+      Just (ImportDeclaration{..}) -> False
+      _ -> True
 
 renderToCTerm :: (HasField fields (Maybe Declaration), HasField fields Span, Foldable f, Functor f) => Blob -> Term f (Record fields) -> Summaries
 renderToCTerm Blob{..} = uncurry Summaries . bimap toMap toMap . List.partition isValidSummary . termToC
@@ -181,6 +186,7 @@ renderToCTerm Blob{..} = uncurry Summaries . bimap toMap toMap . List.partition
 toCategoryName :: Declaration -> T.Text
 toCategoryName declaration = case declaration of
   ClassDeclaration{} -> "Class"
+  ImportDeclaration{} -> "Import"
   FunctionDeclaration{} -> "Function"
   MethodDeclaration{} -> "Method"
   HeadingDeclaration _ _ _ l -> "Heading " <> T.pack (show l)

src/Rendering/Tag.hs

@@ -1,6 +1,8 @@
 {-# LANGUAGE DataKinds, MultiParamTypeClasses, RankNTypes, ScopedTypeVariables, TypeFamilies, TypeOperators, UndecidableInstances #-}
 module Rendering.Tag
 ( renderToTags
+, TagFields(..)
+, defaultTagFields
 ) where
 
 import Analysis.Declaration
@@ -15,33 +17,57 @@ import qualified Data.Text as T
 import Rendering.TOC
 
 -- | Render a 'Term' to a ctags like output (See 'Tag').
-renderToTags :: (HasField fields (Maybe Declaration), HasField fields Span, Foldable f, Functor f) => Blob -> Term f (Record fields) -> [Value]
+renderToTags :: (HasField fields (Maybe Declaration), HasField fields Span, Foldable f, Functor f) => TagFields -> Blob -> Term f (Record fields) -> [Value]
-renderToTags Blob{..} = fmap toJSON . termToC blobPath
+renderToTags fields Blob{..} = fmap toJSON . termToC fields blobPath
   where
-    termToC :: (HasField fields (Maybe Declaration), HasField fields Span, Foldable f, Functor f) => FilePath -> Term f (Record fields) -> [Tag]
+    termToC :: (HasField fields (Maybe Declaration), HasField fields Span, Foldable f, Functor f) => TagFields -> FilePath -> Term f (Record fields) -> [Tag]
-    termToC path = mapMaybe (tagSummary path "unchanged") . termTableOfContentsBy declaration
+    termToC fields path = mapMaybe (tagSummary fields path "unchanged") . termTableOfContentsBy declaration
 
 -- | Construct a 'Tag' from a node annotation and a change type label.
-tagSummary :: (HasField fields (Maybe Declaration), HasField fields Span) => FilePath -> T.Text -> Record fields -> Maybe Tag
+tagSummary :: (HasField fields (Maybe Declaration), HasField fields Span) => TagFields -> FilePath -> T.Text -> Record fields -> Maybe Tag
-tagSummary path _ record = case getDeclaration record of
+tagSummary TagFields{..} path _ record = case getDeclaration record of
   Just ErrorDeclaration{} -> Nothing
-  Just declaration -> Just $ Tag (declarationIdentifier declaration) (T.pack path) (T.pack . show <$> declarationLanguage declaration) (toCategoryName declaration) (declarationText declaration) (getField record)
+  Just declaration -> Just $ Tag
+    { tagSymbol = when tagFieldsShowSymbol (declarationIdentifier declaration)
+    , tagPath = when tagFieldsShowPath (T.pack path)
+    , tagLanguage = if tagFieldsShowLanguage then (T.pack . show <$> declarationLanguage declaration) else Nothing
+    , tagKind = when tagFieldsShowKind (toCategoryName declaration)
+    , tagLine = when tagFieldsShowLine (declarationText declaration)
+    , tagSpan = when tagFieldsShowSpan (getField record)
+    }
   _ -> Nothing
 
-data Tag
+data Tag = Tag
-  = Tag { tagSymbol :: T.Text
+  { tagSymbol :: Maybe T.Text
-        , tagPath :: T.Text
+  , tagPath :: Maybe T.Text
   , tagLanguage :: Maybe T.Text
-        , tagKind :: T.Text
+  , tagKind :: Maybe T.Text
-        , tagLine :: T.Text
+  , tagLine :: Maybe T.Text
-        , tagSpan :: Span
+  , tagSpan :: Maybe Span
-      }
+  } deriving (Generic, Eq, Show)
-  deriving (Generic, Eq, Show)
 
 instance ToJSON Tag where
-  toJSON Tag{..} = object [ "symbol" .= tagSymbol
+  toJSON Tag{..} = objectWithoutNulls [ "symbol" .= tagSymbol
                           , "path" .= tagPath
                           , "language" .= tagLanguage
                           , "kind" .= tagKind
                           , "line" .= tagLine
                           , "span" .= tagSpan ]
+    where objectWithoutNulls = object . filter (\(_, v) -> v /= Null)
+
+when :: Bool -> a -> Maybe a
+when True a = Just a
+when False _ = Nothing
+
+data TagFields = TagFields
+  { tagFieldsShowSymbol :: Bool
+  , tagFieldsShowPath :: Bool
+  , tagFieldsShowLanguage :: Bool
+  , tagFieldsShowKind :: Bool
+  , tagFieldsShowLine :: Bool
+  , tagFieldsShowSpan :: Bool
+  }
+  deriving (Eq, Show)
+
+defaultTagFields :: TagFields
+defaultTagFields = TagFields True True True True True True

src/Semantic.hs

@@ -55,7 +55,7 @@ parseBlob renderer blob@Blob{..}
     ToCTermRenderer         -> decorate (declarationAlgebra blob)                     >=> render (renderToCTerm  blob)
     JSONTermRenderer        -> decorate constructorLabel >=> decorate identifierLabel >=> render (renderJSONTerm blob)
     SExpressionTermRenderer -> decorate constructorLabel . (Nil <$)                   >=> render renderSExpressionTerm
-    TagsTermRenderer        -> decorate (declarationAlgebra blob)                     >=> render (renderToTags blob)
+    TagsTermRenderer fields -> decorate (declarationAlgebra blob)                     >=> render (renderToTags fields blob)
     DOTTermRenderer         ->                                                            render (renderDOTTerm blob)
   | otherwise = throwError (SomeException (NoLanguageForBlob blobPath))
 

src/Semantic/CLI.hs

@@ -26,6 +26,7 @@ import System.IO (Handle, stdin, stdout)
 import qualified Semantic (parseBlobs, diffBlobPairs)
 import Text.Read
 
+
 main :: IO ()
 main = customExecParser (prefs showHelpOnEmpty) arguments >>= uncurry Task.runTaskWithOptions
 
@@ -76,7 +77,11 @@ arguments = info (version <*> helper <*> ((,) <$> optionsParser <*> argumentsPar
       <$> (   flag  (SomeRenderer SExpressionTermRenderer) (SomeRenderer SExpressionTermRenderer) (long "sexpression" <> help "Output s-expression parse trees (default)")
           <|> flag'                                        (SomeRenderer JSONTermRenderer)        (long "json" <> help "Output JSON parse trees")
           <|> flag'                                        (SomeRenderer ToCTermRenderer)         (long "toc" <> help "Output JSON table of contents summary")
-          <|> flag'                                        (SomeRenderer TagsTermRenderer)        (long "tags" <> help "Output JSON tags/symbols")
+          <|> flag'                                        (SomeRenderer . TagsTermRenderer)      (long "tags" <> help "Output JSON tags/symbols")
+              <*> (   option tagFieldsReader (  long "fields"
+                                             <> help "Comma delimited list of specific fields to return (tags output only)."
+                                             <> metavar "FIELDS")
+                  <|> pure defaultTagFields)
           <|> flag'                                        (SomeRenderer DOTTermRenderer)         (long "dot" <> help "Output the term as a DOT graph"))
       <*> (   Right <$> some (argument filePathReader (metavar "FILES..."))
           <|> pure (Left stdin) )
@@ -91,3 +96,15 @@ arguments = info (version <*> helper <*> ((,) <$> optionsParser <*> argumentsPar
     optionsReader options = eitherReader $ \ str -> maybe (Left ("expected one of: " <> intercalate ", " (fmap fst options))) (Right . snd) (find ((== str) . fst) options)
     options options fields = option (optionsReader options) (fields <> showDefaultWith (findOption options) <> metavar (intercalate "|" (fmap fst options)))
     findOption options value = maybe "" fst (find ((== value) . snd) options)
+
+    -- Example: semantic parse --tags --fields=symbol,path,language,kind,line,span
+    tagFieldsReader = eitherReader parseTagFields
+    parseTagFields arg = let fields = splitWhen (== ',') arg in
+                      Right $ TagFields
+                        { tagFieldsShowSymbol = (elem "symbol" fields)
+                        , tagFieldsShowPath = (elem "path" fields)
+                        , tagFieldsShowLanguage = (elem "language" fields)
+                        , tagFieldsShowKind = (elem "kind" fields)
+                        , tagFieldsShowLine = (elem "line" fields)
+                        , tagFieldsShowSpan = (elem "span" fields)
+                        }

test/fixtures/go/grouped-import-declarations.diffA-B.txt

@@ -1,16 +1,19 @@
 (Program
   (Module
     (Identifier))
+  (
     (Import
-    (
     { (TextElement)
     ->(TextElement) }
+      (Empty))
+    (Import
     { (TextElement)
     ->(TextElement) }
-      (
+      (Empty))
-        (Identifier)
+    (Import
     { (TextElement)
-      ->(TextElement) })))
+    ->(TextElement) }
+      (Identifier)))
   (Function
     (Empty)
     (Identifier)

test/fixtures/go/grouped-import-declarations.diffB-A.txt

@@ -1,16 +1,19 @@
 (Program
   (Module
     (Identifier))
+  (
     (Import
-    (
     { (TextElement)
     ->(TextElement) }
+      (Empty))
+    (Import
     { (TextElement)
     ->(TextElement) }
-      (
+      (Empty))
-        (Identifier)
+    (Import
     { (TextElement)
-      ->(TextElement) })))
+    ->(TextElement) }
+      (Identifier)))
   (Function
     (Empty)
     (Identifier)

test/fixtures/go/grouped-import-declarations.parseA.txt

@@ -1,13 +1,16 @@
 (Program
   (Module
     (Identifier))
+  (
     (Import
-    (
       (TextElement)
+      (Empty))
+    (Import
       (TextElement)
-      (
+      (Empty))
-        (Identifier)
+    (Import
-        (TextElement))))
+      (TextElement)
+      (Identifier)))
   (Function
     (Empty)
     (Identifier)

test/fixtures/go/grouped-import-declarations.parseB.txt

@@ -1,13 +1,16 @@
 (Program
   (Module
     (Identifier))
+  (
     (Import
-    (
       (TextElement)
+      (Empty))
+    (Import
       (TextElement)
-      (
+      (Empty))
-        (Identifier)
+    (Import
-        (TextElement))))
+      (TextElement)
+      (Identifier)))
   (Function
     (Empty)
     (Identifier)

test/fixtures/go/import-statements.diffA-B.txt

@@ -1,16 +1,14 @@
 (Program
   (Module
     (Identifier))
-  (Import
   (
-      (Context
     (Comment)
     (Comment)
+    (Import
     { (TextElement)
-      ->(TextElement) })
+    ->(TextElement) }
-      (Context
+      (Empty))
-        (Comment)
+    (Comment))
-        (Empty))))
   (Function
     (Empty)
     (Identifier)

test/fixtures/go/import-statements.diffB-A.txt

@@ -1,16 +1,14 @@
 (Program
   (Module
     (Identifier))
-  (Import
   (
-      (Context
     (Comment)
     (Comment)
+    (Import
     { (TextElement)
-      ->(TextElement) })
+    ->(TextElement) }
-      (Context
+      (Empty))
-        (Comment)
+    (Comment))
-        (Empty))))
   (Function
     (Empty)
     (Identifier)

test/fixtures/go/import-statements.parseA.txt

@@ -1,15 +1,13 @@
 (Program
   (Module
     (Identifier))
-  (Import
   (
-      (Context
     (Comment)
     (Comment)
-        (TextElement))
+    (Import
-      (Context
+      (TextElement)
-        (Comment)
+      (Empty))
-        (Empty))))
+    (Comment))
   (Function
     (Empty)
     (Identifier)

test/fixtures/go/import-statements.parseB.txt

@@ -1,15 +1,13 @@
 (Program
   (Module
     (Identifier))
-  (Import
   (
-      (Context
     (Comment)
     (Comment)
-        (TextElement))
+    (Import
-      (Context
+      (TextElement)
-        (Comment)
+      (Empty))
-        (Empty))))
+    (Comment))
   (Function
     (Empty)
     (Identifier)

test/fixtures/go/single-import-declarations.diffA-B.txt

@@ -3,15 +3,16 @@
     (Identifier))
   (Import
   { (TextElement)
-  ->(TextElement) })
+  ->(TextElement) }
+    (Empty))
   (Import
   { (TextElement)
-  ->(TextElement) })
+  ->(TextElement) }
+    (Empty))
   (Import
-    (
-      (Identifier)
   { (TextElement)
-    ->(TextElement) }))
+  ->(TextElement) }
+    (Identifier))
   (Function
     (Empty)
     (Identifier)

test/fixtures/go/single-import-declarations.diffB-A.txt

@@ -3,15 +3,16 @@
     (Identifier))
   (Import
   { (TextElement)
-  ->(TextElement) })
+  ->(TextElement) }
+    (Empty))
   (Import
   { (TextElement)
-  ->(TextElement) })
+  ->(TextElement) }
+    (Empty))
   (Import
-    (
-      (Identifier)
   { (TextElement)
-    ->(TextElement) }))
+  ->(TextElement) }
+    (Identifier))
   (Function
     (Empty)
     (Identifier)

test/fixtures/go/single-import-declarations.parseA.txt

@@ -2,13 +2,14 @@
   (Module
     (Identifier))
   (Import
-    (TextElement))
+    (TextElement)
+    (Empty))
   (Import
-    (TextElement))
+    (TextElement)
+    (Empty))
   (Import
-    (
+    (TextElement)
-      (Identifier)
+    (Identifier))
-      (TextElement)))
   (Function
     (Empty)
     (Identifier)

test/fixtures/go/single-import-declarations.parseB.txt

@@ -2,13 +2,14 @@
   (Module
     (Identifier))
   (Import
-    (TextElement))
+    (TextElement)
+    (Empty))
   (Import
-    (TextElement))
+    (TextElement)
+    (Empty))
   (Import
-    (
+    (TextElement)
-      (Identifier)
+    (Identifier))
-      (TextElement)))
   (Function
     (Empty)
     (Identifier)

test/fixtures/javascript/import.diffA-B.txt

@@ -1,75 +1,92 @@
 (Program
   (Import
-    (ImportClause
+  { (TextElement)
+  ->(TextElement) }
+    (Empty)
     (Identifier))
-  { (TextElement)
-  ->(TextElement) })
   (Import
-    (ImportClause
+  { (TextElement)
-      (NamespaceImport
+  ->(TextElement) }
   { (Identifier)
-      ->(Identifier) }))
+  ->(Identifier) })
-  { (TextElement)
-  ->(TextElement) })
   (Import
-    (ImportClause
+  { (TextElement)
-      (NamedImports
+  ->(TextElement) }
-        (ImportExportSpecifier
+    (Empty)
+    (ImportSymbol
+    { (Identifier)
+    ->(Identifier) }
+      (Empty)))
+  (
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
+      { (Identifier)
+      ->(Identifier) }
+        (Empty)))
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
         (Empty))))
-  { (TextElement)
+  (
-  ->(TextElement) })
     (Import
-    (ImportClause
-      (NamedImports
-        (ImportExportSpecifier
-        { (Identifier)
-        ->(Identifier) }
-          (Empty))
-        (ImportExportSpecifier
-        { (Identifier)
-        ->(Identifier) }
-          (Empty))))
     { (TextElement)
-  ->(TextElement) })
+    ->(TextElement) }
-  (Import
+      (Empty)
-    (ImportClause
+      (ImportSymbol
-      (NamedImports
-        (ImportExportSpecifier
       { (Identifier)
       ->(Identifier) }
-          (Empty))
+        (Empty)))
-        (ImportExportSpecifier
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
       { (Identifier)
       ->(Identifier) })))
-  { (TextElement)
+  (
-  ->(TextElement) })
     (Import
-    (ImportClause
+    { (TextElement)
-      (Identifier)
+    ->(TextElement) }
-      (NamedImports
+      (Empty)
-        (ImportExportSpecifier
+      (Identifier))
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
-          (Empty))
+        (Empty)))
-        (ImportExportSpecifier
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
       { (Identifier)
       ->(Identifier) })))
-  { (TextElement)
+  (
-  ->(TextElement) })
     (Import
-    (ImportClause
+    { (TextElement)
-      (Identifier)
+    ->(TextElement) }
-      (NamespaceImport
+      (Empty)
+      (Identifier))
+    (Import
+    { (TextElement)
+    ->(TextElement) }
     { (Identifier)
     ->(Identifier) }))
-  { (TextElement)
-  ->(TextElement) })
   (Import
   { (TextElement)
-  ->(TextElement) }))
+  ->(TextElement) }
+    (Empty)))

test/fixtures/javascript/import.diffB-A.txt

@@ -1,75 +1,92 @@
 (Program
   (Import
-    (ImportClause
+  { (TextElement)
+  ->(TextElement) }
+    (Empty)
     (Identifier))
-  { (TextElement)
-  ->(TextElement) })
   (Import
-    (ImportClause
+  { (TextElement)
-      (NamespaceImport
+  ->(TextElement) }
   { (Identifier)
-      ->(Identifier) }))
+  ->(Identifier) })
-  { (TextElement)
-  ->(TextElement) })
   (Import
-    (ImportClause
+  { (TextElement)
-      (NamedImports
+  ->(TextElement) }
-        (ImportExportSpecifier
+    (Empty)
+    (ImportSymbol
+    { (Identifier)
+    ->(Identifier) }
+      (Empty)))
+  (
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
+      { (Identifier)
+      ->(Identifier) }
+        (Empty)))
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
         (Empty))))
-  { (TextElement)
+  (
-  ->(TextElement) })
     (Import
-    (ImportClause
-      (NamedImports
-        (ImportExportSpecifier
-        { (Identifier)
-        ->(Identifier) }
-          (Empty))
-        (ImportExportSpecifier
-        { (Identifier)
-        ->(Identifier) }
-          (Empty))))
     { (TextElement)
-  ->(TextElement) })
+    ->(TextElement) }
-  (Import
+      (Empty)
-    (ImportClause
+      (ImportSymbol
-      (NamedImports
-        (ImportExportSpecifier
       { (Identifier)
       ->(Identifier) }
-          (Empty))
+        (Empty)))
-        (ImportExportSpecifier
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
       { (Identifier)
       ->(Identifier) })))
-  { (TextElement)
+  (
-  ->(TextElement) })
     (Import
-    (ImportClause
+    { (TextElement)
-      (Identifier)
+    ->(TextElement) }
-      (NamedImports
+      (Empty)
-        (ImportExportSpecifier
+      (Identifier))
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
-          (Empty))
+        (Empty)))
-        (ImportExportSpecifier
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
       { (Identifier)
       ->(Identifier) })))
-  { (TextElement)
+  (
-  ->(TextElement) })
     (Import
-    (ImportClause
+    { (TextElement)
-      (Identifier)
+    ->(TextElement) }
-      (NamespaceImport
+      (Empty)
+      (Identifier))
+    (Import
+    { (TextElement)
+    ->(TextElement) }
     { (Identifier)
     ->(Identifier) }))
-  { (TextElement)
-  ->(TextElement) })
   (Import
   { (TextElement)
-  ->(TextElement) }))
+  ->(TextElement) }
+    (Empty)))

test/fixtures/javascript/import.parseA.txt

@@ -1,56 +1,68 @@
 (Program
   (Import
-    (ImportClause
+    (TextElement)
+    (Empty)
     (Identifier))
-    (TextElement))
   (Import
-    (ImportClause
+    (TextElement)
-      (NamespaceImport
+    (Identifier))
-        (Identifier)))
-    (TextElement))
   (Import
-    (ImportClause
+    (TextElement)
-      (NamedImports
+    (Empty)
-        (ImportExportSpecifier
+    (ImportSymbol
+      (Identifier)
+      (Empty)))
+  (
+    (Import
+      (TextElement)
+      (Empty)
+      (ImportSymbol
+        (Identifier)
+        (Empty)))
+    (Import
+      (TextElement)
+      (Empty)
+      (ImportSymbol
         (Identifier)
         (Empty))))
-    (TextElement))
+  (
     (Import
-    (ImportClause
+      (TextElement)
-      (NamedImports
+      (Empty)
-        (ImportExportSpecifier
+      (ImportSymbol
         (Identifier)
-          (Empty))
+        (Empty)))
-        (ImportExportSpecifier
-          (Identifier)
-          (Empty))))
-    (TextElement))
     (Import
-    (ImportClause
+      (TextElement)
-      (NamedImports
+      (Empty)
-        (ImportExportSpecifier
+      (ImportSymbol
-          (Identifier)
-          (Empty))
-        (ImportExportSpecifier
         (Identifier)
         (Identifier))))
-    (TextElement))
+  (
     (Import
-    (ImportClause
+      (TextElement)
+      (Empty)
+      (Identifier))
+    (Import
+      (TextElement)
+      (Empty)
+      (ImportSymbol
         (Identifier)
-      (NamedImports
+        (Empty)))
-        (ImportExportSpecifier
+    (Import
-          (Identifier)
+      (TextElement)
-          (Empty))
+      (Empty)
-        (ImportExportSpecifier
+      (ImportSymbol
         (Identifier)
         (Identifier))))
-    (TextElement))
+  (
     (Import
-    (ImportClause
+      (TextElement)
-      (Identifier)
+      (Empty)
-      (NamespaceImport
+      (Identifier))
+    (Import
+      (TextElement)
       (Identifier)))
-    (TextElement))
   (Import
-    (TextElement)))
+    (TextElement)
+    (Empty)))

test/fixtures/javascript/import.parseB.txt

@@ -1,56 +1,68 @@
 (Program
   (Import
-    (ImportClause
+    (TextElement)
+    (Empty)
     (Identifier))
-    (TextElement))
   (Import
-    (ImportClause
+    (TextElement)
-      (NamespaceImport
+    (Identifier))
-        (Identifier)))
-    (TextElement))
   (Import
-    (ImportClause
+    (TextElement)
-      (NamedImports
+    (Empty)
-        (ImportExportSpecifier
+    (ImportSymbol
+      (Identifier)
+      (Empty)))
+  (
+    (Import
+      (TextElement)
+      (Empty)
+      (ImportSymbol
+        (Identifier)
+        (Empty)))
+    (Import
+      (TextElement)
+      (Empty)
+      (ImportSymbol
         (Identifier)
         (Empty))))
-    (TextElement))
+  (
     (Import
-    (ImportClause
+      (TextElement)
-      (NamedImports
+      (Empty)
-        (ImportExportSpecifier
+      (ImportSymbol
         (Identifier)
-          (Empty))
+        (Empty)))
-        (ImportExportSpecifier
-          (Identifier)
-          (Empty))))
-    (TextElement))
     (Import
-    (ImportClause
+      (TextElement)
-      (NamedImports
+      (Empty)
-        (ImportExportSpecifier
+      (ImportSymbol
-          (Identifier)
-          (Empty))
-        (ImportExportSpecifier
         (Identifier)
         (Identifier))))
-    (TextElement))
+  (
     (Import
-    (ImportClause
+      (TextElement)
+      (Empty)
+      (Identifier))
+    (Import
+      (TextElement)
+      (Empty)
+      (ImportSymbol
         (Identifier)
-      (NamedImports
+        (Empty)))
-        (ImportExportSpecifier
+    (Import
-          (Identifier)
+      (TextElement)
-          (Empty))
+      (Empty)
-        (ImportExportSpecifier
+      (ImportSymbol
         (Identifier)
         (Identifier))))
-    (TextElement))
+  (
     (Import
-    (ImportClause
+      (TextElement)
-      (Identifier)
+      (Empty)
-      (NamespaceImport
+      (Identifier))
+    (Import
+      (TextElement)
       (Identifier)))
-    (TextElement))
   (Import
-    (TextElement)))
+    (TextElement)
+    (Empty)))

test/fixtures/python/import-from-statement.diffA-B.txt

@@ -1,22 +1,28 @@
 (Program
-  (Import
-  {-(ScopeResolution
-    {-(Identifier)-})-}
-    (ScopeResolution
-      (Identifier))
-  {+(ScopeResolution
-    {+(Identifier)+})+})
-  (Import
-  {-(ScopeResolution
-    {-(Identifier)-})-}
-    (ScopeResolution
-      (Identifier))
-    (ScopeResolution
-      (Identifier))
-  {+(ScopeResolution
-    {+(Identifier)+})+})
   (Import
     (ScopeResolution
     { (Identifier)
     ->(Identifier) })
+    (Empty)
+    (
+      (ScopeResolution
+      { (Identifier)
+      ->(Identifier) })))
+  (Import
+    (ScopeResolution
+    { (Identifier)
+    ->(Identifier) })
+    (Empty)
+    (
+    {-(ScopeResolution
+      {-(Identifier)-})-}
+      (ScopeResolution
+        (Identifier))
+    {+(ScopeResolution
+      {+(Identifier)+})+}))
+  (Import
+    (ScopeResolution
+    { (Identifier)
+    ->(Identifier) })
+    (Empty)
     (Identifier)))

test/fixtures/python/import-from-statement.diffB-A.txt

@@ -1,22 +1,28 @@
 (Program
-  (Import
-  {-(ScopeResolution
-    {-(Identifier)-})-}
-    (ScopeResolution
-      (Identifier))
-  {+(ScopeResolution
-    {+(Identifier)+})+})
-  (Import
-  {+(ScopeResolution
-    {+(Identifier)+})+}
-    (ScopeResolution
-      (Identifier))
-    (ScopeResolution
-      (Identifier))
-  {-(ScopeResolution
-    {-(Identifier)-})-})
   (Import
     (ScopeResolution
     { (Identifier)
     ->(Identifier) })
+    (Empty)
+    (
+      (ScopeResolution
+      { (Identifier)
+      ->(Identifier) })))
+  (Import
+    (ScopeResolution
+    { (Identifier)
+    ->(Identifier) })
+    (Empty)
+    (
+    {+(ScopeResolution
+      {+(Identifier)+})+}
+      (ScopeResolution
+        (Identifier))
+    {-(ScopeResolution
+      {-(Identifier)-})-}))
+  (Import
+    (ScopeResolution
+    { (Identifier)
+    ->(Identifier) })
+    (Empty)
     (Identifier)))

test/fixtures/python/import-from-statement.parseA.txt

@@ -2,16 +2,21 @@
   (Import
     (ScopeResolution
       (Identifier))
+    (Empty)
+    (
       (ScopeResolution
-      (Identifier)))
+        (Identifier))))
   (Import
     (ScopeResolution
       (Identifier))
+    (Empty)
+    (
       (ScopeResolution
         (Identifier))
       (ScopeResolution
-      (Identifier)))
+        (Identifier))))
   (Import
     (ScopeResolution
       (Identifier))
+    (Empty)
     (Identifier)))

test/fixtures/python/import-from-statement.parseB.txt

@@ -2,16 +2,21 @@
   (Import
     (ScopeResolution
       (Identifier))
+    (Empty)
+    (
       (ScopeResolution
-      (Identifier)))
+        (Identifier))))
   (Import
     (ScopeResolution
       (Identifier))
+    (Empty)
+    (
       (ScopeResolution
         (Identifier))
       (ScopeResolution
-      (Identifier)))
+        (Identifier))))
   (Import
     (ScopeResolution
       (Identifier))
+    (Empty)
     (Identifier)))

test/fixtures/python/import-statement.diffA-B.txt

@@ -1,22 +1,26 @@
 (Program
+  (
+  {+(Import
+    {+(Identifier)+}
+    {+(Empty)+})+}
     (Import
-  {+(ScopeResolution
+      (Identifier)
-    {+(Identifier)+})+}
+      (Empty))
-    (ScopeResolution
+  {-(Import
-      (Identifier))
-  {-(Let
-    {-(Identifier)-}
     {-(ScopeResolution
       {-(Identifier)-})-}
-    {-(Empty)-})-})
+    {-(Identifier)-})-})
-  (Import
+{+(Import
-    (Let
+  {+(ScopeResolution
-    { (Identifier)
+    {+(Identifier)+}
-    ->(Identifier) }
+    {+(Identifier)+})+}
-      (ScopeResolution
+  {+(Identifier)+})+}
-      { (Identifier)
+{-(
-      ->(Identifier) }
+  {-(Import
-        (Identifier))
-      (Empty))
     {-(ScopeResolution
-    {-(Identifier)-})-}))
+      {-(Identifier)-}
+      {-(Identifier)-})-}
+    {-(Identifier)-})-}
+  {-(Import
+    {-(Identifier)-}
+    {-(Empty)-})-})-})

test/fixtures/python/import-statement.diffB-A.txt

@@ -1,22 +1,26 @@
 (Program
+  (
+  {-(Import
+    {-(Identifier)-}
+    {-(Empty)-})-}
     (Import
-  {-(ScopeResolution
+      (Identifier)
-    {-(Identifier)-})-}
+      (Empty))
-    (ScopeResolution
+  {+(Import
-      (Identifier))
-  {+(Let
-    {+(Identifier)+}
     {+(ScopeResolution
       {+(Identifier)+})+}
-    {+(Empty)+})+})
+    {+(Identifier)+})+})
-  (Import
+{+(
-    (Let
+  {+(Import
-    { (Identifier)
-    ->(Identifier) }
-      (ScopeResolution
-      { (Identifier)
-      ->(Identifier) }
-        (Identifier))
-      (Empty))
     {+(ScopeResolution
-    {+(Identifier)+})+}))
+      {+(Identifier)+}
+      {+(Identifier)+})+}
+    {+(Identifier)+})+}
+  {+(Import
+    {+(Identifier)+}
+    {+(Empty)+})+})+}
+{-(Import
+  {-(ScopeResolution
+    {-(Identifier)-}
+    {-(Identifier)-})-}
+  {-(Identifier)-})-})

test/fixtures/python/import-statement.parseA.txt

@@ -1,18 +1,18 @@
 (Program
+  (
     (Import
-    (ScopeResolution
-      (Identifier))
-    (Let
       (Identifier)
-      (ScopeResolution
-        (Identifier))
-      (Empty)))
-  (Import
-    (Let
-      (Identifier)
-      (ScopeResolution
-        (Identifier)
-        (Identifier))
       (Empty))
+    (Import
       (ScopeResolution
-      (Identifier))))
+        (Identifier))
+      (Identifier)))
+  (
+    (Import
+      (ScopeResolution
+        (Identifier)
+        (Identifier))
+      (Identifier))
+    (Import
+      (Identifier)
+      (Empty))))

test/fixtures/python/import-statement.parseB.txt

@@ -1,13 +1,13 @@
 (Program
+  (
+    (Import
+      (Identifier)
+      (Empty))
+    (Import
+      (Identifier)
+      (Empty)))
   (Import
     (ScopeResolution
+      (Identifier)
       (Identifier))
-    (ScopeResolution
     (Identifier)))
-  (Import
-    (Let
-      (Identifier)
-      (ScopeResolution
-        (Identifier)
-        (Identifier))
-      (Empty))))

test/fixtures/ruby/require.A.rb

@@ -0,0 +1,3 @@
+require "json"
+
+foo(a)

test/fixtures/ruby/require.B.rb

@@ -0,0 +1,3 @@
+require "nokogiri"
+
+autoload(:Bar, "bar.rb")

test/fixtures/ruby/require.diffA-B.txt

@@ -0,0 +1,14 @@
+(Program
+  (Import
+  { (TextElement)
+  ->(TextElement) }
+    (Empty))
+{+(Import
+  {+(
+    {+(Symbol)+}
+    {+(TextElement)+})+}
+  {+(Empty)+})+}
+{-(Call
+  {-(Identifier)-}
+  {-(Identifier)-}
+  {-(Empty)-})-})

test/fixtures/ruby/require.diffB-A.txt

@@ -0,0 +1,14 @@
+(Program
+  (Import
+  { (TextElement)
+  ->(TextElement) }
+    (Empty))
+{+(Call
+  {+(Identifier)+}
+  {+(Identifier)+}
+  {+(Empty)+})+}
+{-(Import
+  {-(
+    {-(Symbol)-}
+    {-(TextElement)-})-}
+  {-(Empty)-})-})

test/fixtures/ruby/require.parseA.txt

@@ -0,0 +1,8 @@
+(Program
+  (Import
+    (TextElement)
+    (Empty))
+  (Call
+    (Identifier)
+    (Identifier)
+    (Empty)))

test/fixtures/ruby/require.parseB.txt

@@ -0,0 +1,9 @@
+(Program
+  (Import
+    (TextElement)
+    (Empty))
+  (Import
+    (
+      (Symbol)
+      (TextElement))
+    (Empty)))

test/fixtures/typescript/import.A.ts

@@ -6,3 +6,4 @@ import { member1 , member2 as alias2 } from "ant-eater";
 import defaultMember, { member1, member2 as alias2 } from "anaconda";
 import defaultMember, * as name from "alligator";
 import "arctic-tern";
+import zip = require("../zip");

test/fixtures/typescript/import.diffA-B.txt

@@ -1,75 +1,95 @@
 (Program
   (Import
-    (ImportClause
+  { (TextElement)
+  ->(TextElement) }
+    (Empty)
     (Identifier))
-  { (TextElement)
-  ->(TextElement) })
   (Import
-    (ImportClause
+  { (TextElement)
-      (NamespaceImport
+  ->(TextElement) }
   { (Identifier)
-      ->(Identifier) }))
+  ->(Identifier) })
-  { (TextElement)
-  ->(TextElement) })
   (Import
-    (ImportClause
+  { (TextElement)
-      (NamedImports
+  ->(TextElement) }
-        (ImportExportSpecifier
+    (Empty)
+    (ImportSymbol
+    { (Identifier)
+    ->(Identifier) }
+      (Empty)))
+  (
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
+      { (Identifier)
+      ->(Identifier) }
+        (Empty)))
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
         (Empty))))
-  { (TextElement)
+  (
-  ->(TextElement) })
     (Import
-    (ImportClause
-      (NamedImports
-        (ImportExportSpecifier
-        { (Identifier)
-        ->(Identifier) }
-          (Empty))
-        (ImportExportSpecifier
-        { (Identifier)
-        ->(Identifier) }
-          (Empty))))
     { (TextElement)
-  ->(TextElement) })
+    ->(TextElement) }
-  (Import
+      (Empty)
-    (ImportClause
+      (ImportSymbol
-      (NamedImports
-        (ImportExportSpecifier
       { (Identifier)
       ->(Identifier) }
-          (Empty))
+        (Empty)))
-        (ImportExportSpecifier
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
       { (Identifier)
       ->(Identifier) })))
-  { (TextElement)
+  (
-  ->(TextElement) })
     (Import
-    (ImportClause
+    { (TextElement)
-      (Identifier)
+    ->(TextElement) }
-      (NamedImports
+      (Empty)
-        (ImportExportSpecifier
+      (Identifier))
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
-          (Empty))
+        (Empty)))
-        (ImportExportSpecifier
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
       { (Identifier)
       ->(Identifier) })))
-  { (TextElement)
+  (
-  ->(TextElement) })
     (Import
-    (ImportClause
+    { (TextElement)
-      (Identifier)
+    ->(TextElement) }
-      (NamespaceImport
+      (Empty)
+      (Identifier))
+    (Import
+    { (TextElement)
+    ->(TextElement) }
     { (Identifier)
     ->(Identifier) }))
-  { (TextElement)
-  ->(TextElement) })
   (Import
   { (TextElement)
-  ->(TextElement) }))
+  ->(TextElement) }
+    (Empty))
+{-(Import
+  {-(TextElement)-}
+  {-(Identifier)-})-})

test/fixtures/typescript/import.diffB-A.txt

@@ -1,75 +1,95 @@
 (Program
   (Import
-    (ImportClause
+  { (TextElement)
+  ->(TextElement) }
+    (Empty)
     (Identifier))
-  { (TextElement)
-  ->(TextElement) })
   (Import
-    (ImportClause
+  { (TextElement)
-      (NamespaceImport
+  ->(TextElement) }
   { (Identifier)
-      ->(Identifier) }))
+  ->(Identifier) })
-  { (TextElement)
-  ->(TextElement) })
   (Import
-    (ImportClause
+  { (TextElement)
-      (NamedImports
+  ->(TextElement) }
-        (ImportExportSpecifier
+    (Empty)
+    (ImportSymbol
+    { (Identifier)
+    ->(Identifier) }
+      (Empty)))
+  (
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
+      { (Identifier)
+      ->(Identifier) }
+        (Empty)))
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
         (Empty))))
-  { (TextElement)
+  (
-  ->(TextElement) })
     (Import
-    (ImportClause
-      (NamedImports
-        (ImportExportSpecifier
-        { (Identifier)
-        ->(Identifier) }
-          (Empty))
-        (ImportExportSpecifier
-        { (Identifier)
-        ->(Identifier) }
-          (Empty))))
     { (TextElement)
-  ->(TextElement) })
+    ->(TextElement) }
-  (Import
+      (Empty)
-    (ImportClause
+      (ImportSymbol
-      (NamedImports
-        (ImportExportSpecifier
       { (Identifier)
       ->(Identifier) }
-          (Empty))
+        (Empty)))
-        (ImportExportSpecifier
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
       { (Identifier)
       ->(Identifier) })))
-  { (TextElement)
+  (
-  ->(TextElement) })
     (Import
-    (ImportClause
+    { (TextElement)
-      (Identifier)
+    ->(TextElement) }
-      (NamedImports
+      (Empty)
-        (ImportExportSpecifier
+      (Identifier))
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
-          (Empty))
+        (Empty)))
-        (ImportExportSpecifier
+    (Import
+    { (TextElement)
+    ->(TextElement) }
+      (Empty)
+      (ImportSymbol
       { (Identifier)
       ->(Identifier) }
       { (Identifier)
       ->(Identifier) })))
-  { (TextElement)
+  (
-  ->(TextElement) })
     (Import
-    (ImportClause
+    { (TextElement)
-      (Identifier)
+    ->(TextElement) }
-      (NamespaceImport
+      (Empty)
+      (Identifier))
+    (Import
+    { (TextElement)
+    ->(TextElement) }
     { (Identifier)
     ->(Identifier) }))
-  { (TextElement)
-  ->(TextElement) })
   (Import
   { (TextElement)
-  ->(TextElement) }))
+  ->(TextElement) }
+    (Empty))
+{+(Import
+  {+(TextElement)+}
+  {+(Identifier)+})+})

test/fixtures/typescript/import.parseA.txt

@@ -1,56 +1,71 @@
 (Program
   (Import
-    (ImportClause
+    (TextElement)
+    (Empty)
     (Identifier))
-    (TextElement))
   (Import
-    (ImportClause
+    (TextElement)
-      (NamespaceImport
+    (Identifier))
-        (Identifier)))
-    (TextElement))
   (Import
-    (ImportClause
+    (TextElement)
-      (NamedImports
+    (Empty)
-        (ImportExportSpecifier
+    (ImportSymbol
+      (Identifier)
+      (Empty)))
+  (
+    (Import
+      (TextElement)
+      (Empty)
+      (ImportSymbol
+        (Identifier)
+        (Empty)))
+    (Import
+      (TextElement)
+      (Empty)
+      (ImportSymbol
         (Identifier)
         (Empty))))
-    (TextElement))
+  (
     (Import
-    (ImportClause
+      (TextElement)
-      (NamedImports
+      (Empty)
-        (ImportExportSpecifier
+      (ImportSymbol
         (Identifier)
-          (Empty))
+        (Empty)))
-        (ImportExportSpecifier
-          (Identifier)
-          (Empty))))
-    (TextElement))
     (Import
-    (ImportClause
+      (TextElement)
-      (NamedImports
+      (Empty)
-        (ImportExportSpecifier
+      (ImportSymbol
-          (Identifier)
-          (Empty))
-        (ImportExportSpecifier
         (Identifier)
         (Identifier))))
-    (TextElement))
+  (
     (Import
-    (ImportClause
+      (TextElement)
+      (Empty)
+      (Identifier))
+    (Import
+      (TextElement)
+      (Empty)
+      (ImportSymbol
         (Identifier)
-      (NamedImports
+        (Empty)))
-        (ImportExportSpecifier
+    (Import
-          (Identifier)
+      (TextElement)
-          (Empty))
+      (Empty)
-        (ImportExportSpecifier
+      (ImportSymbol
         (Identifier)
         (Identifier))))
-    (TextElement))
+  (
     (Import
-    (ImportClause
+      (TextElement)
-      (Identifier)
+      (Empty)
-      (NamespaceImport
+      (Identifier))
+    (Import
+      (TextElement)
       (Identifier)))
-    (TextElement))
   (Import
-    (TextElement)))
+    (TextElement)
+    (Empty))
+  (Import
+    (TextElement)
+    (Identifier)))

test/fixtures/typescript/import.parseB.txt

@@ -1,56 +1,68 @@
 (Program
   (Import
-    (ImportClause
+    (TextElement)
+    (Empty)
     (Identifier))
-    (TextElement))
   (Import
-    (ImportClause
+    (TextElement)
-      (NamespaceImport
+    (Identifier))
-        (Identifier)))
-    (TextElement))
   (Import
-    (ImportClause
+    (TextElement)
-      (NamedImports
+    (Empty)
-        (ImportExportSpecifier
+    (ImportSymbol
+      (Identifier)
+      (Empty)))
+  (
+    (Import
+      (TextElement)
+      (Empty)
+      (ImportSymbol
+        (Identifier)
+        (Empty)))
+    (Import
+      (TextElement)
+      (Empty)
+      (ImportSymbol
         (Identifier)
         (Empty))))
-    (TextElement))
+  (
     (Import
-    (ImportClause
+      (TextElement)
-      (NamedImports
+      (Empty)
-        (ImportExportSpecifier
+      (ImportSymbol
         (Identifier)
-          (Empty))
+        (Empty)))
-        (ImportExportSpecifier
-          (Identifier)
-          (Empty))))
-    (TextElement))
     (Import
-    (ImportClause
+      (TextElement)
-      (NamedImports
+      (Empty)
-        (ImportExportSpecifier
+      (ImportSymbol
-          (Identifier)
-          (Empty))
-        (ImportExportSpecifier
         (Identifier)
         (Identifier))))
-    (TextElement))
+  (
     (Import
-    (ImportClause
+      (TextElement)
+      (Empty)
+      (Identifier))
+    (Import
+      (TextElement)
+      (Empty)
+      (ImportSymbol
         (Identifier)
-      (NamedImports
+        (Empty)))
-        (ImportExportSpecifier
+    (Import
-          (Identifier)
+      (TextElement)
-          (Empty))
+      (Empty)
-        (ImportExportSpecifier
+      (ImportSymbol
         (Identifier)
         (Identifier))))
-    (TextElement))
+  (
     (Import
-    (ImportClause
+      (TextElement)
-      (Identifier)
+      (Empty)
-      (NamespaceImport
+      (Identifier))
+    (Import
+      (TextElement)
       (Identifier)))
-    (TextElement))
   (Import
-    (TextElement)))
+    (TextElement)
+    (Empty)))