Merge pull request #551 from p-alik/semantic-rust

Semantic rust

cabal.project

@@ -17,6 +17,7 @@ packages: .
           semantic-tsx
           semantic-typescript
           semantic-tags
+          semantic-rust
 
 -- Packages brought in from other repos instead of hackage
 -- ATTENTION: remember to update cabal.project.ci when bumping SHAs here!

cabal.project.ci

@@ -13,6 +13,7 @@ packages: .
           semantic-php
           semantic-python
           semantic-ruby
+          semantic-rust
           semantic-scope-graph
           semantic-tsx
           semantic-typescript
@@ -30,7 +31,6 @@ source-repository-package
   location: https://github.com/antitypical/fused-syntax.git
   tag: d11e14581217590a5c67f79cbaeee35ac8acee6a
 
-
 -- Treat warnings as errors for CI builds
 package semantic
   ghc-options: -Werror

script/ghci-flags

@@ -61,6 +61,7 @@ function flags {
   echo "-isemantic-python/src"
   echo "-isemantic-python/test"
   echo "-isemantic-ruby/src"
+  echo "-isemantic-rust/src"
   echo "-isemantic-scope-graph/src"
   echo "-isemantic-tsx/src"
   echo "-isemantic-typescript/src"

semantic-rust/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 GitHub
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

semantic-rust/README.md

@@ -0,0 +1,18 @@
+# Semantic support for Rust
+
+This package implements `semantic` support for [Rust](https://www.rust-lang.org/) using the `semantic-core` intermediate language.
+
+## Generating AST
+
+```
+cd semantic-rust
+cabal v2-repl
+λ> :seti -XOverloadedStrings
+λ> :seti -XTypeApplications
+λ> import Source.Span
+λ> import Source.Range
+λ> import AST.Unmarshal
+λ> TS.parseByteString @Language.Rust.AST.SourceFile @(Source.Span.Span, Source.Range.Range) Language.Rust.Grammar.tree_sitter_rust "let x = 1;"
+Right (SourceFile {ann = (Span {start = Pos {line = 0, column = 0}, end = Pos {line = 0, column = 10}},Range {start = 0, end = 10}), extraChildren = [L1 (DeclarationStatement {getDeclarationStatement = R1 (L1 (L1 (R1 (LetDeclaration {ann = (Span {start = Pos {line = 0, column = 0}, end = Pos {line = 0, column = 10}},Range {start = 0, end = 10}), pattern = Pattern {getPattern = L1 (R1 (L1 (L1 (Identifier {ann = (Span {start = Pos {line = 0, column = 4}, end = Pos {line = 0, column = 5}},Range
+{start = 4, end = 5}), text = "x"}))))}, value = Just (Expression {getExpression = L1 (L1 (L1 (L1 (L1 (Literal {getLiteral = R1 (L1 (IntegerLiteral {ann = (Span {start = Pos {line = 0, column = 8}, end = Pos {line = 0, column = 9}},Range {start = 8, end = 9}), text = "1"}))})))))}), type' = Nothing, extraChildren = Nothing}))))})]})
+```

semantic-rust/Setup.hs

@@ -0,0 +1,2 @@
+import Distribution.Simple
+main = defaultMain

semantic-rust/semantic-rust.cabal

@@ -0,0 +1,87 @@
+cabal-version:       2.4
+
+name:                semantic-rust
+version:             0.0.0.0
+synopsis:            Semantic support for Rust
+description:         Semantic support for Rust.
+homepage:            https://github.com/github/semantic/tree/master/semantic-rust#readme
+bug-reports:         https://github.com/github/semantic/issues
+license:             MIT
+license-file:        LICENSE
+author:              The Semantic authors, Alexei Pastuchov
+maintainer:          opensource+semantic@github.com
+copyright:           (c) 2020 GitHub, Inc.
+category:            Language
+build-type:          Simple
+stability:           alpha
+extra-source-files:  README.md
+
+tested-with:         GHC == 8.6.5
+
+common haskell
+  default-language:    Haskell2010
+  build-depends:       base ^>= 4.13
+                     , fused-effects ^>= 1.0
+                     , fused-syntax
+                     , parsers ^>= 0.12.10
+                     , semantic-ast
+                     , semantic-core ^>= 0.0
+                     , semantic-source ^>= 0.1.0
+                     , semantic-tags ^>= 0.0
+                     , template-haskell ^>= 2.15
+                     , text ^>= 1.2.3
+                     , tree-sitter ^>= 0.9
+                     , tree-sitter-rust ^>= 0.1.0.0
+
+  ghc-options:
+    -Weverything
+    -Wno-missing-local-signatures
+    -Wno-missing-import-lists
+    -Wno-implicit-prelude
+    -Wno-safe
+    -Wno-unsafe
+    -Wno-name-shadowing
+    -Wno-monomorphism-restriction
+    -Wno-missed-specialisations
+    -Wno-all-missed-specialisations
+    -Wno-star-is-type
+  if (impl(ghc >= 8.8))
+    ghc-options: -Wno-missing-deriving-strategies
+
+library
+  import: haskell
+  exposed-modules:
+    Language.Rust
+    Language.Rust.AST
+    Language.Rust.Grammar
+    Language.Rust.Tags
+  hs-source-dirs:      src
+
+test-suite test
+  import: haskell
+  type:             exitcode-stdio-1.0
+  hs-source-dirs:   test
+  main-is:          Test.hs
+  build-depends:    base
+                  , bytestring ^>= 0.10.8.2
+                  , pathtype ^>= 0.8.1
+                  , semantic-ast
+                  , semantic-rust
+                  , tasty
+                  , tasty-hunit
+                  , text
+
+  ghc-options:
+    -Weverything
+    -Wno-missing-local-signatures
+    -Wno-missing-import-lists
+    -Wno-implicit-prelude
+    -Wno-safe
+    -Wno-unsafe
+    -Wno-name-shadowing
+    -Wno-monomorphism-restriction
+    -Wno-missed-specialisations
+    -Wno-all-missed-specialisations
+    -Wno-star-is-type
+  if (impl(ghc >= 8.8))
+    ghc-options: -Wno-missing-deriving-strategies

semantic-rust/src/Language/Rust.hs

@@ -0,0 +1,24 @@
+-- | Semantic functionality for Rust programs.
+module Language.Rust
+( Term(..)
+, Language.Rust.Grammar.tree_sitter_rust
+) where
+
+import           Data.Proxy
+import qualified Language.Rust.AST as Rust
+import qualified Language.Rust.Tags as RustTags
+import qualified Tags.Tagging.Precise as Tags
+import qualified Language.Rust.Grammar (tree_sitter_rust)
+import qualified AST.Unmarshal as TS
+
+newtype Term a = Term { getTerm :: Rust.SourceFile a }
+
+instance TS.SymbolMatching Term where
+  matchedSymbols _ = TS.matchedSymbols (Proxy :: Proxy Rust.SourceFile)
+  showFailure _ = TS.showFailure (Proxy :: Proxy Rust.SourceFile)
+
+instance TS.Unmarshal Term where
+  matchers = fmap (fmap (TS.hoist Term)) TS.matchers
+
+instance Tags.ToTags Term where
+  tags src = Tags.runTagging src . RustTags.tags . getTerm

semantic-rust/src/Language/Rust/AST.hs

@@ -0,0 +1,23 @@
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveGeneric #-}
+{-# LANGUAGE DeriveTraversable #-}
+{-# LANGUAGE DerivingStrategies #-}
+{-# LANGUAGE DuplicateRecordFields #-}
+{-# LANGUAGE FlexibleInstances #-}
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+{-# LANGUAGE MultiParamTypeClasses #-}
+{-# LANGUAGE TemplateHaskell #-}
+{-# LANGUAGE TypeApplications #-}
+
+module Language.Rust.AST
+( module Language.Rust.AST
+, Rust.getTestCorpusDir
+) where
+
+import           AST.GenerateSyntax
+import           AST.Token
+import           Language.Haskell.TH.Syntax (runIO)
+import qualified TreeSitter.Rust as Rust (getNodeTypesPath, getTestCorpusDir, tree_sitter_rust)
+
+runIO Rust.getNodeTypesPath >>= astDeclarationsForLanguage Rust.tree_sitter_rust

semantic-rust/src/Language/Rust/Grammar.hs

@@ -0,0 +1,12 @@
+{-# LANGUAGE TemplateHaskell #-}
+module Language.Rust.Grammar
+( tree_sitter_rust
+, Grammar(..)
+) where
+
+import AST.Grammar.TH
+import Language.Haskell.TH
+import TreeSitter.Rust (tree_sitter_rust)
+
+-- | Statically-known rules corresponding to symbols in the grammar.
+mkStaticallyKnownRuleGrammarData (mkName "Grammar") tree_sitter_rust

semantic-rust/src/Language/Rust/Tags.hs

@@ -0,0 +1,208 @@
+{-# LANGUAGE DefaultSignatures #-}
+{-# LANGUAGE DisambiguateRecordFields #-}
+{-# LANGUAGE FlexibleContexts #-}
+{-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE TypeApplications #-}
+{-# LANGUAGE TypeOperators #-}
+module Language.Rust.Tags
+( ToTags(..)
+) where
+
+import           AST.Element
+import           AST.Token
+import           AST.Traversable1
+import           Control.Effect.Reader
+import           Control.Effect.Writer
+import qualified Language.Rust.AST as Rust
+import           Source.Loc
+import           Source.Source as Source
+import           Tags.Tag()
+import qualified Tags.Tagging.Precise as Tags
+
+class ToTags t where
+  tags
+    :: ( Has (Reader Source) sig m
+       , Has (Writer Tags.Tags) sig m
+       )
+    => t Loc
+    -> m ()
+  default tags
+    :: ( Has (Reader Source) sig m
+       , Has (Writer Tags.Tags) sig m
+       , Traversable1 ToTags t
+       )
+    => t Loc
+    -> m ()
+  tags = gtags
+
+instance (ToTags l, ToTags r) => ToTags (l :+: r) where
+  tags (L1 l) = tags l
+  tags (R1 r) = tags r
+
+instance ToTags (Token sym n) where tags _ = pure ()
+
+gtags
+  :: ( Has (Reader Source) sig m
+     , Has (Writer Tags.Tags) sig m
+     , Traversable1 ToTags t
+     )
+  => t Loc
+  -> m ()
+gtags = traverse1_ @ToTags (const (pure ())) tags
+
+instance ToTags Rust.AbstractType
+instance ToTags Rust.Arguments
+instance ToTags Rust.ArrayExpression
+instance ToTags Rust.ArrayType
+instance ToTags Rust.AssignmentExpression
+instance ToTags Rust.AssociatedType
+instance ToTags Rust.AsyncBlock
+instance ToTags Rust.AttributeItem
+instance ToTags Rust.AwaitExpression
+instance ToTags Rust.BaseFieldInitializer
+instance ToTags Rust.BinaryExpression
+instance ToTags Rust.Block
+instance ToTags Rust.BlockComment
+instance ToTags Rust.BooleanLiteral
+instance ToTags Rust.BoundedType
+instance ToTags Rust.BracketedType
+instance ToTags Rust.BreakExpression
+instance ToTags Rust.CallExpression
+instance ToTags Rust.CapturedPattern
+instance ToTags Rust.CharLiteral
+instance ToTags Rust.ClosureExpression
+instance ToTags Rust.ClosureParameters
+instance ToTags Rust.CompoundAssignmentExpr
+instance ToTags Rust.ConstItem
+instance ToTags Rust.ConstParameter
+instance ToTags Rust.ConstrainedTypeParameter
+instance ToTags Rust.ContinueExpression
+instance ToTags Rust.Crate
+instance ToTags Rust.DeclarationList
+instance ToTags Rust.DeclarationStatement
+instance ToTags Rust.DynamicType
+instance ToTags Rust.EmptyStatement
+instance ToTags Rust.EmptyType
+instance ToTags Rust.EnumItem
+instance ToTags Rust.EnumVariant
+instance ToTags Rust.EnumVariantList
+instance ToTags Rust.EscapeSequence
+instance ToTags Rust.Expression
+instance ToTags Rust.ExternCrateDeclaration
+instance ToTags Rust.ExternModifier
+instance ToTags Rust.FieldDeclaration
+instance ToTags Rust.FieldDeclarationList
+instance ToTags Rust.FieldExpression
+instance ToTags Rust.FieldIdentifier
+instance ToTags Rust.FieldInitializer
+instance ToTags Rust.FieldInitializerList
+instance ToTags Rust.FieldPattern
+instance ToTags Rust.FloatLiteral
+instance ToTags Rust.ForExpression
+instance ToTags Rust.ForLifetimes
+instance ToTags Rust.ForeignModItem
+instance ToTags Rust.FragmentSpecifier
+instance ToTags Rust.FunctionItem
+instance ToTags Rust.FunctionModifiers
+instance ToTags Rust.FunctionSignatureItem
+instance ToTags Rust.FunctionType
+instance ToTags Rust.GenericFunction
+instance ToTags Rust.GenericType
+instance ToTags Rust.GenericTypeWithTurbofish
+instance ToTags Rust.HigherRankedTraitBound
+instance ToTags Rust.Identifier
+instance ToTags Rust.IfExpression
+instance ToTags Rust.IfLetExpression
+instance ToTags Rust.ImplItem
+instance ToTags Rust.IndexExpression
+instance ToTags Rust.InnerAttributeItem
+instance ToTags Rust.IntegerLiteral
+instance ToTags Rust.LetDeclaration
+instance ToTags Rust.Lifetime
+instance ToTags Rust.LineComment
+instance ToTags Rust.Literal
+instance ToTags Rust.LiteralPattern
+instance ToTags Rust.LoopExpression
+instance ToTags Rust.LoopLabel
+instance ToTags Rust.MacroDefinition
+instance ToTags Rust.MacroInvocation
+instance ToTags Rust.MacroRule
+instance ToTags Rust.MatchArm
+instance ToTags Rust.MatchBlock
+instance ToTags Rust.MatchExpression
+instance ToTags Rust.MatchPattern
+instance ToTags Rust.MetaArguments
+instance ToTags Rust.MetaItem
+instance ToTags Rust.Metavariable
+instance ToTags Rust.ModItem
+instance ToTags Rust.MutPattern
+instance ToTags Rust.MutableSpecifier
+instance ToTags Rust.NegativeLiteral
+instance ToTags Rust.OptionalTypeParameter
+instance ToTags Rust.OrderedFieldDeclarationList
+instance ToTags Rust.Parameter
+instance ToTags Rust.Parameters
+instance ToTags Rust.ParenthesizedExpression
+instance ToTags Rust.Pattern
+instance ToTags Rust.PointerType
+instance ToTags Rust.PrimitiveType
+instance ToTags Rust.QualifiedType
+instance ToTags Rust.RangeExpression
+instance ToTags Rust.RangePattern
+instance ToTags Rust.RawStringLiteral
+instance ToTags Rust.RefPattern
+instance ToTags Rust.ReferenceExpression
+instance ToTags Rust.ReferencePattern
+instance ToTags Rust.ReferenceType
+instance ToTags Rust.RemainingFieldPattern
+instance ToTags Rust.RemovedTraitBound
+instance ToTags Rust.ReturnExpression
+instance ToTags Rust.ScopedIdentifier
+instance ToTags Rust.ScopedTypeIdentifier
+instance ToTags Rust.ScopedUseList
+instance ToTags Rust.Self
+instance ToTags Rust.SelfParameter
+instance ToTags Rust.ShorthandFieldIdentifier
+instance ToTags Rust.ShorthandFieldInitializer
+instance ToTags Rust.SlicePattern
+instance ToTags Rust.SourceFile
+instance ToTags Rust.StaticItem
+instance ToTags Rust.StringLiteral
+instance ToTags Rust.StructExpression
+instance ToTags Rust.StructItem
+instance ToTags Rust.StructPattern
+instance ToTags Rust.Super
+instance ToTags Rust.TokenBindingPattern
+instance ToTags Rust.TokenRepetition
+instance ToTags Rust.TokenRepetitionPattern
+instance ToTags Rust.TokenTree
+instance ToTags Rust.TokenTreePattern
+instance ToTags Rust.TraitBounds
+instance ToTags Rust.TraitItem
+instance ToTags Rust.TryExpression
+instance ToTags Rust.TupleExpression
+instance ToTags Rust.TuplePattern
+instance ToTags Rust.TupleStructPattern
+instance ToTags Rust.TupleType
+instance ToTags Rust.Type
+instance ToTags Rust.TypeArguments
+instance ToTags Rust.TypeBinding
+instance ToTags Rust.TypeCastExpression
+instance ToTags Rust.TypeIdentifier
+instance ToTags Rust.TypeItem
+instance ToTags Rust.TypeParameters
+instance ToTags Rust.UnaryExpression
+instance ToTags Rust.UnionItem
+instance ToTags Rust.UnitExpression
+instance ToTags Rust.UnitType
+instance ToTags Rust.UnsafeBlock
+instance ToTags Rust.UseAsClause
+instance ToTags Rust.UseDeclaration
+instance ToTags Rust.UseList
+instance ToTags Rust.UseWildcard
+instance ToTags Rust.VariadicParameter
+instance ToTags Rust.VisibilityModifier
+instance ToTags Rust.WhereClause
+instance ToTags Rust.WherePredicate
+instance ToTags Rust.WhileExpression
+instance ToTags Rust.WhileLetExpression

semantic-rust/test/Test.hs

@@ -0,0 +1,25 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE TypeApplications #-}
+module Main (main) where
+
+import           AST.Test
+import           AST.Unmarshal (parseByteString)
+import qualified Language.Rust.AST as Rust
+import           Language.Rust.Grammar
+import qualified System.Path as Path
+import           Test.Tasty
+import Control.Monad (liftM)
+
+main :: IO ()
+main
+  =   Path.absDir <$> Rust.getTestCorpusDir
+  >>= excludeMacrosCorpus . readCorpusFiles'
+  >>= traverse (testCorpus parse)
+  >>= defaultMain . tests
+  where
+    parse = parseByteString @Rust.SourceFile @() tree_sitter_rust
+    excludeMacrosCorpus l = liftM (filter (f "expressions") ) $ liftM (filter (f "macros") ) l
+      where f p bn = p /= (Path.toString . Path.takeBaseName) bn
+
+tests :: [TestTree] -> TestTree
+tests = testGroup "tree-sitter-rust corpus tests"

semantic.cabal

@@ -304,6 +304,7 @@ library
                      , tree-sitter-python ^>= 0.9.0.1
                      , tree-sitter-ql ^>= 0.1.0.1
                      , tree-sitter-ruby ^>= 0.5.0.0
+                     , tree-sitter-rust ^>= 0.1.0.0
                      , tree-sitter-typescript ^>= 0.5.0.0
                      , tree-sitter-tsx ^>= 0.5.0.0