github/semantic
1
1
Fork 0
mirror of https://github.com/github/semantic.git synced 2024-12-24 07:25:44 +03:00
Code Issues Projects Releases Wiki Activity

Tokenize strings.

Browse Source
This commit is contained in:
Rob Rix 2015-12-16 09:41:46 -05:00
parent 40e9068ed8
commit da793e21d5
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/Haskell.hs
View File

@ -10,6 +10,10 @@ import Text.Parsec.String
data Token = Token { getRange :: Range, getString :: String }
tokenize :: String -> [Token]
tokenize "" = []
tokenize string = uncurry Token <$> rangesAndWordsFrom 0 string
haskellParser :: Parser (Term String Info)
haskellParser = toTerm <$> many anyChar
where toTerm a = Info (Range 0 0) mempty :< Leaf a