on master help sign in
implement HTTP/HTTPS TutorialD importer
commit39de5a326043d81f3c1b9064bf0e797a12b8d3f3 authorAgentM <agentm@themactionfaction.com> dateSun 28 Mar 2021, 23:24 (6 years ago) parenta244f5f
13 files changed+152−37
examples/out_of_the_tarpit.tutd+4−4 project-m36.cabal+7−2 src/bin/ProjectM36/Server/RemoteCallTypes/Json.hs+3 src/bin/TutorialD/Interpreter.hs+1 src/bin/TutorialD/Interpreter/Base.hs+21−1 src/bin/TutorialD/Interpreter/DatabaseContextExpr.hs+5 src/bin/TutorialD/Interpreter/Import/Base.hs+8−3 src/bin/TutorialD/Interpreter/Import/CSV.hs+3−3 src/bin/TutorialD/Interpreter/Import/TutorialD.hs+72−17 src/lib/ProjectM36/Error.hs+10−2 src/lib/ProjectM36/Serialise/Error.hs+1 test/TutorialD/Interpreter.hs+1−1 test/TutorialD/Interpreter/Import/TutorialD.hs+16−4
examples/out_of_the_tarpit.tutd+4−4
@@ -3,9 +3,9 @@
33data Address = Address Text
44data Agent = Agent Text
55data Name = Name Text
6−data Price = Price Double
6+data Price = Price Integer --dollars
77data RoomType = Kitchen | Bathroom | LivingRoom
8−data PriceBand = Low | Medium | High | Premium
8+data PriceBand = Low | Med | High | Premium
99data AreaCode = City | Suburban | Rural
1010data SpeedBand = VeryFast | Fast | Medium | Slow
1111
@@ -57,9 +57,9 @@ constraint noOffersAfterSale (offer join decision where ^dayEarlierThan(@offerDa
5757--1. identify offerPrices > 2000 (arbitrarily-chosen high price band which is not specified in the paper)
5858--2. remove the properties which have been sold
5959--3. group and count sub-relation and restrict by greater than 50
60−constraint noMoreThan50PremiumProperties ((((property where ^gt(offerPrice,2000.0)) minus ((property join decision){address,agent,dateRegistered,photo,price})) group ({} as x) : {z:=gt(count(@x),50)}) {z}){} equals false
60+constraint noMoreThan50PremiumProperties ((((property where ^gt(@price,Price 2000)) minus ((property join decision){address,agent,dateRegistered,photo,price})) group ({} as x) : {z:=gt(count(@x),50)}) {z}){} equals false
6161
62−constraint noBidderCanMakeMoreThan10BidsOnAProperty (((offer group ({all but bidderAddress} as bidder)) : {bidCount:=gt(count(@bidder),10)}) where ^gt(bidCount,10)) {} equals false
62+constraint noBidderCanMakeMoreThan10BidsOnAProperty (((offer group ({all but bidderAddress} as bidder)) : {bidCount:=gt(count(@bidder),10)}) where ^gt(@bidCount,10)) {} equals false
6363
6464-- sample data
6565
project-m36.cabal+7−2
@@ -184,7 +184,12 @@ Executable tutd
184184 list-t,
185185 parser-combinators,
186186 curryer-rpc,
187− prettyprinter
187+ prettyprinter,
188+ cryptohash-sha256,
189+ base16-bytestring,
190+ http-conduit,
191+ modern-uri,
192+ http-types
188193 Other-Modules: TutorialD.Interpreter,
189194 TutorialD.Interpreter.Base,
190195 TutorialD.Interpreter.DatabaseContextExpr,
@@ -264,7 +269,7 @@ Executable bigrel
264269
265270Common commontest
266271 Default-Language: Haskell2010
267− Build-Depends: base, HUnit, Cabal, containers, hashable, unordered-containers, mtl, vector, time, hashable-time, bytestring, uuid, stm, deepseq, deepseq-generics,parallel, cassava, attoparsec, gnuplot, directory, temporary, haskeline, megaparsec, text, base64-bytestring, data-interval, filepath, transformers, stm-containers, list-t, websockets, optparse-applicative, network, aeson, project-m36, random, MonadRandom, semigroups, parser-combinators, winery, curryer-rpc, prettyprinter, base64-bytestring
272+ Build-Depends: base, HUnit, Cabal, containers, hashable, unordered-containers, mtl, vector, time, hashable-time, bytestring, uuid, stm, deepseq, deepseq-generics,parallel, cassava, attoparsec, gnuplot, directory, temporary, haskeline, megaparsec, text, base64-bytestring, data-interval, filepath, transformers, stm-containers, list-t, websockets, optparse-applicative, network, aeson, project-m36, random, MonadRandom, semigroups, parser-combinators, winery, curryer-rpc, prettyprinter, base64-bytestring, modern-uri, http-types, http-conduit, base16-bytestring,cryptohash-sha256
268273 Default-Extensions: OverloadedStrings
269274 GHC-Options: -Wall -threaded
270275 Hs-Source-Dirs: ./src/bin, ., test/
src/bin/ProjectM36/Server/RemoteCallTypes/Json.hs+3
@@ -164,6 +164,9 @@ instance FromJSON SchemaError
164164instance ToJSON MergeError
165165instance FromJSON MergeError
166166
167+instance ToJSON ImportError'
168+instance FromJSON ImportError'
169+
167170instance ToJSON DatabaseContextFunctionError
168171instance FromJSON DatabaseContextFunctionError
169172
src/bin/TutorialD/Interpreter.hs+1
@@ -216,6 +216,7 @@ evalTutorialDInteractive sessionId conn safe interactive expr = case expr of
216216 unsafeError = pure $ DisplayErrorResult "File I/O operation prohibited."
217217 barf :: RelationalError -> IO TutorialDOperatorResult
218218 barf (ScriptError (OtherScriptCompilationError errStr)) = pure (DisplayErrorResult (T.pack errStr))
219+ barf (ParseError err) = pure (DisplayErrorResult err)
219220 barf err = return $ DisplayErrorResult (T.pack (show err))
220221 eHandler io = do
221222 eErr <- io
src/bin/TutorialD/Interpreter/Base.hs+21−1
@@ -44,6 +44,7 @@ import Data.List.NonEmpty as NE
4444import Data.Time.Clock
4545import Data.Time.Format
4646import Control.Monad (void)
47+import Data.Char
4748
4849#if !MIN_VERSION_megaparsec(7,0,0)
4950anySingle :: Parsec Void Text (Token Text)
@@ -82,7 +83,13 @@ type ParseStr = String
8283#endif
8384
8485spaceConsumer :: Parser ()
85−spaceConsumer = Lex.space (void spaceChar) (Lex.skipLineComment "--") (Lex.skipBlockComment "{-" "-}")
86+spaceConsumer = Lex.space (void sc) (Lex.skipLineComment "--") (Lex.skipBlockComment "{-" "-}")
87+ where
88+ sc = takeWhile1P (Just "horizontal whitespace") isHSpace'
89+
90+--backported from megaparsec 9
91+isHSpace' :: Char -> Bool
92+isHSpace' x = isSpace x && x /= '\n' && x /= '\r'
8693
8794opChar :: Parser Char
8895opChar = oneOf (":!#$%&*+./<=>?\\^|-~" :: String)-- remove "@" so it can be used as attribute marker without spaces
@@ -135,6 +142,9 @@ arrow = symbol "->"
135142semi :: Parser Text
136143semi = symbol ";"
137144
145+nline :: Parser Text
146+nline = (T.singleton <$> newline) <|> crlf
147+
138148integer :: Parser Integer
139149#if MIN_VERSION_megaparsec(6,0,0)
140150integer = Lex.signed (pure ()) Lex.decimal <* spaceConsumer
@@ -200,6 +210,9 @@ normalQuotedString = quote *> (T.pack <$> manyTill Lex.charLiteral quote)
200210quotedString :: Parser Text
201211quotedString = try tripleQuotedString <|> normalQuotedString
202212
213+quoted :: Parser a -> Parser a
214+quoted = between quote quote
215+
203216uuidP :: Parser U.UUID
204217uuidP = do
205218 uuidStart <- count 8 hexDigitChar
@@ -228,3 +241,10 @@ colonOp :: Text -> Parser ()
228241colonOp opStr = do
229242 _ <- string opStr <* (void spaceChar <|> eof) <* spaceConsumer
230243 pure ()
244+
245+hex :: Parser Text
246+hex = takeWhileP (Just "hexadecimal")
247+ (\c ->
248+ (c >= '0' && c <= '9')
249+ || (c >= 'a' && c <= 'f'))
250+
src/bin/TutorialD/Interpreter/DatabaseContextExpr.hs+5
@@ -56,6 +56,11 @@ multipleDatabaseContextExprP :: Parser DatabaseContextExpr
5656multipleDatabaseContextExprP =
5757 MultipleExpr <$> sepBy1 databaseContextExprP semi
5858
59+--good for reading from files instead of the repl
60+multilineDatabaseContextExprP :: Parser DatabaseContextExpr
61+multilineDatabaseContextExprP =
62+ MultipleExpr <$> sepBy1 databaseContextExprP (nline <|> semi)
63+
5964insertP :: Parser DatabaseContextExpr
6065insertP = do
6166 reservedOp "insert"
src/bin/TutorialD/Interpreter/Import/Base.hs+8−3
@@ -2,6 +2,8 @@
22module TutorialD.Interpreter.Import.Base where
33import ProjectM36.Base
44import ProjectM36.Error
5+import Text.URI (URI)
6+import Data.Text (Text)
57#if __GLASGOW_HASKELL__ < 804
68import Data.Monoid
79#endif
@@ -12,11 +14,14 @@ data RelVarDataImportOperator = RelVarDataImportOperator RelVarName FilePath (Re
1214instance Show RelVarDataImportOperator where
1315 show (RelVarDataImportOperator rv path _) = "RelVarDataImportOperator " <> show rv <> " " <> path
1416
17+type HashVerification = Maybe Text
18+
1519-- | import data into a database context
16−data DatabaseContextDataImportOperator = DatabaseContextDataImportOperator FilePath (FilePath -> IO (Either RelationalError DatabaseContextExpr))
20+data DatabaseContextDataImportOperator = DatabaseContextDataImportOperator URI HashVerification (URI -> HashVerification -> IO (Either RelationalError DatabaseContextExpr))
1721
1822instance Show DatabaseContextDataImportOperator where
19− show (DatabaseContextDataImportOperator path _) = "DatabaseContextDataImportOperator " <> path
23+ show (DatabaseContextDataImportOperator uri hash _) =
24+ "DatabaseContextDataImportOperator " <> show uri <> " " <> show hash
2025
2126-- perhaps create a structure to import a whole transaction graph section in the future
2227
@@ -24,4 +29,4 @@ evalRelVarDataImportOperator :: RelVarDataImportOperator -> TypeConstructorMappi
2429evalRelVarDataImportOperator (RelVarDataImportOperator relVarName path importFunc) tConsMap attrs = importFunc relVarName tConsMap attrs path
2530
2631evalDatabaseContextDataImportOperator :: DatabaseContextDataImportOperator -> IO (Either RelationalError DatabaseContextExpr)
27−evalDatabaseContextDataImportOperator (DatabaseContextDataImportOperator path importFunc) = importFunc path
32+evalDatabaseContextDataImportOperator (DatabaseContextDataImportOperator uri hash importFunc) = importFunc uri hash
src/bin/TutorialD/Interpreter/Import/CSV.hs+3−3
@@ -14,10 +14,10 @@ importCSVRelation relVarName tConsMap attrs pathIn = do
1414 --TODO: handle filesystem errors
1515 csvData <- try (BS.readFile pathIn) :: IO (Either IOError BS.ByteString)
1616 case csvData of
17− Left err -> return $ Left (ImportError $ T.pack (show err))
17+ Left err -> pure $ Left (ImportError (ImportFileError (T.pack (show err))))
1818 Right csvData' -> case csvAsRelation attrs tConsMap csvData' of
19− Left err -> return $ Left (ParseError $ T.pack (show err))
20− Right csvRel -> return $ Right (Insert relVarName (ExistingRelation csvRel))
19+ Left err -> pure $ Left (ParseError $ T.pack (show err))
20+ Right csvRel -> pure $ Right (Insert relVarName (ExistingRelation csvRel))
2121
2222importCSVP :: Parser RelVarDataImportOperator
2323importCSVP = do
src/bin/TutorialD/Interpreter/Import/TutorialD.hs+72−17
@@ -1,3 +1,4 @@
1+{-# LANGUAGE TypeApplications #-}
12module TutorialD.Interpreter.Import.TutorialD where
23import ProjectM36.Base
34import TutorialD.Interpreter.Import.Base
@@ -9,25 +10,79 @@ import qualified Data.Text as T
910import Control.Exception
1011import qualified Data.ByteString as BS
1112import qualified Data.Text.Encoding as TE
12−--import a file containing TutorialD database context expressions
13+import qualified Text.URI as URI
14+import Text.URI (URI)
15+import Crypto.Hash.SHA256
16+import Data.ByteString.Base16 as B16
17+import Network.HTTP.Simple
18+import Network.HTTP.Types
19+--import a file containing TutorialD database context expressions via local files or HTTP/HTTPS
1320
14−importTutorialD :: FilePath -> IO (Either RelationalError DatabaseContextExpr)
15−importTutorialD pathIn = do
16− eTutdBytes <- try (BS.readFile pathIn) :: IO (Either IOError BS.ByteString)
17− case eTutdBytes of
18− Left err -> return $ Left (ImportError $ T.pack (show err))
19− Right tutdBytes ->
20− case TE.decodeUtf8' tutdBytes of
21− Left err -> pure (Left (ImportError $ T.pack (show err)))
22− Right tutdUtf8 ->
23− case parse multipleDatabaseContextExprP "import" tutdUtf8 of
24− --parseErrorPretty is new in megaparsec 5
25− Left err -> pure (Left (PM36E.ParseError (T.pack (show err))))
26− Right expr -> pure (Right expr)
27−
21+importTutorialDFromFile :: URI -> HashVerification -> IO (Either RelationalError DatabaseContextExpr)
22+importTutorialDFromFile fileURI mHash = do
23+ case URI.uriPath fileURI of
24+ Just (False, _) -> do
25+ eTutdBytes <- try (BS.readFile filePath) :: IO (Either IOError BS.ByteString)
26+ case eTutdBytes of
27+ Left err -> pure $ Left (ImportError (ImportFileError (T.pack (show err))))
28+ Right tutdBytes ->
29+ case validateBytes tutdBytes mHash of
30+ Left err -> pure (Left err)
31+ Right tutdBytes' -> importTutorialDBytes tutdBytes'
32+ _ -> pure (Left (ImportError (InvalidFileURIError (URI.render fileURI))))
33+ where
34+ filePath = drop (length ("file://" :: String)) (URI.renderStr fileURI)
35+
36+
37+importTutorialDBytes :: BS.ByteString -> IO (Either RelationalError DatabaseContextExpr)
38+importTutorialDBytes tutdBytes =
39+ case TE.decodeUtf8' tutdBytes of
40+ Left err -> pure (Left (ImportError (ImportFileDecodeError (T.pack (show err)))))
41+ Right tutdUtf8 ->
42+ case parse (multilineDatabaseContextExprP <* eof) "import" tutdUtf8 of
43+ --parseErrorPretty is new in megaparsec 5
44+ Left err -> pure (Left (PM36E.ParseError (T.pack (errorBundlePretty err))))
45+ Right expr -> pure (Right expr)
46+
47+importTutorialDViaHTTP :: URI -> HashVerification -> IO (Either RelationalError DatabaseContextExpr)
48+importTutorialDViaHTTP uri mHash = do
49+ reqURI <- parseRequest (URI.renderStr uri)
50+ response <- httpBS reqURI
51+ let scode = getResponseStatus response
52+ if scode == ok200 then
53+ let byteBody = getResponseBody response in
54+ case validateBytes byteBody mHash of
55+ Right byteBody' -> do
56+ importTutorialDBytes byteBody'
57+ Left err -> pure (Left err)
58+ else
59+ pure (Left (ImportError (ImportDownloadError (T.pack ("download failed: " <> show scode)))))
60+
61+validateBytes :: BS.ByteString -> HashVerification -> Either RelationalError BS.ByteString
62+validateBytes tutdBytes mHash =
63+ case mHash of
64+ Nothing -> pure tutdBytes
65+ Just expectedHashText ->
66+ case B16.decode (TE.encodeUtf8 expectedHashText) of
67+ Left _ -> Left (ImportError (InvalidSHA256Error expectedHashText))
68+ Right expectedHash | hashBytes == expectedHash -> pure tutdBytes
69+ | otherwise ->
70+ let p = TE.decodeUtf8 . B16.encode in
71+ Left (ImportError (SHA256MismatchError (p expectedHash) (p hashBytes)))
72+ where
73+ hashBytes = hash tutdBytes
74+
75+
2876tutdImportP :: Parser DatabaseContextDataImportOperator
2977tutdImportP = do
3078 reserved ":importtutd"
31− path <- quotedString
79+ uri <- quoted URI.parser
3280 spaceConsumer
33− return $ DatabaseContextDataImportOperator (T.unpack path) importTutorialD
81+ mhash <- optional (quoted hex)
82+ let handler =
83+ case URI.unRText <$> URI.uriScheme uri of
84+ Nothing -> fail "URI scheme missing"
85+ Just "file" -> importTutorialDFromFile
86+ Just scheme | scheme `elem` ["http", "https"] -> importTutorialDViaHTTP
87+ | otherwise -> fail ("unsupported URI scheme: " <> show scheme)
88+ pure $ DatabaseContextDataImportOperator uri mhash handler
src/lib/ProjectM36/Error.hs+10−2
@@ -81,7 +81,7 @@ data RelationalError = NoSuchAttributeNamesError (S.Set AttributeName)
8181 | RelationValuedAttributesNotSupportedError [AttributeName]
8282 | NotificationNameInUseError NotificationName
8383 | NotificationNameNotInUseError NotificationName
84− | ImportError T.Text -- really? This should be broken out into some other error type- this has nothing to do with relational algebra
84+ | ImportError ImportError'
8585 | ExportError T.Text
8686 | UnhandledExceptionError String
8787 | MergeTransactionError MergeError
@@ -144,4 +144,12 @@ data SchemaError = RelVarReferencesMissing (S.Set RelVarName) |
144144 RelVarInReferencedMoreThanOnce RelVarName |
145145 RelVarOutReferencedMoreThanOnce RelVarName
146146 deriving (Show, Eq, Generic, Typeable, NFData)
147−
147+
148+
149+data ImportError' = InvalidSHA256Error T.Text
150+ | SHA256MismatchError T.Text T.Text
151+ | InvalidFileURIError T.Text
152+ | ImportFileDecodeError T.Text
153+ | ImportFileError T.Text
154+ | ImportDownloadError T.Text
155+ deriving (Show, Eq, Generic, Typeable, NFData)
src/lib/ProjectM36/Serialise/Error.hs+1
@@ -12,3 +12,4 @@ deriving via WineryVariant MergeError instance Serialise MergeError
1212deriving via WineryVariant ScriptCompilationError instance Serialise ScriptCompilationError
1313deriving via WineryVariant PersistenceError instance Serialise PersistenceError
1414deriving via WineryVariant SchemaError instance Serialise SchemaError
15+deriving via WineryVariant ImportError' instance Serialise ImportError'
test/TutorialD/Interpreter.hs+1−1
@@ -362,7 +362,7 @@ testComments :: Test
362362testComments = TestCase $ do
363363 (sessionId, dbconn) <- dateExamplesConnection emptyNotificationCallback
364364 mapM_ (executeTutorialD sessionId dbconn) [
365− ":branch testbranch --test comment\n",
365+ ":branch testbranch --test comment",
366366 ":jumphead {- test comment -} master"]
367367
368368-- create a graph and query from two disparate contexts
test/TutorialD/Interpreter/Import/TutorialD.hs+16−4
@@ -2,19 +2,21 @@ import Test.HUnit
22import ProjectM36.Base
33import TutorialD.Interpreter.Import.TutorialD
44import System.Exit
5+import qualified Data.Text as T
56import System.IO.Temp
67import qualified Data.Map as M
78import System.IO
89import qualified Data.ByteString as BS
910import qualified Data.Text.Encoding as TE
11+import Text.URI
1012
1113main :: IO ()
1214main = do
13− tcounts <- runTestTT $ TestList [testTutdImport]
15+ tcounts <- runTestTT $ TestList [testTutdFileImport, testTutdHTTPSImport]
1416 if errors tcounts + failures tcounts > 0 then exitFailure else exitSuccess
1517
16−testTutdImport :: Test
17−testTutdImport = TestCase $
18+testTutdFileImport :: Test
19+testTutdFileImport = TestCase $
1820 withSystemTempFile "m.tutd" $ \tempPath handle -> do
1921 BS.hPut handle (TE.encodeUtf8 "x:=relation{tuple{a 5,b \"spam\"}}; y:=relation{tuple{b \"漢字\"}}")
2022 hClose handle
@@ -24,6 +26,16 @@ testTutdImport = TestCase $
2426 ("b", NakedAtomExpr $ TextAtom "spam")])]),
2527 Assign "y" (MakeRelationFromExprs Nothing
2628 $ TupleExprs () [TupleExpr (M.fromList [("b", NakedAtomExpr (TextAtom "漢字"))])])]
27− imported <- importTutorialD tempPath
29+ fileURI <- mkURI (T.pack ("file://" <> tempPath))
30+ imported <- importTutorialDFromFile fileURI Nothing
2831 assertEqual "import tutd" (Right expectedExpr) imported
2932
33+testTutdHTTPSImport :: Test
34+testTutdHTTPSImport = TestCase $ do
35+ uri <- mkURI "https://raw.githubusercontent.com/agentm/project-m36/master/test/TutorialD/Interpreter/Import/httpimporttest.tutd"
36+ let hash = ""
37+ expected = Right NoOperation
38+ actual <- importTutorialDViaHTTP uri (Just hash)
39+ assertEqual "github https" expected actual
40+
41+