From 3c690106f430c5737330a1064d97ddd58ecdf9aa Mon Sep 17 00:00:00 2001 From: Dillon Kearns Date: Sun, 19 Apr 2020 08:17:51 -0700 Subject: [PATCH] Wire in static http cache to prevent making extra requests. --- generator/src/compile-elm.js | 9 +-- generator/src/elm-pages.js | 1 + src/Pages/Internal/Platform/Cli.elm | 106 ++++++++++++++++++++++++---- tests/StaticHttpRequestsTests.elm | 101 ++++++++++++++++++++------ 4 files changed, 176 insertions(+), 41 deletions(-) diff --git a/generator/src/compile-elm.js b/generator/src/compile-elm.js index 9ab791e1..976dea76 100644 --- a/generator/src/compile-elm.js +++ b/generator/src/compile-elm.js @@ -7,19 +7,20 @@ function runElm(/** @type string */ mode, /** @type any */ callback) { const mainElmFile = "../../src/Main.elm"; const startingDir = process.cwd(); process.chdir(elmBaseDirectory); - compileToString([mainElmFile], {}).then(function(data) { - (function() { + compileToString([mainElmFile], {}).then(function (data) { + (function () { const warnOriginal = console.warn; - console.warn = function() {}; + console.warn = function () { }; eval(data.toString()); const app = Elm.Main.init({ - flags: { secrets: process.env, mode } + flags: { secrets: process.env, mode, staticHttpCache: global.staticHttpCache } }); app.ports.toJsPort.subscribe(payload => { process.chdir(startingDir); if (payload.tag === "Success") { + global.staticHttpCache = payload.args[0].staticHttpCache; callback(payload.args[0]); } else { console.log(payload.args[0]); diff --git a/generator/src/elm-pages.js b/generator/src/elm-pages.js index 9b150b94..bfeaaa72 100755 --- a/generator/src/elm-pages.js +++ b/generator/src/elm-pages.js @@ -13,6 +13,7 @@ const parseFrontmatter = require("./frontmatter.js"); const path = require("path"); const { ensureDirSync, deleteIfExists } = require('./file-helpers.js') global.builtAt = new Date(); +global.staticHttpCache = {}; const contentGlobPath = "content/**/*.emu"; diff --git a/src/Pages/Internal/Platform/Cli.elm b/src/Pages/Internal/Platform/Cli.elm index 800e2329..9e3f3a66 100644 --- a/src/Pages/Internal/Platform/Cli.elm +++ b/src/Pages/Internal/Platform/Cli.elm @@ -48,6 +48,7 @@ type alias ToJsSuccessPayload pathKey = { pages : Dict String (Dict String String) , manifest : Manifest.Config pathKey , filesToGenerate : List FileToGenerate + , staticHttpCache : Dict String String , errors : List String } @@ -66,8 +67,8 @@ toJsCodec = Errors errorList -> errorsTag errorList - Success { pages, manifest, filesToGenerate, errors } -> - success (ToJsSuccessPayload pages manifest filesToGenerate errors) + Success { pages, manifest, filesToGenerate, errors, staticHttpCache } -> + success (ToJsSuccessPayload pages manifest filesToGenerate staticHttpCache errors) ) |> Codec.variant1 "Errors" Errors Codec.string |> Codec.variant1 "Success" @@ -116,6 +117,9 @@ successCodec = ) (Decode.succeed []) ) + |> Codec.field "staticHttpCache" + .staticHttpCache + (Codec.dict Codec.string) |> Codec.field "errors" .errors (Codec.list Codec.string) |> Codec.buildObject @@ -318,13 +322,20 @@ init : init toModel contentCache siteMetadata config flags = case Decode.decodeValue - (Decode.map2 Tuple.pair + (Decode.map3 (\a b c -> ( a, b, c )) (Decode.field "secrets" SecretsDict.decoder) (Decode.field "mode" modeDecoder) + (Decode.field "staticHttpCache" + (Decode.dict + (Decode.string + |> Decode.map Just + ) + ) + ) ) flags of - Ok ( secrets, mode ) -> + Ok ( secrets, mode, staticHttpCache ) -> case contentCache of Ok _ -> case ContentCache.pagesWithErrors contentCache of @@ -341,14 +352,14 @@ init toModel contentCache siteMetadata config flags = staticResponses = case requests of Ok okRequests -> - staticResponsesInit okRequests + staticResponsesInit staticHttpCache okRequests Err errors -> -- TODO need to handle errors better? - staticResponsesInit [] + staticResponsesInit staticHttpCache [] ( updatedRawResponses, effect ) = - sendStaticResponsesIfDone config siteMetadata mode secrets Dict.empty [] staticResponses + sendStaticResponsesIfDone config siteMetadata mode secrets staticHttpCache [] staticResponses in ( Model staticResponses secrets [] updatedRawResponses mode |> toModel , effect @@ -367,11 +378,11 @@ init toModel contentCache siteMetadata config flags = staticResponses = case requests of Ok okRequests -> - staticResponsesInit okRequests + staticResponsesInit staticHttpCache okRequests Err errors -> -- TODO need to handle errors better? - staticResponsesInit [] + staticResponsesInit staticHttpCache [] in updateAndSendPortIfDone config @@ -380,7 +391,7 @@ init toModel contentCache siteMetadata config flags = staticResponses secrets pageErrors - Dict.empty + staticHttpCache mode ) toModel @@ -392,7 +403,7 @@ init toModel contentCache siteMetadata config flags = (Model Dict.empty secrets (metadataParserErrors |> List.map Tuple.second) - Dict.empty + staticHttpCache mode ) toModel @@ -573,13 +584,31 @@ combineMultipleErrors results = results -staticResponsesInit : List ( PagePath pathKey, StaticHttp.Request value ) -> StaticResponses -staticResponsesInit list = +staticResponsesInit : Dict String (Maybe String) -> List ( PagePath pathKey, StaticHttp.Request value ) -> StaticResponses +staticResponsesInit staticHttpCache list = list |> List.map (\( path, staticRequest ) -> + let + entry = + NotFetched (staticRequest |> StaticHttp.map (\_ -> ())) Dict.empty + + updatedEntry = + staticHttpCache + |> dictCompact + |> Dict.toList + |> List.foldl + (\( hashedRequest, response ) entrySoFar -> + entrySoFar + |> addEntry + staticHttpCache + hashedRequest + (Ok response) + ) + entry + in ( PagePath.toString path - , NotFetched (staticRequest |> StaticHttp.map (\_ -> ())) Dict.empty + , updatedEntry ) ) |> Dict.fromList @@ -633,6 +662,36 @@ staticResponsesUpdate newEntry model = } +addEntry : Dict String (Maybe String) -> String -> Result () String -> StaticHttpResult -> StaticHttpResult +addEntry globalRawResponses hashedRequest rawResponse ((NotFetched request rawResponses) as entry) = + let + realUrls = + globalRawResponses + |> dictCompact + |> StaticHttpRequest.resolveUrls ApplicationType.Cli request + |> Tuple.second + |> List.map Secrets.maskedLookup + |> List.map HashRequest.hash + + includesUrl = + List.member + hashedRequest + realUrls + in + if includesUrl then + let + updatedRawResponses = + Dict.insert + hashedRequest + rawResponse + rawResponses + in + NotFetched request updatedRawResponses + + else + entry + + isJust : Maybe a -> Bool isJust maybeValue = case maybeValue of @@ -882,11 +941,19 @@ sendStaticResponsesIfDone config siteMetadata mode secrets allRawResponses error (encodeStaticResponses mode staticResponses) config.manifest generatedOkayFiles + allRawResponses allErrors ) -toJsPayload encodedStatic manifest generated allErrors = +toJsPayload : + Dict String (Dict String String) + -> Manifest.Config pathKey + -> List FileToGenerate + -> Dict String (Maybe String) + -> List { title : String, message : List Terminal.Text, fatal : Bool } + -> Effect pathKey +toJsPayload encodedStatic manifest generated allRawResponses allErrors = SendJsData <| if allErrors |> List.filter .fatal |> List.isEmpty then Success @@ -894,6 +961,15 @@ toJsPayload encodedStatic manifest generated allErrors = encodedStatic manifest generated + (allRawResponses + |> Dict.toList + |> List.filterMap + (\( key, maybeValue ) -> + maybeValue + |> Maybe.map (\value -> ( key, value )) + ) + |> Dict.fromList + ) (List.map BuildError.errorToString allErrors) ) diff --git a/tests/StaticHttpRequestsTests.elm b/tests/StaticHttpRequestsTests.elm index 960b31de..e9d535f1 100644 --- a/tests/StaticHttpRequestsTests.elm +++ b/tests/StaticHttpRequestsTests.elm @@ -6,6 +6,7 @@ import Expect import Html import Json.Decode as JD import Json.Decode.Exploration +import Json.Encode as Encode import OptimizedDecoder as Decode exposing (Decoder) import Pages.ContentCache as ContentCache import Pages.Document as Document @@ -605,11 +606,41 @@ Body: """) ] ) ] + , describe "staticHttpCache" + [ test "it doesn't perform http requests that are provided in the http cache flag" <| + \() -> + startWithHttpCache + [ ( { url = "https://api.github.com/repos/dillonkearns/elm-pages" + , method = "GET" + , headers = [] + , body = StaticHttpBody.EmptyBody + } + , """{"stargazer_count":86}""" + ) + ] + [ ( [] + , StaticHttp.get (Secrets.succeed "https://api.github.com/repos/dillonkearns/elm-pages") starDecoder + ) + ] + |> expectSuccess + [ ( "" + , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages" + , """{"stargazer_count":86}""" + ) + ] + ) + ] + ] ] start : List ( List String, StaticHttp.Request a ) -> ProgramTest Main.Model Main.Msg (Main.Effect PathKey) start pages = + startWithHttpCache [] pages + + +startWithHttpCache : List ( Request.Request, String ) -> List ( List String, StaticHttp.Request a ) -> ProgramTest Main.Model Main.Msg (Main.Effect PathKey) +startWithHttpCache staticHttpCache pages = let document = Document.fromList @@ -671,6 +702,30 @@ start pages = , pathKey = PathKey , onPageChange = \_ -> () } + + encodedFlags = + --{"secrets": + -- {"API_KEY": "ABCD1234","BEARER": "XYZ789"}, "mode": "prod", "staticHttpCache": {} + -- } + Encode.object + [ ( "secrets" + , [ ( "API_KEY", "ABCD1234" ) + , ( "BEARER", "XYZ789" ) + ] + |> Dict.fromList + |> Encode.dict identity Encode.string + ) + , ( "mode", Encode.string "prod" ) + , ( "staticHttpCache", encodedStaticHttpCache ) + ] + + encodedStaticHttpCache = + staticHttpCache + |> List.map + (\( request, httpResponseString ) -> + ( Request.hash request, Encode.string httpResponseString ) + ) + |> Encode.object in {- (Model -> model) @@ -686,9 +741,7 @@ start pages = , view = \_ -> { title = "", body = [] } } |> ProgramTest.withSimulatedEffects simulateEffects - |> ProgramTest.start (flags """{"secrets": - {"API_KEY": "ABCD1234","BEARER": "XYZ789"}, "mode": "prod" - }""") + |> ProgramTest.start (flags (Encode.encode 0 encodedFlags)) flags : String -> JD.Value @@ -837,27 +890,31 @@ expectSuccess expectedRequests previous = |> ProgramTest.expectOutgoingPortValues "toJsPort" (Codec.decoder Main.toJsCodec) - (Expect.equal - [ Main.Success - { pages = - expectedRequests - |> List.map - (\( url, requests ) -> - ( url - , requests - |> List.map - (\( request, response ) -> - ( Request.hash request, response ) + (\value -> + case value of + [ Main.Success portPayload ] -> + portPayload.pages + |> Expect.equal + (expectedRequests + |> List.map + (\( url, requests ) -> + ( url + , requests + |> List.map + (\( request, response ) -> + ( Request.hash request, response ) + ) + |> Dict.fromList ) - |> Dict.fromList - ) + ) + |> Dict.fromList ) - |> Dict.fromList - , manifest = manifest - , filesToGenerate = [] - , errors = [] - } - ] + + [ _ ] -> + Expect.fail "Expected success port." + + _ -> + Expect.fail ("Expected ports to be called once, but instead there were " ++ String.fromInt (List.length value) ++ " calls.") )