Wire in static http cache to prevent making extra requests.

generator/src/compile-elm.js

@@ -7,19 +7,20 @@ function runElm(/** @type string */ mode, /** @type any */ callback) {
   const mainElmFile = "../../src/Main.elm";
   const startingDir = process.cwd();
   process.chdir(elmBaseDirectory);
-  compileToString([mainElmFile], {}).then(function(data) {
+  compileToString([mainElmFile], {}).then(function (data) {
-    (function() {
+    (function () {
       const warnOriginal = console.warn;
-      console.warn = function() {};
+      console.warn = function () { };
       eval(data.toString());
       const app = Elm.Main.init({
-        flags: { secrets: process.env, mode }
+        flags: { secrets: process.env, mode, staticHttpCache: global.staticHttpCache }
       });
 
       app.ports.toJsPort.subscribe(payload => {
         process.chdir(startingDir);
 
         if (payload.tag === "Success") {
+          global.staticHttpCache = payload.args[0].staticHttpCache;
           callback(payload.args[0]);
         } else {
           console.log(payload.args[0]);

generator/src/elm-pages.js

@@ -13,6 +13,7 @@ const parseFrontmatter = require("./frontmatter.js");
 const path = require("path");
 const { ensureDirSync, deleteIfExists } = require('./file-helpers.js')
 global.builtAt = new Date();
+global.staticHttpCache = {};
 
 const contentGlobPath = "content/**/*.emu";
 

src/Pages/Internal/Platform/Cli.elm

@@ -48,6 +48,7 @@ type alias ToJsSuccessPayload pathKey =
     { pages : Dict String (Dict String String)
     , manifest : Manifest.Config pathKey
     , filesToGenerate : List FileToGenerate
+    , staticHttpCache : Dict String String
     , errors : List String
     }
 
@@ -66,8 +67,8 @@ toJsCodec =
                 Errors errorList ->
                     errorsTag errorList
 
-                Success { pages, manifest, filesToGenerate, errors } ->
+                Success { pages, manifest, filesToGenerate, errors, staticHttpCache } ->
-                    success (ToJsSuccessPayload pages manifest filesToGenerate errors)
+                    success (ToJsSuccessPayload pages manifest filesToGenerate staticHttpCache errors)
         )
         |> Codec.variant1 "Errors" Errors Codec.string
         |> Codec.variant1 "Success"
@@ -116,6 +117,9 @@ successCodec =
                 )
                 (Decode.succeed [])
             )
+        |> Codec.field "staticHttpCache"
+            .staticHttpCache
+            (Codec.dict Codec.string)
         |> Codec.field "errors" .errors (Codec.list Codec.string)
         |> Codec.buildObject
 
@@ -318,13 +322,20 @@ init :
 init toModel contentCache siteMetadata config flags =
     case
         Decode.decodeValue
-            (Decode.map2 Tuple.pair
+            (Decode.map3 (\a b c -> ( a, b, c ))
                 (Decode.field "secrets" SecretsDict.decoder)
                 (Decode.field "mode" modeDecoder)
+                (Decode.field "staticHttpCache"
+                    (Decode.dict
+                        (Decode.string
+                            |> Decode.map Just
+                        )
+                    )
+                )
             )
             flags
     of
-        Ok ( secrets, mode ) ->
+        Ok ( secrets, mode, staticHttpCache ) ->
             case contentCache of
                 Ok _ ->
                     case ContentCache.pagesWithErrors contentCache of
@@ -341,14 +352,14 @@ init toModel contentCache siteMetadata config flags =
                                 staticResponses =
                                     case requests of
                                         Ok okRequests ->
-                                            staticResponsesInit okRequests
+                                            staticResponsesInit staticHttpCache okRequests
 
                                         Err errors ->
                                             -- TODO need to handle errors better?
-                                            staticResponsesInit []
+                                            staticResponsesInit staticHttpCache []
 
                                 ( updatedRawResponses, effect ) =
-                                    sendStaticResponsesIfDone config siteMetadata mode secrets Dict.empty [] staticResponses
+                                    sendStaticResponsesIfDone config siteMetadata mode secrets staticHttpCache [] staticResponses
                             in
                             ( Model staticResponses secrets [] updatedRawResponses mode |> toModel
                             , effect
@@ -367,11 +378,11 @@ init toModel contentCache siteMetadata config flags =
                                 staticResponses =
                                     case requests of
                                         Ok okRequests ->
-                                            staticResponsesInit okRequests
+                                            staticResponsesInit staticHttpCache okRequests
 
                                         Err errors ->
                                             -- TODO need to handle errors better?
-                                            staticResponsesInit []
+                                            staticResponsesInit staticHttpCache []
                             in
                             updateAndSendPortIfDone
                                 config
@@ -380,7 +391,7 @@ init toModel contentCache siteMetadata config flags =
                                     staticResponses
                                     secrets
                                     pageErrors
-                                    Dict.empty
+                                    staticHttpCache
                                     mode
                                 )
                                 toModel
@@ -392,7 +403,7 @@ init toModel contentCache siteMetadata config flags =
                         (Model Dict.empty
                             secrets
                             (metadataParserErrors |> List.map Tuple.second)
-                            Dict.empty
+                            staticHttpCache
                             mode
                         )
                         toModel
@@ -573,13 +584,31 @@ combineMultipleErrors results =
         results
 
 
-staticResponsesInit : List ( PagePath pathKey, StaticHttp.Request value ) -> StaticResponses
+staticResponsesInit : Dict String (Maybe String) -> List ( PagePath pathKey, StaticHttp.Request value ) -> StaticResponses
-staticResponsesInit list =
+staticResponsesInit staticHttpCache list =
     list
         |> List.map
             (\( path, staticRequest ) ->
+                let
+                    entry =
+                        NotFetched (staticRequest |> StaticHttp.map (\_ -> ())) Dict.empty
+
+                    updatedEntry =
+                        staticHttpCache
+                            |> dictCompact
+                            |> Dict.toList
+                            |> List.foldl
+                                (\( hashedRequest, response ) entrySoFar ->
+                                    entrySoFar
+                                        |> addEntry
+                                            staticHttpCache
+                                            hashedRequest
+                                            (Ok response)
+                                )
+                                entry
+                in
                 ( PagePath.toString path
-                , NotFetched (staticRequest |> StaticHttp.map (\_ -> ())) Dict.empty
+                , updatedEntry
                 )
             )
         |> Dict.fromList
@@ -633,6 +662,36 @@ staticResponsesUpdate newEntry model =
     }
 
 
+addEntry : Dict String (Maybe String) -> String -> Result () String -> StaticHttpResult -> StaticHttpResult
+addEntry globalRawResponses hashedRequest rawResponse ((NotFetched request rawResponses) as entry) =
+    let
+        realUrls =
+            globalRawResponses
+                |> dictCompact
+                |> StaticHttpRequest.resolveUrls ApplicationType.Cli request
+                |> Tuple.second
+                |> List.map Secrets.maskedLookup
+                |> List.map HashRequest.hash
+
+        includesUrl =
+            List.member
+                hashedRequest
+                realUrls
+    in
+    if includesUrl then
+        let
+            updatedRawResponses =
+                Dict.insert
+                    hashedRequest
+                    rawResponse
+                    rawResponses
+        in
+        NotFetched request updatedRawResponses
+
+    else
+        entry
+
+
 isJust : Maybe a -> Bool
 isJust maybeValue =
     case maybeValue of
@@ -882,11 +941,19 @@ sendStaticResponsesIfDone config siteMetadata mode secrets allRawResponses error
             (encodeStaticResponses mode staticResponses)
             config.manifest
             generatedOkayFiles
+            allRawResponses
             allErrors
         )
 
 
-toJsPayload encodedStatic manifest generated allErrors =
+toJsPayload :
+    Dict String (Dict String String)
+    -> Manifest.Config pathKey
+    -> List FileToGenerate
+    -> Dict String (Maybe String)
+    -> List { title : String, message : List Terminal.Text, fatal : Bool }
+    -> Effect pathKey
+toJsPayload encodedStatic manifest generated allRawResponses allErrors =
     SendJsData <|
         if allErrors |> List.filter .fatal |> List.isEmpty then
             Success
@@ -894,6 +961,15 @@ toJsPayload encodedStatic manifest generated allErrors =
                     encodedStatic
                     manifest
                     generated
+                    (allRawResponses
+                        |> Dict.toList
+                        |> List.filterMap
+                            (\( key, maybeValue ) ->
+                                maybeValue
+                                    |> Maybe.map (\value -> ( key, value ))
+                            )
+                        |> Dict.fromList
+                    )
                     (List.map BuildError.errorToString allErrors)
                 )
 

tests/StaticHttpRequestsTests.elm

@@ -6,6 +6,7 @@ import Expect
 import Html
 import Json.Decode as JD
 import Json.Decode.Exploration
+import Json.Encode as Encode
 import OptimizedDecoder as Decode exposing (Decoder)
 import Pages.ContentCache as ContentCache
 import Pages.Document as Document
@@ -605,11 +606,41 @@ Body: """)
                             ]
                           )
                         ]
+        , describe "staticHttpCache"
+            [ test "it doesn't perform http requests that are provided in the http cache flag" <|
+                \() ->
+                    startWithHttpCache
+                        [ ( { url = "https://api.github.com/repos/dillonkearns/elm-pages"
+                            , method = "GET"
+                            , headers = []
+                            , body = StaticHttpBody.EmptyBody
+                            }
+                          , """{"stargazer_count":86}"""
+                          )
+                        ]
+                        [ ( []
+                          , StaticHttp.get (Secrets.succeed "https://api.github.com/repos/dillonkearns/elm-pages") starDecoder
+                          )
+                        ]
+                        |> expectSuccess
+                            [ ( ""
+                              , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
+                                  , """{"stargazer_count":86}"""
+                                  )
+                                ]
+                              )
+                            ]
+            ]
         ]
 
 
 start : List ( List String, StaticHttp.Request a ) -> ProgramTest Main.Model Main.Msg (Main.Effect PathKey)
 start pages =
+    startWithHttpCache [] pages
+
+
+startWithHttpCache : List ( Request.Request, String ) -> List ( List String, StaticHttp.Request a ) -> ProgramTest Main.Model Main.Msg (Main.Effect PathKey)
+startWithHttpCache staticHttpCache pages =
     let
         document =
             Document.fromList
@@ -671,6 +702,30 @@ start pages =
             , pathKey = PathKey
             , onPageChange = \_ -> ()
             }
+
+        encodedFlags =
+            --{"secrets":
+            --        {"API_KEY": "ABCD1234","BEARER": "XYZ789"}, "mode": "prod", "staticHttpCache": {}
+            --        }
+            Encode.object
+                [ ( "secrets"
+                  , [ ( "API_KEY", "ABCD1234" )
+                    , ( "BEARER", "XYZ789" )
+                    ]
+                        |> Dict.fromList
+                        |> Encode.dict identity Encode.string
+                  )
+                , ( "mode", Encode.string "prod" )
+                , ( "staticHttpCache", encodedStaticHttpCache )
+                ]
+
+        encodedStaticHttpCache =
+            staticHttpCache
+                |> List.map
+                    (\( request, httpResponseString ) ->
+                        ( Request.hash request, Encode.string httpResponseString )
+                    )
+                |> Encode.object
     in
     {-
        (Model -> model)
@@ -686,9 +741,7 @@ start pages =
         , view = \_ -> { title = "", body = [] }
         }
         |> ProgramTest.withSimulatedEffects simulateEffects
-        |> ProgramTest.start (flags """{"secrets":
+        |> ProgramTest.start (flags (Encode.encode 0 encodedFlags))
-        {"API_KEY": "ABCD1234","BEARER": "XYZ789"}, "mode": "prod"
-        }""")
 
 
 flags : String -> JD.Value
@@ -837,27 +890,31 @@ expectSuccess expectedRequests previous =
         |> ProgramTest.expectOutgoingPortValues
             "toJsPort"
             (Codec.decoder Main.toJsCodec)
-            (Expect.equal
+            (\value ->
-                [ Main.Success
+                case value of
-                    { pages =
+                    [ Main.Success portPayload ] ->
-                        expectedRequests
+                        portPayload.pages
-                            |> List.map
+                            |> Expect.equal
-                                (\( url, requests ) ->
+                                (expectedRequests
-                                    ( url
+                                    |> List.map
-                                    , requests
+                                        (\( url, requests ) ->
-                                        |> List.map
+                                            ( url
-                                            (\( request, response ) ->
+                                            , requests
-                                                ( Request.hash request, response )
+                                                |> List.map
+                                                    (\( request, response ) ->
+                                                        ( Request.hash request, response )
+                                                    )
+                                                |> Dict.fromList
                                             )
-                                        |> Dict.fromList
+                                        )
-                                    )
+                                    |> Dict.fromList
                                 )
-                            |> Dict.fromList
+
-                    , manifest = manifest
+                    [ _ ] ->
-                    , filesToGenerate = []
+                        Expect.fail "Expected success port."
-                    , errors = []
+
-                    }
+                    _ ->
-                ]
+                        Expect.fail ("Expected ports to be called once, but instead there were " ++ String.fromInt (List.length value) ++ " calls.")
             )