mirror of
https://github.com/dillonkearns/elm-pages-v3-beta.git
synced 2024-12-23 20:03:31 +03:00
Merge pull request #145 from dillonkearns/performance-tuning
StaticHttp build performance tuning
This commit is contained in:
commit
4ce530f041
@ -24,6 +24,7 @@ import Json.Decode as Decode
|
||||
import Pages.Document as Document exposing (Document)
|
||||
import Pages.Internal.String as String
|
||||
import Pages.PagePath as PagePath exposing (PagePath)
|
||||
import RequestsAndPending exposing (RequestsAndPending)
|
||||
import Task exposing (Task)
|
||||
import TerminalText as Terminal
|
||||
import Url exposing (Url)
|
||||
@ -423,7 +424,7 @@ httpTask url =
|
||||
|
||||
type alias ContentJson body =
|
||||
{ body : body
|
||||
, staticData : Dict String String
|
||||
, staticData : RequestsAndPending
|
||||
}
|
||||
|
||||
|
||||
@ -431,7 +432,7 @@ contentJsonDecoder : Decode.Decoder (ContentJson String)
|
||||
contentJsonDecoder =
|
||||
Decode.map2 ContentJson
|
||||
(Decode.field "body" Decode.string)
|
||||
(Decode.field "staticData" (Decode.dict Decode.string))
|
||||
(Decode.field "staticData" RequestsAndPending.decoder)
|
||||
|
||||
|
||||
update :
|
||||
|
@ -21,6 +21,7 @@ import Pages.Manifest as Manifest
|
||||
import Pages.PagePath as PagePath exposing (PagePath)
|
||||
import Pages.StaticHttp as StaticHttp
|
||||
import Pages.StaticHttpRequest as StaticHttpRequest
|
||||
import RequestsAndPending exposing (RequestsAndPending)
|
||||
import Result.Extra
|
||||
import Task exposing (Task)
|
||||
import Url exposing (Url)
|
||||
@ -250,7 +251,7 @@ type alias Flags =
|
||||
|
||||
type alias ContentJson =
|
||||
{ body : String
|
||||
, staticData : Dict String String
|
||||
, staticData : RequestsAndPending
|
||||
}
|
||||
|
||||
|
||||
@ -258,7 +259,7 @@ contentJsonDecoder : Decode.Decoder ContentJson
|
||||
contentJsonDecoder =
|
||||
Decode.map2 ContentJson
|
||||
(Decode.field "body" Decode.string)
|
||||
(Decode.field "staticData" (Decode.dict Decode.string))
|
||||
(Decode.field "staticData" RequestsAndPending.decoder)
|
||||
|
||||
|
||||
init :
|
||||
|
@ -60,6 +60,7 @@ type alias Model =
|
||||
, errors : List BuildError
|
||||
, allRawResponses : Dict String (Maybe String)
|
||||
, mode : Mode
|
||||
, pendingRequests : List { masked : RequestDetails, unmasked : RequestDetails }
|
||||
}
|
||||
|
||||
|
||||
@ -102,7 +103,8 @@ type alias Config pathKey userMsg userModel metadata view =
|
||||
->
|
||||
StaticHttp.Request
|
||||
(List
|
||||
(Result String
|
||||
(Result
|
||||
String
|
||||
{ path : List String
|
||||
, content : String
|
||||
}
|
||||
@ -265,7 +267,7 @@ init toModel contentCache siteMetadata config flags =
|
||||
StaticResponses.init staticHttpCache siteMetadata config []
|
||||
in
|
||||
StaticResponses.nextStep config siteMetadata mode secrets staticHttpCache [] staticResponses
|
||||
|> nextStepToEffect (Model staticResponses secrets [] staticHttpCache mode)
|
||||
|> nextStepToEffect (Model staticResponses secrets [] staticHttpCache mode [])
|
||||
|> Tuple.mapFirst toModel
|
||||
|
||||
pageErrors ->
|
||||
@ -296,6 +298,7 @@ init toModel contentCache siteMetadata config flags =
|
||||
pageErrors
|
||||
staticHttpCache
|
||||
mode
|
||||
[]
|
||||
)
|
||||
toModel
|
||||
|
||||
@ -308,6 +311,7 @@ init toModel contentCache siteMetadata config flags =
|
||||
(metadataParserErrors |> List.map Tuple.second)
|
||||
staticHttpCache
|
||||
mode
|
||||
[]
|
||||
)
|
||||
toModel
|
||||
|
||||
@ -324,6 +328,7 @@ init toModel contentCache siteMetadata config flags =
|
||||
]
|
||||
Dict.empty
|
||||
Mode.Dev
|
||||
[]
|
||||
)
|
||||
toModel
|
||||
|
||||
@ -363,7 +368,11 @@ update siteMetadata config msg model =
|
||||
updatedModel =
|
||||
(case response of
|
||||
Ok okResponse ->
|
||||
model
|
||||
{ model
|
||||
| pendingRequests =
|
||||
model.pendingRequests
|
||||
|> List.filter (\pending -> pending /= request)
|
||||
}
|
||||
|
||||
Err error ->
|
||||
{ model
|
||||
@ -421,8 +430,18 @@ nextStepToEffect : Model -> StaticResponses.NextStep pathKey -> ( Model, Effect
|
||||
nextStepToEffect model nextStep =
|
||||
case nextStep of
|
||||
StaticResponses.Continue updatedAllRawResponses httpRequests ->
|
||||
( { model | allRawResponses = updatedAllRawResponses }
|
||||
, httpRequests
|
||||
let
|
||||
nextAndPending =
|
||||
model.pendingRequests ++ httpRequests
|
||||
|
||||
doNow =
|
||||
nextAndPending
|
||||
|
||||
pending =
|
||||
[]
|
||||
in
|
||||
( { model | allRawResponses = updatedAllRawResponses, pendingRequests = pending }
|
||||
, doNow
|
||||
|> List.map Effect.FetchHttp
|
||||
|> Effect.Batch
|
||||
)
|
||||
@ -446,7 +465,8 @@ staticResponseForPage :
|
||||
}
|
||||
)
|
||||
->
|
||||
Result (List BuildError)
|
||||
Result
|
||||
(List BuildError)
|
||||
(List
|
||||
( PagePath pathKey
|
||||
, StaticHttp.Request
|
||||
|
@ -11,6 +11,8 @@ import Pages.PagePath as PagePath exposing (PagePath)
|
||||
import Pages.StaticHttp as StaticHttp exposing (RequestDetails)
|
||||
import Pages.StaticHttp.Request as HashRequest
|
||||
import Pages.StaticHttpRequest as StaticHttpRequest
|
||||
import RequestsAndPending exposing (RequestsAndPending)
|
||||
import Result.Extra
|
||||
import Secrets
|
||||
import SecretsDict exposing (SecretsDict)
|
||||
import Set
|
||||
@ -49,7 +51,8 @@ init :
|
||||
->
|
||||
StaticHttp.Request
|
||||
(List
|
||||
(Result String
|
||||
(Result
|
||||
String
|
||||
{ path : List String
|
||||
, content : String
|
||||
}
|
||||
@ -110,23 +113,9 @@ init staticHttpCache siteMetadataResult config list =
|
||||
let
|
||||
entry =
|
||||
NotFetched (staticRequest |> StaticHttp.map (\_ -> ())) Dict.empty
|
||||
|
||||
updatedEntry =
|
||||
staticHttpCache
|
||||
|> dictCompact
|
||||
|> Dict.toList
|
||||
|> List.foldl
|
||||
(\( hashedRequest, response ) entrySoFar ->
|
||||
entrySoFar
|
||||
|> addEntry
|
||||
staticHttpCache
|
||||
hashedRequest
|
||||
(Ok response)
|
||||
)
|
||||
entry
|
||||
in
|
||||
( PagePath.toString path
|
||||
, updatedEntry
|
||||
, entry
|
||||
)
|
||||
)
|
||||
|> List.append [ generateFilesStaticRequest ]
|
||||
@ -160,82 +149,17 @@ update newEntry model =
|
||||
in
|
||||
{ model
|
||||
| allRawResponses = updatedAllResponses
|
||||
, staticResponses =
|
||||
case model.staticResponses of
|
||||
StaticResponses staticResponses ->
|
||||
staticResponses
|
||||
|> Dict.map
|
||||
(\pageUrl entry ->
|
||||
case entry of
|
||||
NotFetched request rawResponses ->
|
||||
let
|
||||
realUrls =
|
||||
updatedAllResponses
|
||||
|> dictCompact
|
||||
|> StaticHttpRequest.resolveUrls ApplicationType.Cli request
|
||||
|> Tuple.second
|
||||
|> List.map Secrets.maskedLookup
|
||||
|> List.map HashRequest.hash
|
||||
|
||||
includesUrl =
|
||||
List.member
|
||||
(HashRequest.hash newEntry.request.masked)
|
||||
realUrls
|
||||
in
|
||||
if includesUrl then
|
||||
let
|
||||
updatedRawResponses =
|
||||
Dict.insert
|
||||
(HashRequest.hash newEntry.request.masked)
|
||||
newEntry.response
|
||||
rawResponses
|
||||
in
|
||||
NotFetched request updatedRawResponses
|
||||
|
||||
else
|
||||
entry
|
||||
)
|
||||
|> StaticResponses
|
||||
}
|
||||
|
||||
|
||||
addEntry :
|
||||
Dict String (Maybe String)
|
||||
-> String
|
||||
-> Result () String
|
||||
-> StaticHttpResult
|
||||
-> StaticHttpResult
|
||||
addEntry globalRawResponses hashedRequest rawResponse ((NotFetched request rawResponses) as entry) =
|
||||
let
|
||||
realUrls =
|
||||
globalRawResponses
|
||||
|> dictCompact
|
||||
|> StaticHttpRequest.resolveUrls ApplicationType.Cli request
|
||||
|> Tuple.second
|
||||
|> List.map Secrets.maskedLookup
|
||||
|> List.map HashRequest.hash
|
||||
|
||||
includesUrl =
|
||||
List.member
|
||||
hashedRequest
|
||||
realUrls
|
||||
in
|
||||
if includesUrl then
|
||||
let
|
||||
updatedRawResponses =
|
||||
Dict.insert
|
||||
hashedRequest
|
||||
rawResponse
|
||||
rawResponses
|
||||
in
|
||||
NotFetched request updatedRawResponses
|
||||
|
||||
else
|
||||
entry
|
||||
dictCompact : Dict String (Maybe a) -> Dict String a
|
||||
dictCompact dict =
|
||||
dict
|
||||
|> Dict.Extra.filterMap (\key value -> value)
|
||||
|
||||
|
||||
encode : Mode -> StaticResponses -> Dict String (Dict String String)
|
||||
encode mode (StaticResponses staticResponses) =
|
||||
encode : RequestsAndPending -> Mode -> StaticResponses -> Dict String (Dict String String)
|
||||
encode requestsAndPending mode (StaticResponses staticResponses) =
|
||||
staticResponses
|
||||
|> Dict.filter
|
||||
(\key value ->
|
||||
@ -244,36 +168,16 @@ encode mode (StaticResponses staticResponses) =
|
||||
|> Dict.map
|
||||
(\path result ->
|
||||
case result of
|
||||
NotFetched request rawResponsesDict ->
|
||||
let
|
||||
relevantResponses =
|
||||
Dict.map
|
||||
(\_ ->
|
||||
-- TODO avoid running this code at all if there are errors here
|
||||
Result.withDefault ""
|
||||
)
|
||||
rawResponsesDict
|
||||
|
||||
strippedResponses : Dict String String
|
||||
strippedResponses =
|
||||
-- TODO should this return an Err and handle that here?
|
||||
StaticHttpRequest.strippedResponses ApplicationType.Cli request relevantResponses
|
||||
in
|
||||
NotFetched request _ ->
|
||||
case mode of
|
||||
Mode.Dev ->
|
||||
relevantResponses
|
||||
StaticHttpRequest.strippedResponses ApplicationType.Cli request requestsAndPending
|
||||
|
||||
Mode.Prod ->
|
||||
strippedResponses
|
||||
StaticHttpRequest.strippedResponses ApplicationType.Cli request requestsAndPending
|
||||
)
|
||||
|
||||
|
||||
dictCompact : Dict String (Maybe a) -> Dict String a
|
||||
dictCompact dict =
|
||||
dict
|
||||
|> Dict.Extra.filterMap (\key value -> value)
|
||||
|
||||
|
||||
cliDictKey : String
|
||||
cliDictKey =
|
||||
"////elm-pages-CLI////"
|
||||
@ -297,7 +201,8 @@ nextStep :
|
||||
->
|
||||
StaticHttp.Request
|
||||
(List
|
||||
(Result String
|
||||
(Result
|
||||
String
|
||||
{ path : List String
|
||||
, content : String
|
||||
}
|
||||
@ -307,7 +212,7 @@ nextStep :
|
||||
-> Result (List BuildError) (List ( PagePath pathKey, metadata ))
|
||||
-> Mode
|
||||
-> SecretsDict
|
||||
-> Dict String (Maybe String)
|
||||
-> RequestsAndPending
|
||||
-> List BuildError
|
||||
-> StaticResponses
|
||||
-> NextStep pathKey
|
||||
@ -353,7 +258,7 @@ nextStep config siteMetadata mode secrets allRawResponses errors (StaticResponse
|
||||
resolvedGenerateFilesResult =
|
||||
StaticHttpRequest.resolve ApplicationType.Cli
|
||||
(config.generateFiles metadataForGenerateFiles)
|
||||
(allRawResponses |> Dict.Extra.filterMap (\key value -> value))
|
||||
(allRawResponses |> Dict.Extra.filterMap (\key value -> Just value))
|
||||
|
||||
generatedOkayFiles : List { path : List String, content : String }
|
||||
generatedOkayFiles =
|
||||
@ -400,41 +305,28 @@ nextStep config siteMetadata mode secrets allRawResponses errors (StaticResponse
|
||||
case entry of
|
||||
NotFetched request rawResponses ->
|
||||
let
|
||||
usableRawResponses : Dict String String
|
||||
usableRawResponses =
|
||||
rawResponses
|
||||
|> Dict.Extra.filterMap
|
||||
(\key value ->
|
||||
value
|
||||
|> Result.map Just
|
||||
|> Result.withDefault Nothing
|
||||
)
|
||||
staticRequestsStatus =
|
||||
allRawResponses
|
||||
|> StaticHttpRequest.cacheRequestResolution ApplicationType.Cli request
|
||||
|
||||
hasPermanentError =
|
||||
usableRawResponses
|
||||
|> StaticHttpRequest.permanentError ApplicationType.Cli request
|
||||
|> isJust
|
||||
case staticRequestsStatus of
|
||||
StaticHttpRequest.HasPermanentError _ ->
|
||||
True
|
||||
|
||||
_ ->
|
||||
False
|
||||
|
||||
hasPermanentHttpError =
|
||||
not (List.isEmpty errors)
|
||||
|
||||
--|> List.any
|
||||
-- (\error ->
|
||||
-- case error of
|
||||
-- FailedStaticHttpRequestError _ ->
|
||||
-- True
|
||||
--
|
||||
-- _ ->
|
||||
-- False
|
||||
-- )
|
||||
( allUrlsKnown, knownUrlsToFetch ) =
|
||||
StaticHttpRequest.resolveUrls
|
||||
ApplicationType.Cli
|
||||
request
|
||||
(rawResponses
|
||||
|> Dict.map (\key value -> value |> Result.withDefault "")
|
||||
|> Dict.union (allRawResponses |> Dict.Extra.filterMap (\_ value -> value))
|
||||
)
|
||||
case staticRequestsStatus of
|
||||
StaticHttpRequest.Incomplete newUrlsToFetch ->
|
||||
( False, newUrlsToFetch )
|
||||
|
||||
_ ->
|
||||
( True, [] )
|
||||
|
||||
fetchedAllKnownUrls =
|
||||
(rawResponses
|
||||
@ -461,24 +353,26 @@ nextStep config siteMetadata mode secrets allRawResponses errors (StaticResponse
|
||||
staticResponses
|
||||
|> Dict.toList
|
||||
|> List.concatMap
|
||||
(\( path, NotFetched request rawResponses ) ->
|
||||
(\( path, NotFetched request _ ) ->
|
||||
let
|
||||
usableRawResponses : Dict String String
|
||||
usableRawResponses =
|
||||
rawResponses
|
||||
|> Dict.Extra.filterMap
|
||||
(\key value ->
|
||||
value
|
||||
|> Result.map Just
|
||||
|> Result.withDefault Nothing
|
||||
)
|
||||
|
||||
maybePermanentError =
|
||||
StaticHttpRequest.permanentError
|
||||
staticRequestsStatus =
|
||||
StaticHttpRequest.cacheRequestResolution
|
||||
ApplicationType.Cli
|
||||
request
|
||||
usableRawResponses
|
||||
|
||||
usableRawResponses : RequestsAndPending
|
||||
usableRawResponses =
|
||||
allRawResponses
|
||||
|
||||
maybePermanentError =
|
||||
case staticRequestsStatus of
|
||||
StaticHttpRequest.HasPermanentError theError ->
|
||||
Just theError
|
||||
|
||||
_ ->
|
||||
Nothing
|
||||
|
||||
decoderErrors =
|
||||
maybePermanentError
|
||||
|> Maybe.map (StaticHttpRequest.toBuildError path)
|
||||
@ -495,7 +389,7 @@ nextStep config siteMetadata mode secrets allRawResponses errors (StaticResponse
|
||||
staticResponses
|
||||
|> Dict.toList
|
||||
|> List.map
|
||||
(\( path, NotFetched request rawResponses ) ->
|
||||
(\( path, NotFetched request _ ) ->
|
||||
( path, request )
|
||||
)
|
||||
in
|
||||
@ -546,7 +440,7 @@ nextStep config siteMetadata mode secrets allRawResponses errors (StaticResponse
|
||||
|
||||
else
|
||||
ToJsPayload.toJsPayload
|
||||
(encode mode (StaticResponses staticResponses))
|
||||
(encode allRawResponses mode (StaticResponses staticResponses))
|
||||
config.manifest
|
||||
generatedOkayFiles
|
||||
allRawResponses
|
||||
@ -561,10 +455,10 @@ performStaticHttpRequests :
|
||||
-> Result (List BuildError) (List { unmasked : RequestDetails, masked : RequestDetails })
|
||||
performStaticHttpRequests allRawResponses secrets staticRequests =
|
||||
staticRequests
|
||||
-- TODO look for performance bottleneck in this double nesting
|
||||
|> List.map
|
||||
(\( pagePath, request ) ->
|
||||
allRawResponses
|
||||
|> dictCompact
|
||||
|> StaticHttpRequest.resolveUrls ApplicationType.Cli request
|
||||
|> Tuple.second
|
||||
)
|
||||
|
@ -86,6 +86,7 @@ import Pages.Internal.StaticHttpBody as Body
|
||||
import Pages.Secrets
|
||||
import Pages.StaticHttp.Request as HashRequest
|
||||
import Pages.StaticHttpRequest exposing (Request(..))
|
||||
import RequestsAndPending exposing (RequestsAndPending)
|
||||
import Secrets
|
||||
|
||||
|
||||
@ -241,7 +242,7 @@ map2 fn request1 request2 =
|
||||
case ( request1, request2 ) of
|
||||
( Request ( urls1, lookupFn1 ), Request ( urls2, lookupFn2 ) ) ->
|
||||
let
|
||||
value : ApplicationType -> Dict String String -> Result Pages.StaticHttpRequest.Error ( Dict String String, Request c )
|
||||
value : ApplicationType -> RequestsAndPending -> Result Pages.StaticHttpRequest.Error ( Dict String String, Request c )
|
||||
value appType rawResponses =
|
||||
let
|
||||
value1 =
|
||||
@ -339,20 +340,26 @@ combineReducedDicts dict1 dict2 =
|
||||
)
|
||||
|
||||
|
||||
lookup : ApplicationType -> Pages.StaticHttpRequest.Request value -> Dict String String -> Result Pages.StaticHttpRequest.Error ( Dict String String, value )
|
||||
lookup appType requestInfo rawResponses =
|
||||
lookup : ApplicationType -> Pages.StaticHttpRequest.Request value -> RequestsAndPending -> Result Pages.StaticHttpRequest.Error ( Dict String String, value )
|
||||
lookup =
|
||||
lookupHelp Dict.empty
|
||||
|
||||
|
||||
lookupHelp : Dict String String -> ApplicationType -> Pages.StaticHttpRequest.Request value -> RequestsAndPending -> Result Pages.StaticHttpRequest.Error ( Dict String String, value )
|
||||
lookupHelp strippedSoFar appType requestInfo rawResponses =
|
||||
case requestInfo of
|
||||
Request ( urls, lookupFn ) ->
|
||||
lookupFn appType rawResponses
|
||||
|> Result.andThen
|
||||
(\( strippedResponses, nextRequest ) ->
|
||||
lookup appType
|
||||
lookupHelp (Dict.union strippedResponses strippedSoFar)
|
||||
appType
|
||||
(addUrls urls nextRequest)
|
||||
strippedResponses
|
||||
rawResponses
|
||||
)
|
||||
|
||||
Done value ->
|
||||
Ok ( rawResponses, value )
|
||||
Ok ( strippedSoFar, value )
|
||||
|
||||
|
||||
addUrls : List (Pages.Secrets.Value HashRequest.Request) -> Pages.StaticHttpRequest.Request value -> Pages.StaticHttpRequest.Request value
|
||||
@ -440,7 +447,7 @@ succeed value =
|
||||
Request
|
||||
( []
|
||||
, \appType rawResponses ->
|
||||
Ok ( rawResponses, Done value )
|
||||
Ok ( Dict.empty, Done value )
|
||||
)
|
||||
|
||||
|
||||
@ -595,12 +602,12 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
case appType of
|
||||
ApplicationType.Cli ->
|
||||
rawResponseDict
|
||||
|> Dict.get (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash)
|
||||
|> RequestsAndPending.get (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash)
|
||||
|> (\maybeResponse ->
|
||||
case maybeResponse of
|
||||
Just rawResponse ->
|
||||
Ok
|
||||
( rawResponseDict
|
||||
( Dict.singleton (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash) rawResponse
|
||||
, rawResponse
|
||||
)
|
||||
|
||||
@ -650,12 +657,12 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
|
||||
ApplicationType.Browser ->
|
||||
rawResponseDict
|
||||
|> Dict.get (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash)
|
||||
|> RequestsAndPending.get (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash)
|
||||
|> (\maybeResponse ->
|
||||
case maybeResponse of
|
||||
Just rawResponse ->
|
||||
Ok
|
||||
( rawResponseDict
|
||||
( Dict.singleton (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash) rawResponse
|
||||
, rawResponse
|
||||
)
|
||||
|
||||
@ -690,13 +697,12 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
( [ requestWithSecrets ]
|
||||
, \appType rawResponseDict ->
|
||||
rawResponseDict
|
||||
|> Dict.get (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash)
|
||||
|> RequestsAndPending.get (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash)
|
||||
|> (\maybeResponse ->
|
||||
case maybeResponse of
|
||||
Just rawResponse ->
|
||||
Ok
|
||||
( rawResponseDict
|
||||
-- |> Dict.update url (\maybeValue -> Just """{"fake": 123}""")
|
||||
( Dict.singleton (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash) rawResponse
|
||||
, rawResponse
|
||||
)
|
||||
|
||||
@ -710,7 +716,6 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
(\( strippedResponses, rawResponse ) ->
|
||||
rawResponse
|
||||
|> Json.Decode.decodeString decoder
|
||||
-- |> Result.mapError Json.Decode.Exploration.errorsToString
|
||||
|> (\decodeResult ->
|
||||
case decodeResult of
|
||||
Err error ->
|
||||
@ -740,13 +745,12 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
( [ requestWithSecrets ]
|
||||
, \appType rawResponseDict ->
|
||||
rawResponseDict
|
||||
|> Dict.get (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash)
|
||||
|> RequestsAndPending.get (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash)
|
||||
|> (\maybeResponse ->
|
||||
case maybeResponse of
|
||||
Just rawResponse ->
|
||||
Ok
|
||||
( rawResponseDict
|
||||
-- |> Dict.update url (\maybeValue -> Just """{"fake": 123}""")
|
||||
( Dict.singleton (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash) rawResponse
|
||||
, rawResponse
|
||||
)
|
||||
|
||||
@ -765,9 +769,7 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
|> Result.map
|
||||
(\finalRequest ->
|
||||
( strippedResponses
|
||||
|> Dict.insert
|
||||
(Secrets.maskedLookup requestWithSecrets |> HashRequest.hash)
|
||||
rawResponse
|
||||
|> Dict.insert (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash) rawResponse
|
||||
, finalRequest
|
||||
)
|
||||
)
|
||||
|
@ -1,31 +1,38 @@
|
||||
module Pages.StaticHttpRequest exposing (Error(..), Request(..), permanentError, resolve, resolveUrls, strippedResponses, toBuildError, urls)
|
||||
module Pages.StaticHttpRequest exposing (Error(..), Request(..), Status(..), cacheRequestResolution, permanentError, resolve, resolveUrls, strippedResponses, toBuildError, urls)
|
||||
|
||||
import BuildError exposing (BuildError)
|
||||
import Dict exposing (Dict)
|
||||
import Pages.Internal.ApplicationType as ApplicationType exposing (ApplicationType)
|
||||
import Pages.Internal.StaticHttpBody as StaticHttpBody
|
||||
import Pages.StaticHttp.Request
|
||||
import RequestsAndPending exposing (RequestsAndPending)
|
||||
import Secrets
|
||||
import TerminalText as Terminal
|
||||
|
||||
|
||||
type Request value
|
||||
= Request ( List (Secrets.Value Pages.StaticHttp.Request.Request), ApplicationType -> Dict String String -> Result Error ( Dict String String, Request value ) )
|
||||
= Request ( List (Secrets.Value Pages.StaticHttp.Request.Request), ApplicationType -> RequestsAndPending -> Result Error ( Dict String String, Request value ) )
|
||||
| Done value
|
||||
|
||||
|
||||
strippedResponses : ApplicationType -> Request value -> Dict String String -> Dict String String
|
||||
strippedResponses appType request rawResponses =
|
||||
strippedResponses : ApplicationType -> Request value -> RequestsAndPending -> Dict String String
|
||||
strippedResponses =
|
||||
strippedResponsesHelp Dict.empty
|
||||
|
||||
|
||||
strippedResponsesHelp : Dict String String -> ApplicationType -> Request value -> RequestsAndPending -> Dict String String
|
||||
strippedResponsesHelp usedSoFar appType request rawResponses =
|
||||
case request of
|
||||
Request ( list, lookupFn ) ->
|
||||
case lookupFn appType rawResponses of
|
||||
Err error ->
|
||||
rawResponses
|
||||
usedSoFar
|
||||
|
||||
Ok ( partiallyStrippedResponses, followupRequest ) ->
|
||||
strippedResponses appType followupRequest partiallyStrippedResponses
|
||||
strippedResponsesHelp (Dict.union usedSoFar partiallyStrippedResponses) appType followupRequest rawResponses
|
||||
|
||||
Done value ->
|
||||
rawResponses
|
||||
usedSoFar
|
||||
|
||||
|
||||
type Error
|
||||
@ -78,7 +85,7 @@ toBuildError path error =
|
||||
}
|
||||
|
||||
|
||||
permanentError : ApplicationType -> Request value -> Dict String String -> Maybe Error
|
||||
permanentError : ApplicationType -> Request value -> RequestsAndPending -> Maybe Error
|
||||
permanentError appType request rawResponses =
|
||||
case request of
|
||||
Request ( urlList, lookupFn ) ->
|
||||
@ -101,7 +108,7 @@ permanentError appType request rawResponses =
|
||||
Nothing
|
||||
|
||||
|
||||
resolve : ApplicationType -> Request value -> Dict String String -> Result Error value
|
||||
resolve : ApplicationType -> Request value -> RequestsAndPending -> Result Error value
|
||||
resolve appType request rawResponses =
|
||||
case request of
|
||||
Request ( urlList, lookupFn ) ->
|
||||
@ -116,19 +123,62 @@ resolve appType request rawResponses =
|
||||
Ok value
|
||||
|
||||
|
||||
resolveUrls : ApplicationType -> Request value -> Dict String String -> ( Bool, List (Secrets.Value Pages.StaticHttp.Request.Request) )
|
||||
resolveUrls : ApplicationType -> Request value -> RequestsAndPending -> ( Bool, List (Secrets.Value Pages.StaticHttp.Request.Request) )
|
||||
resolveUrls appType request rawResponses =
|
||||
case request of
|
||||
Request ( urlList, lookupFn ) ->
|
||||
case lookupFn appType rawResponses of
|
||||
Ok ( partiallyStrippedResponses, nextRequest ) ->
|
||||
Ok ( _, nextRequest ) ->
|
||||
resolveUrls appType nextRequest rawResponses
|
||||
|> Tuple.mapSecond ((++) urlList)
|
||||
|
||||
Err error ->
|
||||
Err _ ->
|
||||
( False
|
||||
, urlList
|
||||
)
|
||||
|
||||
Done value ->
|
||||
Done _ ->
|
||||
( True, [] )
|
||||
|
||||
|
||||
cacheRequestResolution :
|
||||
ApplicationType
|
||||
-> Request value
|
||||
-> RequestsAndPending
|
||||
-> Status value
|
||||
cacheRequestResolution =
|
||||
cacheRequestResolutionHelp []
|
||||
|
||||
|
||||
type Status value
|
||||
= Incomplete (List (Secrets.Value Pages.StaticHttp.Request.Request))
|
||||
| HasPermanentError Error
|
||||
| Complete value -- TODO include stripped responses?
|
||||
|
||||
|
||||
cacheRequestResolutionHelp :
|
||||
List (Secrets.Value Pages.StaticHttp.Request.Request)
|
||||
-> ApplicationType
|
||||
-> Request value
|
||||
-> RequestsAndPending
|
||||
-> Status value
|
||||
cacheRequestResolutionHelp foundUrls appType request rawResponses =
|
||||
case request of
|
||||
Request ( urlList, lookupFn ) ->
|
||||
case lookupFn appType rawResponses of
|
||||
Ok ( partiallyStrippedResponses, nextRequest ) ->
|
||||
cacheRequestResolutionHelp urlList appType nextRequest rawResponses
|
||||
|
||||
Err error ->
|
||||
case error of
|
||||
MissingHttpResponse string ->
|
||||
Incomplete (urlList ++ foundUrls)
|
||||
|
||||
DecoderError string ->
|
||||
HasPermanentError error
|
||||
|
||||
UserCalledStaticHttpFail string ->
|
||||
HasPermanentError error
|
||||
|
||||
Done value ->
|
||||
Complete value
|
||||
|
31
src/RequestsAndPending.elm
Normal file
31
src/RequestsAndPending.elm
Normal file
@ -0,0 +1,31 @@
|
||||
module RequestsAndPending exposing (..)
|
||||
|
||||
import Dict exposing (Dict)
|
||||
import Json.Decode as Decode
|
||||
import List.Extra as Dict
|
||||
|
||||
|
||||
type alias RequestsAndPending =
|
||||
Dict String (Maybe String)
|
||||
|
||||
|
||||
init : RequestsAndPending
|
||||
init =
|
||||
Dict.empty
|
||||
|
||||
|
||||
get : String -> RequestsAndPending -> Maybe String
|
||||
get key requestsAndPending =
|
||||
requestsAndPending
|
||||
|> Dict.get key
|
||||
|> Maybe.andThen identity
|
||||
|
||||
|
||||
insert : String -> String -> RequestsAndPending -> RequestsAndPending
|
||||
insert key value requestsAndPending =
|
||||
Dict.insert key (Just value) requestsAndPending
|
||||
|
||||
|
||||
decoder : Decode.Decoder RequestsAndPending
|
||||
decoder =
|
||||
Decode.dict (Decode.string |> Decode.map Just)
|
@ -557,7 +557,13 @@ I got an error making an HTTP request to this URL: https://api.github.com/repos/
|
||||
|
||||
Bad status: 404
|
||||
Status message: TODO: if you need this, please report to https://github.com/avh4/elm-program-test/issues
|
||||
Body: """)
|
||||
Body:
|
||||
|
||||
-- STATIC HTTP DECODING ERROR ----------------------------------------------------- elm-pages
|
||||
|
||||
|
||||
|
||||
Payload sent back invalid JSON""")
|
||||
, test "uses real secrets to perform request and masked secrets to store and lookup response" <|
|
||||
\() ->
|
||||
start
|
||||
@ -780,7 +786,8 @@ startWithHttpCache =
|
||||
startLowLevel :
|
||||
StaticHttp.Request
|
||||
(List
|
||||
(Result String
|
||||
(Result
|
||||
String
|
||||
{ path : List String
|
||||
, content : String
|
||||
}
|
||||
@ -954,6 +961,9 @@ expectErrorsPort expectedPlainString actualPorts =
|
||||
actualRichTerminalString
|
||||
|> normalizeErrorExpectEqual expectedPlainString
|
||||
|
||||
[] ->
|
||||
Expect.fail "Expected single error port. Didn't receive any ports."
|
||||
|
||||
_ ->
|
||||
Expect.fail <| "Expected single error port. Got\n" ++ String.join "\n\n" (List.map Debug.toString actualPorts)
|
||||
|
||||
|
@ -21,7 +21,7 @@ requestsDict requestMap =
|
||||
|> List.map
|
||||
(\( request, response ) ->
|
||||
( request |> Request.hash
|
||||
, response
|
||||
, Just response
|
||||
)
|
||||
)
|
||||
|> Dict.fromList
|
||||
|
Loading…
Reference in New Issue
Block a user