mirror of
https://github.com/dillonkearns/elm-pages-v3-beta.git
synced 2024-11-27 21:29:55 +03:00
Handle merging stripped JSON data from multiple data sources.
This commit is contained in:
Dillon Kearns
parent
cf92e7d268
commit
8b4144ef15
@ -67,7 +67,7 @@ import Pages.Internal.ApplicationType exposing (ApplicationType)
|
||||
import Pages.Internal.StaticHttpBody as Body
|
||||
import Pages.Secrets
|
||||
import Pages.StaticHttp.Request as HashRequest
|
||||
import Pages.StaticHttpRequest exposing (RawRequest(..))
|
||||
import Pages.StaticHttpRequest exposing (RawRequest(..), WhatToDo)
|
||||
import RequestsAndPending exposing (RequestsAndPending)
|
||||
|
||||
|
||||
@ -226,7 +226,7 @@ map2 fn request1 request2 =
|
||||
case ( request1, request2 ) of
|
||||
( Request ( urls1, lookupFn1 ), Request ( urls2, lookupFn2 ) ) ->
|
||||
let
|
||||
value : ApplicationType -> RequestsAndPending -> Result Pages.StaticHttpRequest.Error ( Dict String String, DataSource c )
|
||||
value : ApplicationType -> RequestsAndPending -> Result Pages.StaticHttpRequest.Error ( Dict String WhatToDo, DataSource c )
|
||||
value appType rawResponses =
|
||||
case ( lookupFn1 appType rawResponses, lookupFn2 appType rawResponses ) of
|
||||
( Ok ( newDict1, newValue1 ), Ok ( newDict2, newValue2 ) ) ->
|
||||
@ -273,25 +273,18 @@ map2 fn request1 request2 =
|
||||
This is assuming that there are no duplicate URLs, so it can safely choose between either a raw or a reduced response.
|
||||
It would not work correctly if it chose between two responses that were reduced with different `Json.Decode.Exploration.Decoder`s.
|
||||
-}
|
||||
combineReducedDicts : Dict String String -> Dict String String -> Dict String String
|
||||
combineReducedDicts : Dict String WhatToDo -> Dict String WhatToDo -> Dict String WhatToDo
|
||||
combineReducedDicts dict1 dict2 =
|
||||
(Dict.toList dict1 ++ Dict.toList dict2)
|
||||
|> Dict.Extra.fromListDedupe
|
||||
(\response1 response2 ->
|
||||
if String.length response1 < String.length response2 then
|
||||
response1
|
||||
|
||||
else
|
||||
response2
|
||||
)
|
||||
|> Dict.Extra.fromListDedupe Pages.StaticHttpRequest.merge
|
||||
|
||||
|
||||
lookup : ApplicationType -> DataSource value -> RequestsAndPending -> Result Pages.StaticHttpRequest.Error ( Dict String String, value )
|
||||
lookup : ApplicationType -> DataSource value -> RequestsAndPending -> Result Pages.StaticHttpRequest.Error ( Dict String WhatToDo, value )
|
||||
lookup =
|
||||
lookupHelp Dict.empty
|
||||
|
||||
|
||||
lookupHelp : Dict String String -> ApplicationType -> DataSource value -> RequestsAndPending -> Result Pages.StaticHttpRequest.Error ( Dict String String, value )
|
||||
lookupHelp : Dict String WhatToDo -> ApplicationType -> DataSource value -> RequestsAndPending -> Result Pages.StaticHttpRequest.Error ( Dict String WhatToDo, value )
|
||||
lookupHelp strippedSoFar appType requestInfo rawResponses =
|
||||
case requestInfo of
|
||||
Request ( urls, lookupFn ) ->
|
||||
|
||||
@ -255,7 +255,10 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
case maybeResponse of
|
||||
Just rawResponse ->
|
||||
Ok
|
||||
( Dict.singleton (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash) rawResponse
|
||||
( Dict.singleton (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash)
|
||||
(Pages.StaticHttpRequest.StripResponse
|
||||
(Decode.map (\_ -> ()) decoder)
|
||||
)
|
||||
, rawResponse
|
||||
)
|
||||
|
||||
@ -267,17 +270,12 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
)
|
||||
|> Result.andThen
|
||||
(\( strippedResponses, rawResponse ) ->
|
||||
let
|
||||
reduced =
|
||||
Json.Decode.Exploration.stripString (Internal.OptimizedDecoder.jde decoder) rawResponse
|
||||
|> Result.withDefault "TODO"
|
||||
in
|
||||
rawResponse
|
||||
|> Json.Decode.Exploration.decodeString (decoder |> Internal.OptimizedDecoder.jde)
|
||||
|> (\decodeResult ->
|
||||
case decodeResult of
|
||||
Json.Decode.Exploration.BadJson ->
|
||||
Pages.StaticHttpRequest.DecoderError "Payload sent back invalid JSON" |> Err
|
||||
Pages.StaticHttpRequest.DecoderError ("Payload sent back invalid JSON\n" ++ rawResponse) |> Err
|
||||
|
||||
Json.Decode.Exploration.Errors errors ->
|
||||
errors
|
||||
@ -297,7 +295,9 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
( strippedResponses
|
||||
|> Dict.insert
|
||||
(Secrets.maskedLookup requestWithSecrets |> HashRequest.hash)
|
||||
reduced
|
||||
(Pages.StaticHttpRequest.StripResponse
|
||||
(Decode.map (\_ -> ()) decoder)
|
||||
)
|
||||
, finalRequest
|
||||
)
|
||||
)
|
||||
@ -310,7 +310,8 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
case maybeResponse of
|
||||
Just rawResponse ->
|
||||
Ok
|
||||
( Dict.singleton (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash) rawResponse
|
||||
( Dict.singleton (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash)
|
||||
Pages.StaticHttpRequest.UseRawResponse
|
||||
, rawResponse
|
||||
)
|
||||
|
||||
@ -326,8 +327,15 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
|> Json.Decode.decodeString (decoder |> Internal.OptimizedDecoder.jd)
|
||||
|> (\decodeResult ->
|
||||
case decodeResult of
|
||||
Err _ ->
|
||||
Pages.StaticHttpRequest.DecoderError "Payload sent back invalid JSON" |> Err
|
||||
Err error ->
|
||||
Pages.StaticHttpRequest.DecoderError
|
||||
("Payload sent back invalid JSON\n"
|
||||
++ rawResponse
|
||||
++ "\n KEYS"
|
||||
++ (Dict.keys strippedResponses |> String.join " - ")
|
||||
++ Json.Decode.errorToString error
|
||||
)
|
||||
|> Err
|
||||
|
||||
Ok a ->
|
||||
Ok a
|
||||
@ -335,7 +343,9 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
|> Result.map Done
|
||||
|> Result.map
|
||||
(\finalRequest ->
|
||||
( strippedResponses, finalRequest )
|
||||
( strippedResponses
|
||||
, finalRequest
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
@ -350,7 +360,8 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
case maybeResponse of
|
||||
Just rawResponse ->
|
||||
Ok
|
||||
( Dict.singleton (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash) rawResponse
|
||||
( Dict.singleton (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash)
|
||||
Pages.StaticHttpRequest.UseRawResponse
|
||||
, rawResponse
|
||||
)
|
||||
|
||||
@ -381,7 +392,7 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
( strippedResponses
|
||||
|> Dict.insert
|
||||
(Secrets.maskedLookup requestWithSecrets |> HashRequest.hash)
|
||||
rawResponse
|
||||
Pages.StaticHttpRequest.UseRawResponse
|
||||
, finalRequest
|
||||
)
|
||||
)
|
||||
@ -398,7 +409,7 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
case maybeResponse of
|
||||
Just rawResponse ->
|
||||
Ok
|
||||
( Dict.singleton (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash) rawResponse
|
||||
( Dict.singleton (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash) Pages.StaticHttpRequest.UseRawResponse
|
||||
, rawResponse
|
||||
)
|
||||
|
||||
@ -417,7 +428,7 @@ unoptimizedRequest requestWithSecrets expect =
|
||||
|> Result.map
|
||||
(\finalRequest ->
|
||||
( strippedResponses
|
||||
|> Dict.insert (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash) rawResponse
|
||||
|> Dict.insert (Secrets.maskedLookup requestWithSecrets |> HashRequest.hash) Pages.StaticHttpRequest.UseRawResponse
|
||||
, finalRequest
|
||||
)
|
||||
)
|
||||
|
||||
@ -176,13 +176,13 @@ encode requestsAndPending mode staticResponses =
|
||||
NotFetched request _ ->
|
||||
case mode of
|
||||
Mode.Dev ->
|
||||
StaticHttpRequest.strippedResponses ApplicationType.Cli request requestsAndPending
|
||||
StaticHttpRequest.strippedResponsesEncode ApplicationType.Cli request requestsAndPending
|
||||
|
||||
Mode.Prod ->
|
||||
StaticHttpRequest.strippedResponses ApplicationType.Cli request requestsAndPending
|
||||
StaticHttpRequest.strippedResponsesEncode ApplicationType.Cli request requestsAndPending
|
||||
|
||||
Mode.ElmToHtmlBeta ->
|
||||
StaticHttpRequest.strippedResponses ApplicationType.Cli request requestsAndPending
|
||||
StaticHttpRequest.strippedResponsesEncode ApplicationType.Cli request requestsAndPending
|
||||
)
|
||||
|
||||
|
||||
|
||||
@ -1,7 +1,11 @@
|
||||
module Pages.StaticHttpRequest exposing (Error(..), RawRequest(..), Status(..), cacheRequestResolution, resolve, resolveUrls, strippedResponses, toBuildError)
|
||||
module Pages.StaticHttpRequest exposing (Error(..), RawRequest(..), Status(..), WhatToDo(..), cacheRequestResolution, merge, resolve, resolveUrls, strippedResponses, strippedResponsesEncode, toBuildError)
|
||||
|
||||
import BuildError exposing (BuildError)
|
||||
import Dict exposing (Dict)
|
||||
import Dict.Extra
|
||||
import Internal.OptimizedDecoder
|
||||
import Json.Decode.Exploration
|
||||
import OptimizedDecoder
|
||||
import Pages.Internal.ApplicationType exposing (ApplicationType)
|
||||
import Pages.StaticHttp.Request
|
||||
import RequestsAndPending exposing (RequestsAndPending)
|
||||
@ -12,17 +16,58 @@ import TerminalText as Terminal
|
||||
type RawRequest value
|
||||
= Request
|
||||
( List (Secrets.Value Pages.StaticHttp.Request.Request)
|
||||
, ApplicationType -> RequestsAndPending -> Result Error ( Dict String String, RawRequest value )
|
||||
, ApplicationType -> RequestsAndPending -> Result Error ( Dict String WhatToDo, RawRequest value )
|
||||
)
|
||||
| Done value
|
||||
|
||||
|
||||
strippedResponses : ApplicationType -> RawRequest value -> RequestsAndPending -> Dict String String
|
||||
type WhatToDo
|
||||
= UseRawResponse
|
||||
| StripResponse (OptimizedDecoder.Decoder ())
|
||||
|
||||
|
||||
merge : WhatToDo -> WhatToDo -> WhatToDo
|
||||
merge whatToDo1 whatToDo2 =
|
||||
case ( whatToDo1, whatToDo2 ) of
|
||||
( StripResponse strip1, StripResponse strip2 ) ->
|
||||
StripResponse (OptimizedDecoder.map2 (\_ _ -> ()) strip1 strip2)
|
||||
|
||||
( StripResponse strip1, _ ) ->
|
||||
StripResponse strip1
|
||||
|
||||
( _, StripResponse strip1 ) ->
|
||||
StripResponse strip1
|
||||
|
||||
_ ->
|
||||
UseRawResponse
|
||||
|
||||
|
||||
strippedResponses : ApplicationType -> RawRequest value -> RequestsAndPending -> Dict String WhatToDo
|
||||
strippedResponses =
|
||||
strippedResponsesHelp Dict.empty
|
||||
|
||||
|
||||
strippedResponsesHelp : Dict String String -> ApplicationType -> RawRequest value -> RequestsAndPending -> Dict String String
|
||||
strippedResponsesEncode : ApplicationType -> RawRequest value -> RequestsAndPending -> Dict String String
|
||||
strippedResponsesEncode appType rawRequest requestsAndPending =
|
||||
strippedResponses appType rawRequest requestsAndPending
|
||||
|> Dict.map
|
||||
(\k whatToDo ->
|
||||
case whatToDo of
|
||||
UseRawResponse ->
|
||||
Dict.get k requestsAndPending
|
||||
|> Maybe.withDefault Nothing
|
||||
|> Maybe.withDefault ""
|
||||
|
||||
StripResponse decoder ->
|
||||
Dict.get k requestsAndPending
|
||||
|> Maybe.withDefault Nothing
|
||||
|> Maybe.withDefault ""
|
||||
|> Json.Decode.Exploration.stripString (Internal.OptimizedDecoder.jde decoder)
|
||||
|> Result.withDefault "ERROR"
|
||||
)
|
||||
|
||||
|
||||
strippedResponsesHelp : Dict String WhatToDo -> ApplicationType -> RawRequest value -> RequestsAndPending -> Dict String WhatToDo
|
||||
strippedResponsesHelp usedSoFar appType request rawResponses =
|
||||
case request of
|
||||
Request ( _, lookupFn ) ->
|
||||
@ -31,7 +76,15 @@ strippedResponsesHelp usedSoFar appType request rawResponses =
|
||||
usedSoFar
|
||||
|
||||
Ok ( partiallyStrippedResponses, followupRequest ) ->
|
||||
strippedResponsesHelp (Dict.union usedSoFar partiallyStrippedResponses) appType followupRequest rawResponses
|
||||
strippedResponsesHelp
|
||||
(((usedSoFar |> Dict.toList)
|
||||
++ (partiallyStrippedResponses |> Dict.toList)
|
||||
)
|
||||
|> Dict.Extra.fromListDedupe merge
|
||||
)
|
||||
appType
|
||||
followupRequest
|
||||
rawResponses
|
||||
|
||||
Done _ ->
|
||||
usedSoFar
|
||||
|
||||
@ -78,6 +78,31 @@ all =
|
||||
]
|
||||
)
|
||||
]
|
||||
, test "the stripped JSON from the same request with different decoders is merged so the decoders succeed" <|
|
||||
\() ->
|
||||
start
|
||||
[ ( [ "post-1" ]
|
||||
, DataSource.map2 Tuple.pair
|
||||
(DataSource.Http.get (Secrets.succeed "https://api.github.com/repos/dillonkearns/elm-pages")
|
||||
(Decode.field "stargazer_count" Decode.int)
|
||||
)
|
||||
(DataSource.Http.get (Secrets.succeed "https://api.github.com/repos/dillonkearns/elm-pages")
|
||||
(Decode.field "language" Decode.string)
|
||||
)
|
||||
)
|
||||
]
|
||||
|> ProgramTest.simulateHttpOk
|
||||
"GET"
|
||||
"https://api.github.com/repos/dillonkearns/elm-pages"
|
||||
"""{ "stargazer_count": 86, "language": "Elm" }"""
|
||||
|> expectSuccess
|
||||
[ ( "post-1"
|
||||
, [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
|
||||
, """{"stargazer_count":86,"language":"Elm"}"""
|
||||
)
|
||||
]
|
||||
)
|
||||
]
|
||||
, test "andThen" <|
|
||||
\() ->
|
||||
start
|
||||
|
||||
Loading…
Reference in New Issue
Block a user