Change url hashing function to use JSON encoding.

This commit is contained in:
Dillon Kearns 2020-01-03 12:18:18 -08:00
parent 919e69a0ee
commit f2e8c70883
5 changed files with 274 additions and 279 deletions

View File

@ -28,6 +28,7 @@ import Pages.ImagePath as ImagePath
import Pages.Manifest as Manifest
import Pages.PagePath as PagePath exposing (PagePath)
import Pages.StaticHttp as StaticHttp exposing (RequestDetails)
import Pages.StaticHttp.Request as HashRequest
import Pages.StaticHttpRequest as StaticHttpRequest
import Secrets
import SecretsDict exposing (SecretsDict)
@ -532,22 +533,13 @@ staticResponsesInit list =
|> Dict.fromList
hashUrl : RequestDetails -> String
hashUrl requestDetails =
"["
++ requestDetails.method
++ "]"
++ requestDetails.url
++ String.join "," (requestDetails.headers |> List.map (\( key, value ) -> key ++ " : " ++ value))
staticResponsesUpdate : { request : { masked : RequestDetails, unmasked : RequestDetails }, response : Result () String } -> Model -> Model
staticResponsesUpdate newEntry model =
let
updatedAllResponses =
model.allRawResponses
-- @@@@@@@@@ TODO handle errors here, change Dict to have `Result` instead of `Maybe`
|> Dict.insert (hashUrl newEntry.request.masked) (Just (newEntry.response |> Result.withDefault "TODO"))
|> Dict.insert (HashRequest.hash newEntry.request.masked) (Just (newEntry.response |> Result.withDefault "TODO"))
in
{ model
| allRawResponses = updatedAllResponses
@ -563,17 +555,17 @@ staticResponsesUpdate newEntry model =
(updatedAllResponses |> dictCompact)
|> Tuple.second
|> List.map Secrets.maskedLookup
|> List.map hashUrl
|> List.map HashRequest.hash
includesUrl =
List.member (hashUrl newEntry.request.masked)
List.member (HashRequest.hash newEntry.request.masked)
realUrls
in
if includesUrl then
let
updatedRawResponses =
rawResponses
|> Dict.insert (hashUrl newEntry.request.masked) newEntry.response
|> Dict.insert (HashRequest.hash newEntry.request.masked) newEntry.response
in
NotFetched request updatedRawResponses
@ -636,7 +628,7 @@ sendStaticResponsesIfDone mode secrets allRawResponses errors staticResponses ma
fetchedAllKnownUrls =
(knownUrlsToFetch
|> List.map Secrets.maskedLookup
|> List.map hashUrl
|> List.map HashRequest.hash
|> Set.fromList
|> Set.size
)
@ -701,7 +693,7 @@ sendStaticResponsesIfDone mode secrets allRawResponses errors staticResponses ma
dictOfNewUrlsToPerform =
urlsToPerform
|> List.map .masked
|> List.map hashUrl
|> List.map HashRequest.hash
|> List.map (\hashedUrl -> ( hashedUrl, Nothing ))
|> Dict.fromList
@ -712,7 +704,7 @@ sendStaticResponsesIfDone mode secrets allRawResponses errors staticResponses ma
|> List.map
(\secureUrl ->
-- ( hashUrl secureUrl, { unmasked = secureUrl, masked = secureUrl } )
( hashUrl secureUrl.masked, secureUrl )
( HashRequest.hash secureUrl.masked, secureUrl )
)
|> Dict.fromList

View File

@ -51,7 +51,9 @@ in [this article introducing StaticHttp requests and some concepts around it](ht
import Dict exposing (Dict)
import Dict.Extra
import Json.Decode.Exploration as Decode exposing (Decoder)
import Json.Encode as Encode
import Pages.Secrets
import Pages.StaticHttp.Request as HashRequest
import Pages.StaticHttpRequest exposing (Request(..))
import Secrets
@ -406,16 +408,15 @@ get url decoder =
{-| The full details to perform a StaticHttp request.
-}
type alias RequestDetails =
{ url : String, method : String, headers : List ( String, String ) }
HashRequest.Request
hashRequest : RequestDetails -> String
hashRequest requestDetails =
"["
++ requestDetails.method
++ "]"
++ requestDetails.url
++ String.join "," (requestDetails.headers |> List.map (\( key, value ) -> key ++ " : " ++ value))
--"["
-- ++ requestDetails.method
-- ++ "]"
-- ++ requestDetails.url
-- ++ String.join "," (requestDetails.headers |> List.map (\( key, value ) -> key ++ " : " ++ value))
requestToString : RequestDetails -> String
@ -436,7 +437,7 @@ request urlWithSecrets decoder =
( [ urlWithSecrets ]
, \rawResponseDict ->
rawResponseDict
|> Dict.get (Secrets.maskedLookup urlWithSecrets |> hashRequest)
|> Dict.get (Secrets.maskedLookup urlWithSecrets |> HashRequest.hash)
|> (\maybeResponse ->
case maybeResponse of
Just rawResponse ->
@ -486,7 +487,7 @@ request urlWithSecrets decoder =
(\finalRequest ->
( strippedResponses
|> Dict.insert
(Secrets.maskedLookup urlWithSecrets |> hashRequest)
(Secrets.maskedLookup urlWithSecrets |> HashRequest.hash)
reduced
, finalRequest
)

View File

@ -0,0 +1,25 @@
module Pages.StaticHttp.Request exposing (Request, hash)
import Json.Encode as Encode
type alias Request =
{ url : String
, method : String
, headers : List ( String, String )
}
hash : Request -> String
hash requestDetails =
Encode.object
[ ( "method", Encode.string requestDetails.method )
, ( "url", Encode.string requestDetails.url )
, ( "headers", Encode.list hashHeader requestDetails.headers )
]
|> Encode.encode 0
hashHeader : ( String, String ) -> Encode.Value
hashHeader ( name, value ) =
Encode.string <| name ++ ": " ++ value

View File

@ -13,6 +13,7 @@ import Pages.Internal.Platform.Cli as Main exposing (..)
import Pages.Manifest as Manifest
import Pages.PagePath as PagePath
import Pages.StaticHttp as StaticHttp
import Pages.StaticHttp.Request as Request
import ProgramTest exposing (ProgramTest)
import Regex
import Secrets
@ -38,25 +39,17 @@ all =
"GET"
"https://api.github.com/repos/dillonkearns/elm-pages"
"""{ "stargazer_count": 86 }"""
|> ProgramTest.expectOutgoingPortValues
"toJsPort"
(Codec.decoder Main.toJsCodec)
(Expect.equal
[ Main.Success
{ pages =
Dict.fromList
[ ( "/"
, Dict.fromList
[ ( "[GET]https://api.github.com/repos/dillonkearns/elm-pages"
, """{"stargazer_count":86}"""
)
]
)
]
, manifest = manifest
|> expectSuccess
[ ( "/"
, [ ( { method = "GET"
, url = "https://api.github.com/repos/dillonkearns/elm-pages"
, headers = []
}
, """{"stargazer_count":86}"""
)
]
)
)
]
, test "andThen" <|
\() ->
start
@ -76,32 +69,27 @@ all =
"GET"
"NEXT-REQUEST"
"""null"""
|> ProgramTest.expectOutgoingPortValues
"toJsPort"
(Codec.decoder Main.toJsCodec)
(Expect.equal
[ Main.Success
{ pages =
Dict.fromList
[ ( "/elm-pages"
, Dict.fromList
[ ( "[GET]https://api.github.com/repos/dillonkearns/elm-pages"
, """null"""
)
, ( "[GET]NEXT-REQUEST"
, """null"""
)
]
)
]
, manifest = manifest
|> expectSuccess
[ ( "/elm-pages"
, [ ( { method = "GET"
, url = "https://api.github.com/repos/dillonkearns/elm-pages"
, headers = []
}
, """null"""
)
, ( { method = "GET"
, url = "NEXT-REQUEST"
, headers = []
}
, """null"""
)
]
)
)
]
, test "andThen chain avoids repeat requests" <|
\() ->
let
get url decoder =
getReq url decoder =
StaticHttp.request
(Secrets.succeed
{ url = url
@ -113,13 +101,13 @@ all =
pokemonDetailRequest : StaticHttp.Request ()
pokemonDetailRequest =
get
getReq
"https://pokeapi.co/api/v2/pokemon/"
(Decode.list
(Decode.field "url" Decode.string
|> Decode.map
(\url ->
get url
getReq url
(Decode.field "image" Decode.string)
)
)
@ -185,55 +173,44 @@ all =
"GET"
"url10"
"""{"image": "image10.jpg"}"""
|> ProgramTest.expectOutgoingPortValues
"toJsPort"
(Codec.decoder Main.toJsCodec)
(Expect.equal
[ Main.Success
{ pages =
Dict.fromList
[ ( "/elm-pages"
, Dict.fromList
[ ( "[GET]https://pokeapi.co/api/v2/pokemon/"
, """[{"url":"url1"},{"url":"url2"},{"url":"url3"},{"url":"url4"},{"url":"url5"},{"url":"url6"},{"url":"url7"},{"url":"url8"},{"url":"url9"},{"url":"url10"}]"""
)
, ( "[GET]url1"
, """{"image":"image1.jpg"}"""
)
, ( "[GET]url2"
, """{"image":"image2.jpg"}"""
)
, ( "[GET]url3"
, """{"image":"image3.jpg"}"""
)
, ( "[GET]url4"
, """{"image":"image4.jpg"}"""
)
, ( "[GET]url5"
, """{"image":"image5.jpg"}"""
)
, ( "[GET]url6"
, """{"image":"image6.jpg"}"""
)
, ( "[GET]url7"
, """{"image":"image7.jpg"}"""
)
, ( "[GET]url8"
, """{"image":"image8.jpg"}"""
)
, ( "[GET]url9"
, """{"image":"image9.jpg"}"""
)
, ( "[GET]url10"
, """{"image":"image10.jpg"}"""
)
]
)
]
, manifest = manifest
}
|> expectSuccess
[ ( "/elm-pages"
, [ ( get "https://pokeapi.co/api/v2/pokemon/"
, """[{"url":"url1"},{"url":"url2"},{"url":"url3"},{"url":"url4"},{"url":"url5"},{"url":"url6"},{"url":"url7"},{"url":"url8"},{"url":"url9"},{"url":"url10"}]"""
)
, ( get "url1"
, """{"image":"image1.jpg"}"""
)
, ( get "url2"
, """{"image":"image2.jpg"}"""
)
, ( get "url3"
, """{"image":"image3.jpg"}"""
)
, ( get "url4"
, """{"image":"image4.jpg"}"""
)
, ( get "url5"
, """{"image":"image5.jpg"}"""
)
, ( get "url6"
, """{"image":"image6.jpg"}"""
)
, ( get "url7"
, """{"image":"image7.jpg"}"""
)
, ( get "url8"
, """{"image":"image8.jpg"}"""
)
, ( get "url9"
, """{"image":"image9.jpg"}"""
)
, ( get "url10"
, """{"image":"image10.jpg"}"""
)
]
)
)
]
, test "port is sent out once all requests are finished" <|
\() ->
start
@ -252,32 +229,20 @@ all =
"GET"
"https://api.github.com/repos/dillonkearns/elm-pages-starter"
"""{ "stargazer_count": 22 }"""
|> ProgramTest.expectOutgoingPortValues
"toJsPort"
(Codec.decoder Main.toJsCodec)
(Expect.equal
[ Main.Success
{ pages =
Dict.fromList
[ ( "/elm-pages"
, Dict.fromList
[ ( "[GET]https://api.github.com/repos/dillonkearns/elm-pages"
, """{"stargazer_count":86}"""
)
]
)
, ( "/elm-pages-starter"
, Dict.fromList
[ ( "[GET]https://api.github.com/repos/dillonkearns/elm-pages-starter"
, """{"stargazer_count":22}"""
)
]
)
]
, manifest = manifest
}
|> expectSuccess
[ ( "/elm-pages"
, [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
, """{"stargazer_count":86}"""
)
]
)
)
, ( "/elm-pages-starter"
, [ ( get "https://api.github.com/repos/dillonkearns/elm-pages-starter"
, """{"stargazer_count":22}"""
)
]
)
]
, test "reduced JSON is sent out" <|
\() ->
start
@ -289,25 +254,14 @@ all =
"GET"
"https://api.github.com/repos/dillonkearns/elm-pages"
"""{ "stargazer_count": 86, "unused_field": 123 }"""
|> ProgramTest.expectOutgoingPortValues
"toJsPort"
(Codec.decoder Main.toJsCodec)
(Expect.equal
[ Main.Success
{ pages =
Dict.fromList
[ ( "/"
, Dict.fromList
[ ( "[GET]https://api.github.com/repos/dillonkearns/elm-pages"
, """{"stargazer_count":86}"""
)
]
)
]
, manifest = manifest
}
|> expectSuccess
[ ( "/"
, [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
, """{"stargazer_count":86}"""
)
]
)
)
]
, test "POST method works" <|
\() ->
start
@ -326,25 +280,17 @@ all =
"POST"
"https://api.github.com/repos/dillonkearns/elm-pages"
"""{ "stargazer_count": 86, "unused_field": 123 }"""
|> ProgramTest.expectOutgoingPortValues
"toJsPort"
(Codec.decoder Main.toJsCodec)
(Expect.equal
[ Main.Success
{ pages =
Dict.fromList
[ ( "/"
, Dict.fromList
[ ( "[POST]https://api.github.com/repos/dillonkearns/elm-pages"
, """{"stargazer_count":86}"""
)
]
)
]
, manifest = manifest
|> expectSuccess
[ ( "/"
, [ ( { method = "POST"
, url = "https://api.github.com/repos/dillonkearns/elm-pages"
, headers = []
}
, """{"stargazer_count":86}"""
)
]
)
)
]
, test "json is reduced from andThen chains" <|
\() ->
start
@ -364,28 +310,17 @@ all =
"GET"
"https://api.github.com/repos/dillonkearns/elm-pages-starter"
"""{ "stargazer_count": 50, "unused_field": 456 }"""
|> ProgramTest.expectOutgoingPortValues
"toJsPort"
(Codec.decoder Main.toJsCodec)
(Expect.equal
[ Main.Success
{ pages =
Dict.fromList
[ ( "/"
, Dict.fromList
[ ( "[GET]https://api.github.com/repos/dillonkearns/elm-pages"
, """{"stargazer_count":100}"""
)
, ( "[GET]https://api.github.com/repos/dillonkearns/elm-pages-starter"
, """{"stargazer_count":50}"""
)
]
)
]
, manifest = manifest
}
|> expectSuccess
[ ( "/"
, [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
, """{"stargazer_count":100}"""
)
, ( get "https://api.github.com/repos/dillonkearns/elm-pages-starter"
, """{"stargazer_count":50}"""
)
]
)
)
]
, test "reduced json is preserved by StaticHttp.map2" <|
\() ->
start
@ -403,28 +338,17 @@ all =
"GET"
"https://api.github.com/repos/dillonkearns/elm-pages-starter"
"""{ "stargazer_count": 50, "unused_field": 456 }"""
|> ProgramTest.expectOutgoingPortValues
"toJsPort"
(Codec.decoder Main.toJsCodec)
(Expect.equal
[ Main.Success
{ pages =
Dict.fromList
[ ( "/"
, Dict.fromList
[ ( "[GET]https://api.github.com/repos/dillonkearns/elm-pages"
, """{"stargazer_count":100}"""
)
, ( "[GET]https://api.github.com/repos/dillonkearns/elm-pages-starter"
, """{"stargazer_count":50}"""
)
]
)
]
, manifest = manifest
}
|> expectSuccess
[ ( "/"
, [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
, """{"stargazer_count":100}"""
)
, ( get "https://api.github.com/repos/dillonkearns/elm-pages-starter"
, """{"stargazer_count":50}"""
)
]
)
)
]
, test "the port sends out even if there are no http requests" <|
\() ->
start
@ -432,21 +356,7 @@ all =
, StaticHttp.succeed ()
)
]
|> ProgramTest.expectOutgoingPortValues
"toJsPort"
(Codec.decoder Main.toJsCodec)
(Expect.equal
[ Main.Success
{ pages =
Dict.fromList
[ ( "/"
, Dict.fromList []
)
]
, manifest = manifest
}
]
)
|> expectSuccess [ ( "/", [] ) ]
, test "the port sends out when there are duplicate http requests for the same page" <|
\() ->
start
@ -460,21 +370,14 @@ all =
"GET"
"http://example.com"
"""null"""
|> ProgramTest.expectOutgoingPortValues
"toJsPort"
(Codec.decoder Main.toJsCodec)
(Expect.equal
[ Main.Success
{ pages =
Dict.fromList
[ ( "/"
, Dict.fromList [ ( "[GET]http://example.com", "null" ) ]
)
]
, manifest = manifest
}
|> expectSuccess
[ ( "/"
, [ ( get "http://example.com"
, """null"""
)
]
)
)
]
, test "an error is sent out for decoder failures" <|
\() ->
start
@ -601,25 +504,19 @@ Bad status: 404""")
, body = """{ "stargazer_count": 86 }"""
}
)
|> ProgramTest.expectOutgoingPortValues
"toJsPort"
(Codec.decoder Main.toJsCodec)
(Expect.equal
[ Main.Success
{ pages =
Dict.fromList
[ ( "/"
, Dict.fromList
[ ( "[GET]https://api.github.com/repos/dillonkearns/elm-pages?apiKey=<API_KEY>Authorization : Bearer <BEARER>"
, """{}"""
)
]
)
]
, manifest = manifest
|> expectSuccess
[ ( "/"
, [ ( { method = "GET"
, url = "https://api.github.com/repos/dillonkearns/elm-pages?apiKey=<API_KEY>"
, headers =
[ ( "Authorization", "Bearer <BEARER>" )
]
}
, """{}"""
)
]
)
)
]
]
@ -777,3 +674,60 @@ manifest =
starDecoder =
Decode.field "stargazer_count" Decode.int
thingy =
[ ( "/"
, [ ( { method = "GET"
, url = "https://api.github.com/repos/dillonkearns/elm-pages"
, headers = []
}
, """{"stargazer_count":86}"""
)
]
)
]
--type alias Request =
-- { method : String
-- , url : String
-- , headers : List String
-- }
expectSuccess : List ( String, List ( Request.Request, String ) ) -> ProgramTest model msg effect -> Expect.Expectation
expectSuccess expectedRequests previous =
previous
|> ProgramTest.expectOutgoingPortValues
"toJsPort"
(Codec.decoder Main.toJsCodec)
(Expect.equal
[ Main.Success
{ pages =
expectedRequests
|> List.map
(\( url, requests ) ->
( url
, requests
|> List.map
(\( request, response ) ->
( Request.hash request, response )
)
|> Dict.fromList
)
)
|> Dict.fromList
, manifest = manifest
}
]
)
get : String -> Request.Request
get url =
{ method = "GET"
, url = url
, headers = []
}

View File

@ -4,6 +4,7 @@ import Dict exposing (Dict)
import Expect
import Json.Decode.Exploration as Decode
import Pages.StaticHttp as StaticHttp
import Pages.StaticHttp.Request as Request
import Pages.StaticHttpRequest as StaticHttpRequest
import Secrets
import Test exposing (Test, describe, only, test)
@ -13,6 +14,25 @@ getWithoutSecrets url =
StaticHttp.get (Secrets.succeed url)
requestsDict requestMap =
requestMap
|> List.map
(\( request, response ) ->
( request |> Request.hash
, response
)
)
|> Dict.fromList
get : String -> Request.Request
get url =
{ method = "GET"
, url = url
, headers = []
}
all : Test
all =
describe "Static Http Requests"
@ -26,13 +46,13 @@ all =
)
|> (\request ->
StaticHttpRequest.resolveUrls request
(Dict.fromList
[ ( "[GET]first", "null" )
, ( "[GET]NEXT", "null" )
(requestsDict
[ ( get "first", "null" )
, ( get "NEXT", "null" )
]
)
|> Tuple.mapSecond (List.map Secrets.maskedLookup)
|> Expect.equal ( True, [ get "first", get "NEXT" ] )
|> Expect.equal ( True, [ getReq "first", getReq "NEXT" ] )
)
, test "andThen staring with done" <|
\() ->
@ -43,12 +63,12 @@ all =
)
|> (\request ->
StaticHttpRequest.resolveUrls request
(Dict.fromList
[ ( "[GET]NEXT", "null" )
(requestsDict
[ ( get "NEXT", "null" )
]
)
|> Tuple.mapSecond (List.map Secrets.maskedLookup)
|> Expect.equal ( True, [ get "NEXT" ] )
|> Expect.equal ( True, [ getReq "NEXT" ] )
)
, test "map" <|
\() ->
@ -61,13 +81,13 @@ all =
|> StaticHttp.map (\_ -> ())
|> (\request ->
StaticHttpRequest.resolveUrls request
(Dict.fromList
[ ( "[GET]first", "null" )
, ( "[GET]NEXT", "null" )
(requestsDict
[ ( get "first", "null" )
, ( get "NEXT", "null" )
]
)
|> Tuple.mapSecond (List.map Secrets.maskedLookup)
|> Expect.equal ( True, [ get "first", get "NEXT" ] )
|> Expect.equal ( True, [ getReq "first", getReq "NEXT" ] )
)
, test "andThen chain with 1 response available and 1 pending" <|
\() ->
@ -78,12 +98,12 @@ all =
)
|> (\request ->
StaticHttpRequest.resolveUrls request
(Dict.fromList
[ ( "[GET]first", "null" )
(requestsDict
[ ( get "first", "null" )
]
)
|> Tuple.mapSecond (List.map Secrets.maskedLookup)
|> Expect.equal ( False, [ get "first", get "NEXT" ] )
|> Expect.equal ( False, [ getReq "first", getReq "NEXT" ] )
)
, test "andThen chain with 1 response available and 2 pending" <|
\() ->
@ -99,12 +119,15 @@ all =
)
|> (\request ->
StaticHttpRequest.resolveUrls request
(Dict.fromList [ ( "[GET]first", "1" ) ])
(requestsDict
[ ( get "first", "1" )
]
)
|> Tuple.mapSecond (List.map Secrets.maskedLookup)
|> Expect.equal ( False, [ get "first", get "NEXT" ] )
|> Expect.equal ( False, [ getReq "first", getReq "NEXT" ] )
)
]
get url =
getReq url =
{ url = url, method = "GET", headers = [] }