Merge pull request #196 from dillonkearns/worker-threads

Worker threads

.prettierrc

@@ -0,0 +1,7 @@
+{
+	"endOfLine": "lf",
+	"semi": true,
+	"singleQuote": false,
+	"tabWidth": 2,
+	"trailingComma": "es5"
+}

cypress.json

@@ -1,3 +1,4 @@
 {
-    "defaultCommandTimeout": 4000
+    "defaultCommandTimeout": 4000,
+    "baseUrl": "http://localhost:1234"
 }

cypress/fixtures/example.json

@@ -0,0 +1,5 @@
+{
+  "name": "Using fixtures to represent data",
+  "email": "hello@cypress.io",
+  "body": "Fixtures are a great way to mock data for responses to routes"
+}

cypress/integration/elm-pages-dev-base-path.spec.js

@@ -0,0 +1,44 @@
+context("dev server with base path", () => {
+  it("404 message", () => {
+    cy.visit("/qwer/asdf", { failOnStatusCode: false });
+    cy.contains("No route found for /asdf");
+  });
+
+  it("navigates to root page", () => {
+    cy.visit("/qwer/");
+    cy.contains("This is the index page");
+  });
+
+  it("loads file data sources", () => {
+    cy.writeFile(
+      "examples/end-to-end/my-json-data.json",
+      JSON.stringify({
+        greeting: "Hello, World!",
+      })
+    );
+    cy.visit("/qwer/file-data");
+
+    cy.contains("Greeting: Hello, World!");
+    cy.writeFile(
+      "examples/end-to-end/my-json-data.json",
+      JSON.stringify({
+        greeting: "Goodbye, World!",
+      })
+    );
+    cy.contains("Greeting: Goodbye, World!");
+    cy.writeFile(
+      "examples/end-to-end/my-json-data.json",
+      JSON.stringify({
+        greeting: null,
+      })
+    );
+
+    cy.contains(`I encountered some errors while decoding this JSON:
+
+At path /jsonFile/greeting
+
+  I expected a string here, but instead found this value:
+
+    null`);
+  });
+});

cypress/integration/page-changes.spec.js

@@ -0,0 +1,7 @@
+context("dev server with base path", () => {
+  it("navigating with a link successfully resolves data sources", () => {
+    cy.visit("/links");
+    cy.contains("Root page").click();
+    cy.contains("This is the index page.");
+  });
+});

cypress/plugins/index.js

@@ -0,0 +1,22 @@
+/// <reference types="cypress" />
+// ***********************************************************
+// This example plugins/index.js can be used to load plugins
+//
+// You can change the location of this file or turn off loading
+// the plugins file with the 'pluginsFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/plugins-guide
+// ***********************************************************
+
+// This function is called when a project is opened or re-opened (e.g. due to
+// the project's config changing)
+
+/**
+ * @type {Cypress.PluginConfig}
+ */
+// eslint-disable-next-line no-unused-vars
+module.exports = (on, config) => {
+  // `on` is used to hook into various events Cypress emits
+  // `config` is the resolved Cypress config
+}

cypress/support/commands.js

@@ -0,0 +1,25 @@
+// ***********************************************
+// This example commands.js shows you how to
+// create various custom commands and overwrite
+// existing commands.
+//
+// For more comprehensive examples of custom
+// commands please read more here:
+// https://on.cypress.io/custom-commands
+// ***********************************************
+//
+//
+// -- This is a parent command --
+// Cypress.Commands.add('login', (email, password) => { ... })
+//
+//
+// -- This is a child command --
+// Cypress.Commands.add('drag', { prevSubject: 'element'}, (subject, options) => { ... })
+//
+//
+// -- This is a dual command --
+// Cypress.Commands.add('dismiss', { prevSubject: 'optional'}, (subject, options) => { ... })
+//
+//
+// -- This will overwrite an existing command --
+// Cypress.Commands.overwrite('visit', (originalFn, url, options) => { ... })

cypress/support/index.js

@@ -0,0 +1,20 @@
+// ***********************************************************
+// This example support/index.js is processed and
+// loaded automatically before your test files.
+//
+// This is a great place to put global configuration and
+// behavior that modifies Cypress.
+//
+// You can change the location of this file or turn off
+// automatically serving support files with the
+// 'supportFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/configuration
+// ***********************************************************
+
+// Import commands.js using ES2015 syntax:
+import './commands'
+
+// Alternatively you can use CommonJS syntax:
+// require('./commands')

elm.json

@@ -41,7 +41,9 @@
         "elm-community/list-extra": "8.3.0 <= v < 9.0.0",
         "miniBill/elm-codec": "2.0.0 <= v < 3.0.0",
         "noahzgordon/elm-color-extra": "1.0.2 <= v < 2.0.0",
+        "robinheghan/murmur3": "1.0.0 <= v < 2.0.0",
         "tripokey/elm-fuzzy": "5.2.1 <= v < 6.0.0",
+        "vito/elm-ansi": "10.0.1 <= v < 11.0.0",
         "zwilias/json-decode-exploration": "6.0.0 <= v < 7.0.0"
     },
     "test-dependencies": {

examples/docs/elm.json

@@ -32,8 +32,10 @@
             "miniBill/elm-codec": "2.0.0",
             "noahzgordon/elm-color-extra": "1.0.2",
             "pablohirafuji/elm-syntax-highlight": "3.4.0",
+            "robinheghan/murmur3": "1.0.0",
             "rtfeldman/elm-css": "16.1.1",
             "tripokey/elm-fuzzy": "5.2.1",
+            "vito/elm-ansi": "10.0.1",
             "zwilias/json-decode-exploration": "6.0.0"
         },
         "indirect": {

examples/docs/package-lock.json

@@ -29,6 +29,7 @@
         "commander": "^7.2.0",
         "connect": "^3.7.0",
         "cross-spawn": "7.0.3",
+        "elm-doc-preview": "^5.0.5",
         "elm-hot": "^1.1.6",
         "elm-optimize-level-2": "^0.1.5",
         "fs-extra": "^10.0.0",
@@ -36,6 +37,7 @@
         "gray-matter": "^4.0.3",
         "kleur": "^4.1.4",
         "micromatch": "^4.0.4",
+        "node-worker-threads-pool": "^1.5.0",
         "serve-static": "^1.14.1",
         "terser": "^5.7.0",
         "xhr2": "^0.2.1"
@@ -49,7 +51,8 @@
         "@types/micromatch": "^4.0.1",
         "@types/node": "12.20.12",
         "@types/serve-static": "^1.13.9",
-        "elm-review": "^2.5.1",
+        "cypress": "^7.4.0",
+        "elm-review": "^2.5.3",
         "elm-test": "^0.19.1-revision7",
         "elm-tooling": "^1.3.0",
         "elm-verify-examples": "^5.0.0",
@@ -1091,9 +1094,11 @@
         "commander": "^7.2.0",
         "connect": "^3.7.0",
         "cross-spawn": "7.0.3",
+        "cypress": "^7.4.0",
+        "elm-doc-preview": "^5.0.5",
         "elm-hot": "^1.1.6",
         "elm-optimize-level-2": "^0.1.5",
-        "elm-review": "^2.5.1",
+        "elm-review": "^2.5.3",
         "elm-test": "^0.19.1-revision7",
         "elm-tooling": "^1.3.0",
         "elm-verify-examples": "^5.0.0",
@@ -1103,6 +1108,7 @@
         "kleur": "^4.1.4",
         "micromatch": "^4.0.4",
         "mocha": "^8.4.0",
+        "node-worker-threads-pool": "^1.5.0",
         "serve-static": "^1.14.1",
         "terser": "^5.7.0",
         "typescript": "^4.2.4",

examples/docs/src/Shared.elm

@@ -1,4 +1,4 @@
-module Shared exposing (Data, Model, Msg, SharedMsg(..), template)
+module Shared exposing (Data, Model, Msg, template)
 
 import Browser.Navigation
 import DataSource
@@ -8,13 +8,14 @@ import Html.Styled
 import Pages.Flags
 import Pages.PageUrl exposing (PageUrl)
 import Path exposing (Path)
+import Route exposing (Route)
 import SharedTemplate exposing (SharedTemplate)
 import TableOfContents
 import View exposing (View)
 import View.Header
 
 
-template : SharedTemplate Msg Model Data SharedMsg msg
+template : SharedTemplate Msg Model Data msg
 template =
     { init = init
     , update = update
@@ -22,7 +23,6 @@ template =
     , data = data
     , subscriptions = subscriptions
     , onPageChange = Just OnPageChange
-    , sharedMsg = SharedMsg
     }
 
 
@@ -33,17 +33,13 @@ type Msg
         , fragment : Maybe String
         }
     | ToggleMobileMenu
-    | SharedMsg SharedMsg
+    | IncrementFromChild
 
 
 type alias Data =
     TableOfContents.TableOfContents TableOfContents.Data
 
 
-type SharedMsg
-    = IncrementFromChild
-
-
 type alias Model =
     { showMobileMenu : Bool
     , counter : Int
@@ -83,8 +79,6 @@ update msg model =
         ToggleMobileMenu ->
             ( { model | showMobileMenu = not model.showMobileMenu }, Cmd.none )
 
-        SharedMsg globalMsg ->
-            case globalMsg of
         IncrementFromChild ->
             ( { model | counter = model.counter + 1 }, Cmd.none )
 
@@ -103,7 +97,7 @@ view :
     Data
     ->
         { path : Path
-        , frontmatter : route
+        , route : Maybe Route
         }
     -> Model
     -> (Msg -> msg)

examples/end-to-end/.gitignore

@@ -0,0 +1,6 @@
+node_modules/
+elm-stuff/
+dist/
+.cache/
+.elm-pages/
+functions/render/elm-pages-cli.js

examples/end-to-end/README.md

@@ -0,0 +1 @@
+# README

examples/end-to-end/elm-application.json

@@ -0,0 +1,8 @@
+{
+    "name": "dmy/elm-doc-preview",
+    "summary": "Offline documentation previewer",
+    "version": "5.0.0",
+    "exposed-modules": [
+        "Page"
+    ]
+}

examples/end-to-end/elm-tooling.json

@@ -0,0 +1,8 @@
+{
+    "tools": {
+        "elm": "0.19.1",
+        "elm-format": "0.8.4",
+        "elm-json": "0.2.10",
+        "elm-test-rs": "1.0.0"
+    }
+}

examples/end-to-end/elm.json

@@ -0,0 +1,56 @@
+{
+    "type": "application",
+    "source-directories": [
+        "src",
+        "../../src",
+        ".elm-pages",
+        "../../plugins"
+    ],
+    "elm-version": "0.19.1",
+    "dependencies": {
+        "direct": {
+            "MartinSStewart/elm-serialize": "1.2.5",
+            "avh4/elm-color": "1.0.0",
+            "danyx23/elm-mimetype": "4.0.1",
+            "dillonkearns/elm-bcp47-language-tag": "1.0.1",
+            "dillonkearns/elm-markdown": "6.0.1",
+            "elm/browser": "1.0.2",
+            "elm/core": "1.0.5",
+            "elm/html": "1.0.0",
+            "elm/http": "2.0.0",
+            "elm/json": "1.1.3",
+            "elm/regex": "1.0.0",
+            "elm/time": "1.0.0",
+            "elm/url": "1.0.0",
+            "elm-community/dict-extra": "2.4.0",
+            "elm-community/list-extra": "8.3.0",
+            "matheus23/elm-default-tailwind-modules": "2.0.1",
+            "miniBill/elm-codec": "2.0.0",
+            "noahzgordon/elm-color-extra": "1.0.2",
+            "pablohirafuji/elm-syntax-highlight": "3.4.0",
+            "robinheghan/murmur3": "1.0.0",
+            "rtfeldman/elm-css": "16.1.1",
+            "tripokey/elm-fuzzy": "5.2.1",
+            "vito/elm-ansi": "10.0.1",
+            "zwilias/json-decode-exploration": "6.0.0"
+        },
+        "indirect": {
+            "bburdette/toop": "1.0.1",
+            "danfishgold/base64-bytes": "1.1.0",
+            "elm/bytes": "1.0.8",
+            "elm/file": "1.0.5",
+            "elm/parser": "1.1.0",
+            "elm/random": "1.0.0",
+            "elm/virtual-dom": "1.0.2",
+            "fredcy/elm-parseint": "2.0.1",
+            "mgold/elm-nonempty-list": "4.2.0",
+            "rtfeldman/elm-hex": "1.0.0"
+        }
+    },
+    "test-dependencies": {
+        "direct": {
+            "elm-explorations/test": "1.2.2"
+        },
+        "indirect": {}
+    }
+}

examples/end-to-end/greeting.txt

@@ -0,0 +1 @@
+Hello there!

examples/end-to-end/my-json-data.json

@@ -0,0 +1,3 @@
+{
+	"greeting": "Hello, World!"
+}

examples/end-to-end/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "elm-pages-example",
+  "version": "1.0.0",
+  "description": "Example site built with elm-pages.",
+  "scripts": {
+    "start": "elm-pages dev",
+    "serve": "npm run build && http-server ./dist -a localhost -p 3000 -c-1",
+    "build": "elm-pages build"
+  },
+  "author": "Dillon Kearns",
+  "license": "BSD-3",
+  "devDependencies": {
+    "elm-oembed": "0.0.6",
+    "elm-pages": "file:../..",
+    "elm-tooling": "^1.3.0",
+    "http-server": "^0.11.1"
+  }
+}

examples/end-to-end/public/images/elm-logo.svg

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 17.1.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+   viewBox="0 0 323.141 322.95" enable-background="new 0 0 323.141 322.95" xml:space="preserve">
+<g>
+  <polygon
+    fill="#F0AD00"
+    points="161.649,152.782 231.514,82.916 91.783,82.916"/>
+
+  <polygon
+    fill="#7FD13B"
+    points="8.867,0 79.241,70.375 232.213,70.375 161.838,0"/>
+
+  <rect
+    fill="#7FD13B"
+    x="192.99"
+    y="107.392"
+    transform="matrix(0.7071 0.7071 -0.7071 0.7071 186.4727 -127.2386)"
+    width="107.676"
+    height="108.167"/>
+
+  <polygon
+    fill="#60B5CC"
+    points="323.298,143.724 323.298,0 179.573,0"/>
+
+  <polygon
+    fill="#5A6378"
+    points="152.781,161.649 0,8.868 0,314.432"/>
+
+  <polygon
+    fill="#F0AD00"
+    points="255.522,246.655 323.298,314.432 323.298,178.879"/>
+
+  <polygon
+    fill="#60B5CC"
+    points="161.649,170.517 8.869,323.298 314.43,323.298"/>
+</g>
+</svg>

examples/end-to-end/public/images/github.svg

@@ -0,0 +1 @@
+<svg role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>GitHub icon</title><path d="M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12"/></svg>

examples/end-to-end/public/images/icon.svg

@@ -0,0 +1,2 @@
+<svg version="1.1" viewBox="251.0485 144.52063 56.114286 74.5" width="50px" height="74.5"><defs><linearGradient id="grad1" x1="0%" y1="0%" x2="100%" y2="0%"><stop offset="10%" style="stop-color:rgba(1.96%,45.88%,90.2%,1);stop-opacity:1"></stop><stop offset="100%" style="stop-color:rgba(0%,94.9%,37.65%,1);stop-opacity:1"></stop></linearGradient></defs><metadata></metadata><g id="Canvas_11" stroke="none" fill="url(#grad1)" stroke-opacity="1" fill-opacity="1" stroke-dasharray="none"><g id="Canvas_11: Layer 1"><g id="Group_38"><g id="Graphic_32"><path d="M 252.5485 146.02063 L 252.5485 217.52063 L 305.66277 217.52063 L 305.66277 161.68254 L 290.00087 146.02063 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="3"></path></g><g id="Line_34"><line x1="266.07286" y1="182.8279" x2="290.75465" y2="183.00997" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"></line></g><g id="Line_35"><line x1="266.07286" y1="191.84156" x2="290.75465" y2="192.02363" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"></line></g><g id="Line_36"><line x1="266.07286" y1="200.85522" x2="290.75465" y2="201.0373" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"></line></g><g id="Line_37"><line x1="266.07286" y1="164.80058" x2="278.3874" y2="164.94049" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"></line></g></g></g></g></svg>
+

examples/end-to-end/public/index.js

@@ -0,0 +1,6 @@
+export default {
+  load: function (elmLoaded) {},
+  flags: function () {
+    return null;
+  },
+};

examples/end-to-end/public/style.css

@@ -0,0 +1,79 @@
+@import url("https://rsms.me/inter/inter.css");
+@import url("https://fonts.googleapis.com/css2?family=IBM+Plex+Mono&display=swap");
+
+body {
+  font-family: "Inter var" !important;
+}
+
+pre.elmsh {
+  padding: 10px;
+  margin: 0;
+  text-align: left;
+  overflow: auto;
+  height: 100%;
+  width: 500px;
+  font-size: 14px;
+  font-family: "IBM Plex Mono" !important;
+}
+code.elmsh {
+  padding: 0;
+}
+.elmsh-line:before {
+  content: attr(data-elmsh-lc);
+  display: inline-block;
+  text-align: right;
+  width: 40px;
+  padding: 0 20px 0 0;
+  opacity: 0.3;
+}
+.elmsh {
+  color: #f8f8f2;
+  background: #1e1e1e;
+}
+.elmsh-hl {
+  background: #4864aa;
+}
+.elmsh-add {
+  background: #003800;
+}
+.elmsh-del {
+  background: #380000;
+}
+.elmsh-comm {
+  color: #d4d4d4;
+}
+.elmsh1 {
+  color: #74b0df;
+}
+.elmsh2 {
+  color: #ce9178;
+}
+.elmsh3 {
+  color: #ff00ff;
+}
+.elmsh4 {
+  color: #4f76ac;
+}
+.elmsh5 {
+  color: #3dc9b0;
+}
+.elmsh6 {
+  color: #74b0df;
+}
+.elmsh7 {
+  color: #ce9178;
+}
+.elmsh-elm-ts,
+.elmsh-js-dk,
+.elmsh-css-p {
+  font-style: italic;
+  color: #4f76ac;
+}
+.elmsh-js-ce {
+  font-style: italic;
+  color: #5bb498;
+}
+.elmsh-css-ar-i {
+  font-weight: bold;
+  color: #ff0000;
+}

examples/end-to-end/public/syntax.css

@@ -0,0 +1,43 @@
+pre.elmsh {
+  padding: 10px;
+  margin: 0;
+  text-align: left;
+  overflow: auto;
+  padding: 20px !important;
+}
+
+code.elmsh {
+  padding: 0;
+}
+
+code {
+  font-family: 'Roboto Mono' !important;
+  font-size: 20px !important;
+  line-height: 28px;
+}
+
+.elmsh-line:before {
+  /* content: attr(data-elmsh-lc); */
+  display: inline-block;
+  text-align: right;
+  width: 40px;
+  padding: 0 20px 0 0;
+  opacity: 0.3;
+}
+
+.elmsh {
+    color: #f8f8f2;
+    background: #000;
+}
+.elmsh-hl {background: #343434;}
+.elmsh-add {background: #003800;}
+.elmsh-del {background: #380000;}
+.elmsh-comm {color: #75715e;}
+.elmsh1 {color: #ae81ff;}
+.elmsh2 {color: #e6db74;}
+.elmsh3 {color: #66d9ef;}
+.elmsh4 {color: #f92672;}
+.elmsh5 {color: #a6e22e;}
+.elmsh6 {color: #ae81ff;}
+.elmsh7 {color: #fd971f;}
+

examples/end-to-end/src/Api.elm

@@ -0,0 +1,18 @@
+module Api exposing (routes)
+
+import ApiRoute
+import DataSource exposing (DataSource)
+import DataSource.Http
+import Html exposing (Html)
+import Json.Encode
+import OptimizedDecoder as Decode
+import Route exposing (Route)
+import Secrets
+
+
+routes :
+    DataSource (List Route)
+    -> (Html Never -> String)
+    -> List (ApiRoute.Done ApiRoute.Response)
+routes getStaticRoutes htmlToString =
+    []

examples/end-to-end/src/MarkdownRenderer.elm

@@ -0,0 +1,233 @@
+module MarkdownRenderer exposing (renderer)
+
+import Html.Styled as Html
+import Html.Styled.Attributes as Attr exposing (css)
+import Markdown.Block as Block exposing (ListItem(..), Task(..))
+import Markdown.Html
+import Markdown.Renderer
+import SyntaxHighlight
+import Tailwind.Utilities as Tw
+
+
+renderer : Markdown.Renderer.Renderer (Html.Html msg)
+renderer =
+    { heading = heading
+    , paragraph = Html.p []
+    , thematicBreak = Html.hr [] []
+    , text = Html.text
+    , strong = \content -> Html.strong [ css [ Tw.font_bold ] ] content
+    , emphasis = \content -> Html.em [ css [ Tw.italic ] ] content
+    , blockQuote = Html.blockquote []
+    , codeSpan =
+        \content ->
+            Html.code
+                [ css
+                    [ Tw.font_semibold
+                    , Tw.font_medium
+                    ]
+                ]
+                [ Html.text content ]
+
+    --, codeSpan = code
+    , link =
+        \{ destination } body ->
+            Html.a
+                [ Attr.href destination
+                , css
+                    [ Tw.underline
+                    ]
+                ]
+                body
+    , hardLineBreak = Html.br [] []
+    , image =
+        \image ->
+            case image.title of
+                Just _ ->
+                    Html.img [ Attr.src image.src, Attr.alt image.alt ] []
+
+                Nothing ->
+                    Html.img [ Attr.src image.src, Attr.alt image.alt ] []
+    , unorderedList =
+        \items ->
+            Html.ul []
+                (items
+                    |> List.map
+                        (\item ->
+                            case item of
+                                Block.ListItem task children ->
+                                    let
+                                        checkbox =
+                                            case task of
+                                                Block.NoTask ->
+                                                    Html.text ""
+
+                                                Block.IncompleteTask ->
+                                                    Html.input
+                                                        [ Attr.disabled True
+                                                        , Attr.checked False
+                                                        , Attr.type_ "checkbox"
+                                                        ]
+                                                        []
+
+                                                Block.CompletedTask ->
+                                                    Html.input
+                                                        [ Attr.disabled True
+                                                        , Attr.checked True
+                                                        , Attr.type_ "checkbox"
+                                                        ]
+                                                        []
+                                    in
+                                    Html.li [] (checkbox :: children)
+                        )
+                )
+    , orderedList =
+        \startingIndex items ->
+            Html.ol
+                (case startingIndex of
+                    1 ->
+                        [ Attr.start startingIndex ]
+
+                    _ ->
+                        []
+                )
+                (items
+                    |> List.map
+                        (\itemBlocks ->
+                            Html.li []
+                                itemBlocks
+                        )
+                )
+    , html = Markdown.Html.oneOf []
+    , codeBlock = codeBlock
+
+    --\{ body, language } ->
+    --    let
+    --        classes =
+    --            -- Only the first word is used in the class
+    --            case Maybe.map String.words language of
+    --                Just (actualLanguage :: _) ->
+    --                    [ Attr.class <| "language-" ++ actualLanguage ]
+    --
+    --                _ ->
+    --                    []
+    --    in
+    --    Html.pre []
+    --        [ Html.code classes
+    --            [ Html.text body
+    --            ]
+    --        ]
+    , table = Html.table []
+    , tableHeader = Html.thead []
+    , tableBody = Html.tbody []
+    , tableRow = Html.tr []
+    , strikethrough =
+        \children -> Html.del [] children
+    , tableHeaderCell =
+        \maybeAlignment ->
+            let
+                attrs =
+                    maybeAlignment
+                        |> Maybe.map
+                            (\alignment ->
+                                case alignment of
+                                    Block.AlignLeft ->
+                                        "left"
+
+                                    Block.AlignCenter ->
+                                        "center"
+
+                                    Block.AlignRight ->
+                                        "right"
+                            )
+                        |> Maybe.map Attr.align
+                        |> Maybe.map List.singleton
+                        |> Maybe.withDefault []
+            in
+            Html.th attrs
+    , tableCell =
+        \maybeAlignment ->
+            let
+                attrs =
+                    maybeAlignment
+                        |> Maybe.map
+                            (\alignment ->
+                                case alignment of
+                                    Block.AlignLeft ->
+                                        "left"
+
+                                    Block.AlignCenter ->
+                                        "center"
+
+                                    Block.AlignRight ->
+                                        "right"
+                            )
+                        |> Maybe.map Attr.align
+                        |> Maybe.map List.singleton
+                        |> Maybe.withDefault []
+            in
+            Html.td attrs
+    }
+
+
+rawTextToId : String -> String
+rawTextToId rawText =
+    rawText
+        |> String.split " "
+        |> String.join "-"
+        |> String.toLower
+
+
+heading : { level : Block.HeadingLevel, rawText : String, children : List (Html.Html msg) } -> Html.Html msg
+heading { level, rawText, children } =
+    (case level of
+        Block.H1 ->
+            Html.h1
+
+        Block.H2 ->
+            Html.h2
+
+        Block.H3 ->
+            Html.h3
+
+        Block.H4 ->
+            Html.h4
+
+        Block.H5 ->
+            Html.h5
+
+        Block.H6 ->
+            Html.h6
+    )
+        [ Attr.id (rawTextToId rawText)
+        , Attr.attribute "name" (rawTextToId rawText)
+        , css
+            [ Tw.font_bold
+            , Tw.text_2xl
+            , Tw.mt_8
+            , Tw.mb_4
+            ]
+        ]
+        children
+
+
+
+--code : String -> Element msg
+--code snippet =
+--    Element.el
+--        [ Element.Background.color
+--            (Element.rgba255 50 50 50 0.07)
+--        , Element.Border.rounded 2
+--        , Element.paddingXY 5 3
+--        , Font.family [ Font.typeface "Roboto Mono", Font.monospace ]
+--        ]
+--        (Element.text snippet)
+--
+--
+
+
+codeBlock : { body : String, language : Maybe String } -> Html.Html msg
+codeBlock details =
+    SyntaxHighlight.elm details.body
+        |> Result.map (SyntaxHighlight.toBlockHtml (Just 1))
+        |> Result.map Html.fromUnstyled
+        |> Result.withDefault (Html.pre [] [ Html.code [] [ Html.text details.body ] ])

examples/end-to-end/src/Page/FileData.elm

@@ -0,0 +1,79 @@
+module Page.FileData exposing (Data, Model, Msg, page)
+
+import DataSource exposing (DataSource)
+import DataSource.File
+import Head
+import Head.Seo as Seo
+import Html.Styled exposing (text)
+import OptimizedDecoder as Decode
+import Page exposing (Page, PageWithState, StaticPayload)
+import Pages.PageUrl exposing (PageUrl)
+import Pages.Url
+import Shared
+import View exposing (View)
+
+
+type alias Model =
+    ()
+
+
+type alias Msg =
+    Never
+
+
+type alias RouteParams =
+    {}
+
+
+page : Page RouteParams Data
+page =
+    Page.single
+        { head = head
+        , data = data
+        }
+        |> Page.buildNoState { view = view }
+
+
+type alias Data =
+    { greeting : String
+    }
+
+
+data : DataSource Data
+data =
+    "my-json-data.json"
+        |> DataSource.File.jsonFile (Decode.field "greeting" Decode.string)
+        |> DataSource.map Data
+
+
+head :
+    StaticPayload Data RouteParams
+    -> List Head.Tag
+head static =
+    Seo.summary
+        { canonicalUrlOverride = Nothing
+        , siteName = "elm-pages"
+        , image =
+            { url = Pages.Url.external "TODO"
+            , alt = "elm-pages logo"
+            , dimensions = Nothing
+            , mimeType = Nothing
+            }
+        , description = "TODO"
+        , locale = Nothing
+        , title = "TODO title" -- metadata.title -- TODO
+        }
+        |> Seo.website
+
+
+view :
+    Maybe PageUrl
+    -> Shared.Model
+    -> StaticPayload Data RouteParams
+    -> View Msg
+view maybeUrl sharedModel static =
+    { title = "Index page"
+    , body =
+        [ text <| "Greeting: " ++ static.data.greeting
+        ]
+    }

examples/end-to-end/src/Page/Index.elm

@@ -0,0 +1,76 @@
+module Page.Index exposing (Data, Model, Msg, page)
+
+import DataSource exposing (DataSource)
+import DataSource.File
+import Head
+import Head.Seo as Seo
+import Html.Styled exposing (text)
+import Page exposing (Page, PageWithState, StaticPayload)
+import Pages.PageUrl exposing (PageUrl)
+import Pages.Url
+import Shared
+import View exposing (View)
+
+
+type alias Model =
+    ()
+
+
+type alias Msg =
+    Never
+
+
+type alias RouteParams =
+    {}
+
+
+page : Page RouteParams Data
+page =
+    Page.single
+        { head = head
+        , data = data
+        }
+        |> Page.buildNoState { view = view }
+
+
+type alias Data =
+    String
+
+
+data : DataSource Data
+data =
+    DataSource.File.rawFile "greeting.txt"
+
+
+head :
+    StaticPayload Data RouteParams
+    -> List Head.Tag
+head static =
+    Seo.summary
+        { canonicalUrlOverride = Nothing
+        , siteName = "elm-pages"
+        , image =
+            { url = Pages.Url.external "TODO"
+            , alt = "elm-pages logo"
+            , dimensions = Nothing
+            , mimeType = Nothing
+            }
+        , description = "TODO"
+        , locale = Nothing
+        , title = "TODO title" -- metadata.title -- TODO
+        }
+        |> Seo.website
+
+
+view :
+    Maybe PageUrl
+    -> Shared.Model
+    -> StaticPayload Data RouteParams
+    -> View Msg
+view maybeUrl sharedModel static =
+    { title = "Index page"
+    , body =
+        [ text "This is the index page."
+        , text <| "Greeting: " ++ static.data
+        ]
+    }

examples/end-to-end/src/Page/Links.elm

@@ -0,0 +1,76 @@
+module Page.Links exposing (Data, Model, Msg, page)
+
+import DataSource exposing (DataSource)
+import Head
+import Head.Seo as Seo
+import Html.Styled as Html exposing (text)
+import Html.Styled.Attributes as Attr
+import Page exposing (Page, PageWithState, StaticPayload)
+import Pages.PageUrl exposing (PageUrl)
+import Pages.Url
+import Shared
+import View exposing (View)
+
+
+type alias Model =
+    ()
+
+
+type alias Msg =
+    Never
+
+
+type alias RouteParams =
+    {}
+
+
+page : Page RouteParams Data
+page =
+    Page.single
+        { head = head
+        , data = data
+        }
+        |> Page.buildNoState { view = view }
+
+
+type alias Data =
+    ()
+
+
+data : DataSource Data
+data =
+    DataSource.succeed ()
+
+
+head :
+    StaticPayload Data RouteParams
+    -> List Head.Tag
+head static =
+    Seo.summary
+        { canonicalUrlOverride = Nothing
+        , siteName = "elm-pages"
+        , image =
+            { url = Pages.Url.external "TODO"
+            , alt = "elm-pages logo"
+            , dimensions = Nothing
+            , mimeType = Nothing
+            }
+        , description = "TODO"
+        , locale = Nothing
+        , title = "TODO title" -- metadata.title -- TODO
+        }
+        |> Seo.website
+
+
+view :
+    Maybe PageUrl
+    -> Shared.Model
+    -> StaticPayload Data RouteParams
+    -> View Msg
+view maybeUrl sharedModel static =
+    { title = "Links"
+    , body =
+        [ Html.a [ Attr.href "/" ]
+            [ text "Root page" ]
+        ]
+    }

examples/end-to-end/src/Shared.elm

@@ -0,0 +1,105 @@
+module Shared exposing (Data, Model, Msg(..), SharedMsg(..), template)
+
+import Browser.Navigation
+import Css.Global
+import DataSource
+import DataSource.Http
+import Html exposing (Html)
+import Html.Styled
+import OptimizedDecoder as D
+import Pages.Flags
+import Pages.PageUrl exposing (PageUrl)
+import Path exposing (Path)
+import Route exposing (Route)
+import Secrets
+import SharedTemplate exposing (SharedTemplate)
+import Tailwind.Utilities
+import View exposing (View)
+
+
+template : SharedTemplate Msg Model Data msg
+template =
+    { init = init
+    , update = update
+    , view = view
+    , data = data
+    , subscriptions = subscriptions
+    , onPageChange = Just OnPageChange
+    }
+
+
+type Msg
+    = OnPageChange
+        { path : Path
+        , query : Maybe String
+        , fragment : Maybe String
+        }
+
+
+type alias Data =
+    ()
+
+
+type SharedMsg
+    = NoOp
+
+
+type alias Model =
+    { showMobileMenu : Bool
+    }
+
+
+init :
+    Maybe Browser.Navigation.Key
+    -> Pages.Flags.Flags
+    ->
+        Maybe
+            { path :
+                { path : Path
+                , query : Maybe String
+                , fragment : Maybe String
+                }
+            , metadata : route
+            , pageUrl : Maybe PageUrl
+            }
+    -> ( Model, Cmd Msg )
+init _ flags maybePagePath =
+    ( { showMobileMenu = False }
+    , Cmd.none
+    )
+
+
+update : Msg -> Model -> ( Model, Cmd Msg )
+update msg model =
+    case msg of
+        OnPageChange _ ->
+            ( { model | showMobileMenu = False }, Cmd.none )
+
+
+subscriptions : Path -> Model -> Sub Msg
+subscriptions _ _ =
+    Sub.none
+
+
+data : DataSource.DataSource Data
+data =
+    DataSource.succeed ()
+
+
+view :
+    Data
+    ->
+        { path : Path
+        , route : Maybe Route
+        }
+    -> Model
+    -> (Msg -> msg)
+    -> View msg
+    -> { body : Html msg, title : String }
+view stars page model toMsg pageView =
+    { body =
+        Html.Styled.div []
+            pageView.body
+            |> Html.Styled.toUnstyled
+    , title = pageView.title
+    }

examples/end-to-end/src/Site.elm

@@ -0,0 +1,93 @@
+module Site exposing (config)
+
+import Cloudinary
+import DataSource
+import Head
+import MimeType
+import Pages.Manifest as Manifest
+import Pages.Url
+import Route exposing (Route)
+import SiteConfig exposing (SiteConfig)
+
+
+config : SiteConfig Data
+config =
+    \routes ->
+        { data = data
+        , canonicalUrl = canonicalUrl
+        , manifest = manifest
+        , head = head
+        }
+
+
+type alias Data =
+    { siteName : String
+    }
+
+
+data : DataSource.DataSource Data
+data =
+    DataSource.map Data
+        --(StaticFile.request "site-name.txt" StaticFile.body)
+        (DataSource.succeed "site-name")
+
+
+head : Data -> List Head.Tag
+head static =
+    [ Head.icon [ ( 32, 32 ) ] MimeType.Png (cloudinaryIcon MimeType.Png 32)
+    , Head.icon [ ( 16, 16 ) ] MimeType.Png (cloudinaryIcon MimeType.Png 16)
+    , Head.appleTouchIcon (Just 180) (cloudinaryIcon MimeType.Png 180)
+    , Head.appleTouchIcon (Just 192) (cloudinaryIcon MimeType.Png 192)
+    , Head.sitemapLink "/sitemap.xml"
+    ]
+
+
+canonicalUrl : String
+canonicalUrl =
+    "https://elm-pages.com"
+
+
+manifest : Data -> Manifest.Config
+manifest static =
+    Manifest.init
+        { name = static.siteName
+        , description = "elm-pages - " ++ tagline
+        , startUrl = Route.Index |> Route.toPath
+        , icons =
+            [ icon webp 192
+            , icon webp 512
+            , icon MimeType.Png 192
+            , icon MimeType.Png 512
+            ]
+        }
+        |> Manifest.withShortName "elm-pages"
+
+
+tagline : String
+tagline =
+    "A statically typed site generator"
+
+
+webp : MimeType.MimeImage
+webp =
+    MimeType.OtherImage "webp"
+
+
+icon :
+    MimeType.MimeImage
+    -> Int
+    -> Manifest.Icon
+icon format width =
+    { src = cloudinaryIcon format width
+    , sizes = [ ( width, width ) ]
+    , mimeType = format |> Just
+    , purposes = [ Manifest.IconPurposeAny, Manifest.IconPurposeMaskable ]
+    }
+
+
+cloudinaryIcon :
+    MimeType.MimeImage
+    -> Int
+    -> Pages.Url.Url
+cloudinaryIcon mimeType width =
+    Cloudinary.urlSquare "v1603234028/elm-pages/elm-pages-icon" (Just mimeType) width

examples/end-to-end/src/View.elm

@@ -0,0 +1,23 @@
+module View exposing (View, map, placeholder)
+
+import Html.Styled exposing (text)
+
+
+type alias View msg =
+    { title : String
+    , body : List (Html.Styled.Html msg)
+    }
+
+
+map : (msg1 -> msg2) -> View msg1 -> View msg2
+map fn doc =
+    { title = doc.title
+    , body = List.map (Html.Styled.map fn) doc.body
+    }
+
+
+placeholder : String -> View msg
+placeholder moduleName =
+    { title = "Placeholder - " ++ moduleName
+    , body = [ text moduleName ]
+    }

examples/pokedex/elm.json

@@ -29,8 +29,10 @@
             "miniBill/elm-codec": "1.2.0",
             "noahzgordon/elm-color-extra": "1.0.2",
             "pablohirafuji/elm-syntax-highlight": "3.4.0",
+            "robinheghan/murmur3": "1.0.0",
             "rtfeldman/elm-css": "16.1.1",
             "tripokey/elm-fuzzy": "5.2.1",
+            "vito/elm-ansi": "10.0.1",
             "zwilias/json-decode-exploration": "6.0.0"
         },
         "indirect": {

examples/pokedex/src/Shared.elm

@@ -6,11 +6,12 @@ import Html exposing (Html)
 import Pages.Flags
 import Pages.PageUrl exposing (PageUrl)
 import Path exposing (Path)
+import Route exposing (Route)
 import SharedTemplate exposing (SharedTemplate)
 import View exposing (View)
 
 
-template : SharedTemplate Msg Model Data SharedMsg msg
+template : SharedTemplate Msg Model Data msg
 template =
     { init = init
     , update = update
@@ -18,7 +19,6 @@ template =
     , data = data
     , subscriptions = subscriptions
     , onPageChange = Just OnPageChange
-    , sharedMsg = SharedMsg
     }
 
 
@@ -88,7 +88,7 @@ view :
     Data
     ->
         { path : Path
-        , frontmatter : route
+        , route : Maybe Route
         }
     -> Model
     -> (Msg -> msg)

examples/routing/elm.json

@@ -28,8 +28,10 @@
             "miniBill/elm-codec": "2.0.0",
             "noahzgordon/elm-color-extra": "1.0.2",
             "pablohirafuji/elm-syntax-highlight": "3.4.0",
+            "robinheghan/murmur3": "1.0.0",
             "rtfeldman/elm-css": "16.1.1",
             "tripokey/elm-fuzzy": "5.2.1",
+            "vito/elm-ansi": "10.0.1",
             "zwilias/json-decode-exploration": "6.0.0"
         },
         "indirect": {

examples/slides/elm.json

@@ -9,6 +9,7 @@
     "elm-version": "0.19.1",
     "dependencies": {
         "direct": {
+            "MartinSStewart/elm-serialize": "1.2.5",
             "avh4/elm-color": "1.0.0",
             "billstclair/elm-xml-eeue56": "1.0.1",
             "danyx23/elm-mimetype": "4.0.1",
@@ -46,6 +47,8 @@
             "zwilias/json-decode-exploration": "6.0.0"
         },
         "indirect": {
+            "bburdette/toop": "1.0.1",
+            "danfishgold/base64-bytes": "1.1.0",
             "elm/bytes": "1.0.8",
             "elm/file": "1.0.5",
             "elm/parser": "1.1.0",

generator/src/Page.elm

@@ -70,7 +70,7 @@ type alias PageWithState routeParams templateData templateModel templateMsg =
         StaticPayload templateData routeParams
         -> List Head.Tag
     , init : Maybe PageUrl -> Shared.Model -> StaticPayload templateData routeParams -> ( templateModel, Cmd templateMsg )
-    , update : PageUrl -> StaticPayload templateData routeParams -> Maybe Browser.Navigation.Key -> templateMsg -> templateModel -> Shared.Model -> ( templateModel, Cmd templateMsg, Maybe Shared.SharedMsg )
+    , update : PageUrl -> StaticPayload templateData routeParams -> Maybe Browser.Navigation.Key -> templateMsg -> templateModel -> Shared.Model -> ( templateModel, Cmd templateMsg, Maybe Shared.Msg )
     , subscriptions : Maybe PageUrl -> routeParams -> Path -> templateModel -> Shared.Model -> Sub templateMsg
     , handleRoute : { moduleName : List String, routePattern : RoutePattern } -> (routeParams -> List ( String, String )) -> routeParams -> DataSource (Maybe NotFoundReason)
     , kind : String
@@ -188,7 +188,7 @@ buildWithSharedState :
         -> StaticPayload templateData routeParams
         -> View templateMsg
     , init : Maybe PageUrl -> Shared.Model -> StaticPayload templateData routeParams -> ( templateModel, Cmd templateMsg )
-    , update : PageUrl -> Maybe Browser.Navigation.Key -> Shared.Model -> StaticPayload templateData routeParams -> templateMsg -> templateModel -> ( templateModel, Cmd templateMsg, Maybe Shared.SharedMsg )
+    , update : PageUrl -> Maybe Browser.Navigation.Key -> Shared.Model -> StaticPayload templateData routeParams -> templateMsg -> templateModel -> ( templateModel, Cmd templateMsg, Maybe Shared.Msg )
     , subscriptions : Maybe PageUrl -> routeParams -> Path -> templateModel -> Shared.Model -> Sub templateMsg
     }
     -> Builder routeParams templateData

generator/src/SharedTemplate.elm

@@ -10,7 +10,7 @@ import Route exposing (Route)
 import View exposing (View)
 
 
-type alias SharedTemplate msg sharedModel sharedData sharedMsg mappedMsg =
+type alias SharedTemplate msg sharedModel sharedData mappedMsg =
     { init :
         Maybe Browser.Navigation.Key
         -> Flags
@@ -30,7 +30,7 @@ type alias SharedTemplate msg sharedModel sharedData sharedMsg mappedMsg =
         sharedData
         ->
             { path : Path
-            , frontmatter : Maybe Route
+            , route : Maybe Route
             }
         -> sharedModel
         -> (msg -> mappedMsg)
@@ -46,5 +46,4 @@ type alias SharedTemplate msg sharedModel sharedData sharedMsg mappedMsg =
              }
              -> msg
             )
-    , sharedMsg : sharedMsg -> msg
     }

generator/src/basepath-middleware.js

@@ -0,0 +1,38 @@
+const parseUrl = require("url").parse;
+
+// this middleware is only active when (config.base !== '/')
+
+module.exports = function baseMiddleware(base) {
+  // Keep the named function. The name is visible in debug logs via `DEBUG=connect:dispatcher ...`
+  return function viteBaseMiddleware(req, res, next) {
+    const url = req.url;
+    const parsed = parseUrl(url);
+    const path = parsed.pathname || "/";
+
+    if (path.startsWith(base)) {
+      // rewrite url to remove base.. this ensures that other middleware does
+      // not need to consider base being prepended or not
+      req.url = url.replace(base, "/");
+      return next();
+    }
+
+    if (path === "/" || path === "/index.html") {
+      // redirect root visit to based url
+      res.writeHead(302, {
+        Location: base,
+      });
+      res.end();
+      return;
+    } else if (req.headers.accept && req.headers.accept.includes("text/html")) {
+      // non-based page visit
+      res.statusCode = 404;
+      res.end(
+        `The server is configured with a public base URL of ${base} - ` +
+          `did you mean to visit ${base}${url.slice(1)} instead?`
+      );
+      return;
+    }
+
+    next();
+  };
+};

generator/src/build.js

@@ -1,18 +1,25 @@
 const fs = require("./dir-helpers.js");
+const fsPromises = require("fs").promises;
+
+const { restoreColor } = require("./error-formatter");
 const path = require("path");
 const spawnCallback = require("cross-spawn").spawn;
 const codegen = require("./codegen.js");
 const terser = require("terser");
-const matter = require("gray-matter");
+const os = require("os");
-const globby = require("globby");
+const { Worker, SHARE_ENV } = require("worker_threads");
-const preRenderHtml = require("./pre-render-html.js");
+const { ensureDirSync } = require("./file-helpers.js");
+let pool = [];
+let pagesReady;
+let pages = new Promise((resolve, reject) => {
+  pagesReady = resolve;
+});
 
 const DIR_PATH = path.join(process.cwd());
 const OUTPUT_FILE_NAME = "elm.js";
 
-let foundErrors = false;
 process.on("unhandledRejection", (error) => {
-  console.error(error);
+  console.error("Unhandled: ", error);
   process.exitCode = 1;
 });
 
@@ -23,146 +30,103 @@ const ELM_FILE_PATH = path.join(
 );
 
 async function ensureRequiredDirs() {
-  await fs.tryMkdir(`dist`);
+  ensureDirSync(`dist`);
+  ensureDirSync(path.join(process.cwd(), ".elm-pages", "http-response-cache"));
 }
 
 async function run(options) {
   await ensureRequiredDirs();
-  XMLHttpRequest = require("xhr2");
+  // since init/update are never called in pre-renders, and DataSource.Http is called using undici
+  // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
+  XMLHttpRequest = {};
 
-  const generateCode = codegen.generate();
+  const generateCode = codegen.generate(options.base);
 
   const copyDone = copyAssets();
   await generateCode;
   const cliDone = runCli(options);
   const compileClientDone = compileElm(options);
 
+  try {
     await Promise.all([copyDone, cliDone, compileClientDone]);
+  } catch (error) {
+    console.log(error);
+  }
+}
+
+/**
+ * @param {string} basePath
+ */
+function initWorker(basePath) {
+  return new Promise((resolve, reject) => {
+    let newWorker = {
+      worker: new Worker(path.join(__dirname, "./render-worker.js"), {
+        env: SHARE_ENV,
+        workerData: { basePath },
+      }),
+    };
+    newWorker.worker.once("online", () => {
+      newWorker.worker.on("message", (message) => {
+        if (message.tag === "all-paths") {
+          pagesReady(JSON.parse(message.data));
+        } else if (message.tag === "error") {
+          process.exitCode = 1;
+          console.error(restoreColor(message.data.errorsJson));
+          buildNextPage(newWorker);
+        } else if (message.tag === "done") {
+          buildNextPage(newWorker);
+        } else {
+          throw `Unhandled tag ${message.tag}`;
+        }
+      });
+      newWorker.worker.on("error", (error) => {
+        console.error("Unhandled worker exception", error);
+        process.exitCode = 1;
+        buildNextPage(newWorker);
+      });
+      resolve(newWorker);
+    });
+  });
+}
+
+/**
+ */
+function prepareStaticPathsNew(thread) {
+  thread.worker.postMessage({
+    mode: "build",
+    tag: "render",
+    pathname: "/all-paths.json",
+  });
+}
+
+async function buildNextPage(thread) {
+  let nextPage = (await pages).pop();
+  if (nextPage) {
+    thread.worker.postMessage({
+      mode: "build",
+      tag: "render",
+      pathname: nextPage,
+    });
+  } else {
+    thread.worker.terminate();
+  }
 }
 
 async function runCli(options) {
   await compileCliApp(options);
-  runElmApp();
+  const cpuCount = os.cpus().length;
-}
+  console.log("Threads: ", cpuCount);
 
-function runElmApp() {
+  const getPathsWorker = initWorker(options.base);
-  process.on("beforeExit", (code) => {
+  getPathsWorker.then(prepareStaticPathsNew);
-    if (foundErrors) {
+  const threadsToCreate = Math.max(1, cpuCount / 2 - 1);
-      process.exitCode = 1;
+  pool.push(getPathsWorker);
-    } else {
+  for (let index = 0; index < threadsToCreate - 1; index++) {
+    pool.push(initWorker(options.base));
   }
+  pool.forEach((threadPromise) => {
+    threadPromise.then(buildNextPage);
   });
-
-  return new Promise((resolve, _) => {
-    const mode /** @type { "dev" | "prod" } */ = "elm-to-html-beta";
-    const staticHttpCache = {};
-    const app = require(ELM_FILE_PATH).Elm.TemplateModulesBeta.init({
-      flags: { secrets: process.env, mode, staticHttpCache },
-    });
-
-    app.ports.toJsPort.subscribe(async (/** @type { FromElm }  */ fromElm) => {
-      // console.log({ fromElm });
-      if (fromElm.command === "log") {
-        console.log(fromElm.value);
-      } else if (fromElm.tag === "InitialData") {
-        generateFiles(fromElm.args[0].filesToGenerate);
-      } else if (fromElm.tag === "PageProgress") {
-        outputString(fromElm);
-      } else if (fromElm.tag === "ReadFile") {
-        const filePath = fromElm.args[0];
-        try {
-          const fileContents = (await fs.readFile(filePath)).toString();
-          const parsedFile = matter(fileContents);
-          app.ports.fromJsPort.send({
-            tag: "GotFile",
-            data: {
-              filePath,
-              parsedFrontmatter: parsedFile.data,
-              withoutFrontmatter: parsedFile.content,
-              rawFile: fileContents,
-              jsonFile: jsonOrNull(fileContents),
-            },
-          });
-        } catch (error) {
-          app.ports.fromJsPort.send({
-            tag: "BuildError",
-            data: { filePath },
-          });
-        }
-      } else if (fromElm.tag === "Glob") {
-        const globPattern = fromElm.args[0];
-        const matchedPaths = await globby(globPattern);
-
-        app.ports.fromJsPort.send({
-          tag: "GotGlob",
-          data: { pattern: globPattern, result: matchedPaths },
-        });
-      } else if (fromElm.tag === "Errors") {
-        console.error(fromElm.args[0].errorString);
-        foundErrors = true;
-      } else {
-        console.log(fromElm);
-        throw "Unknown port tag.";
-      }
-    });
-  });
-}
-
-/**
- * @param {{ path: string; content: string; }[]} filesToGenerate
- */
-async function generateFiles(filesToGenerate) {
-  filesToGenerate.forEach(async ({ path: pathToGenerate, content }) => {
-    const fullPath = `dist/${pathToGenerate}`;
-    console.log(`Generating file /${pathToGenerate}`);
-    await fs.tryMkdir(path.dirname(fullPath));
-    fs.writeFile(fullPath, content);
-  });
-}
-
-/**
- * @param {string} route
- */
-function cleanRoute(route) {
-  return route.replace(/(^\/|\/$)/, "");
-}
-
-/**
- * @param {string} cleanedRoute
- */
-function pathToRoot(cleanedRoute) {
-  return cleanedRoute === ""
-    ? cleanedRoute
-    : cleanedRoute
-        .split("/")
-        .map((_) => "..")
-        .join("/")
-        .replace(/\.$/, "./");
-}
-
-/**
- * @param {string} route
- */
-function baseRoute(route) {
-  const cleanedRoute = cleanRoute(route);
-  return cleanedRoute === "" ? "./" : pathToRoot(route);
-}
-
-async function outputString(/** @type { PageProgress } */ fromElm) {
-  const args = fromElm.args[0];
-  console.log(`Pre-rendered /${args.route}`);
-  const normalizedRoute = args.route.replace(/index$/, "");
-  // await fs.mkdir(`./dist/${normalizedRoute}`, { recursive: true });
-  await fs.tryMkdir(`./dist/${normalizedRoute}`);
-  const contentJsonString = JSON.stringify({
-    is404: args.is404,
-    staticData: args.contentJson,
-  });
-  fs.writeFile(
-    `dist/${normalizedRoute}/index.html`,
-    preRenderHtml(args, contentJsonString, false)
-  );
-  fs.writeFile(`dist/${normalizedRoute}/content.json`, contentJsonString);
 }
 
 async function compileElm(options) {
@@ -175,19 +139,33 @@ async function compileElm(options) {
   }
 }
 
-function spawnElmMake(options, elmEntrypointPath, outputPath, cwd) {
+function elmOptimizeLevel2(elmEntrypointPath, outputPath, cwd) {
   return new Promise((resolve, reject) => {
     const fullOutputPath = cwd ? path.join(cwd, outputPath) : outputPath;
-    if (fs.existsSync(fullOutputPath)) {
+    const subprocess = spawnCallback(
-      fs.rmSync(fullOutputPath, {
+      `elm-optimize-level-2`,
-        force: true /* ignore errors if file doesn't exist */,
+      [elmEntrypointPath, "--output", outputPath],
-      });
+      {
-    }
+        // ignore stdout
-    const subprocess = runElm(options, elmEntrypointPath, outputPath, cwd);
+        // stdio: ["inherit", "ignore", "inherit"],
 
+        cwd: cwd,
+      }
+    );
+    let commandOutput = "";
+
+    subprocess.stderr.on("data", function (data) {
+      commandOutput += data;
+    });
+
+    subprocess.on("exit", async (code) => {
+      if (code !== 0) {
+        process.exitCode = 1;
+        reject(commandOutput);
+      }
+    });
     subprocess.on("close", async (code) => {
-      const fileOutputExists = await fs.exists(fullOutputPath);
+      if (code == 0 && (await fs.fileExists(fullOutputPath))) {
-      if (code == 0 && fileOutputExists) {
         resolve();
       } else {
         process.exitCode = 1;
@@ -202,30 +180,55 @@ function spawnElmMake(options, elmEntrypointPath, outputPath, cwd) {
  * @param {string} outputPath
  * @param {string} cwd
  */
-function runElm(options, elmEntrypointPath, outputPath, cwd) {
+async function spawnElmMake(options, elmEntrypointPath, outputPath, cwd) {
   if (options.debug) {
-    console.log("Running elm make");
+    await runElmMake(elmEntrypointPath, outputPath, cwd);
-    return spawnCallback(
-      `elm`,
-      ["make", elmEntrypointPath, "--output", outputPath, "--debug"],
-      {
-        // ignore stdout
-        stdio: ["inherit", "ignore", "inherit"],
-        cwd: cwd,
-      }
-    );
   } else {
-    console.log("Running elm-optimize-level-2");
+    await elmOptimizeLevel2(elmEntrypointPath, outputPath, cwd);
-    return spawnCallback(
+  }
-      `elm-optimize-level-2`,
+}
-      [elmEntrypointPath, "--output", outputPath],
+
+function runElmMake(elmEntrypointPath, outputPath, cwd) {
+  return new Promise(async (resolve, reject) => {
+    const subprocess = spawnCallback(
+      `elm`,
+      [
+        "make",
+        elmEntrypointPath,
+        "--output",
+        outputPath,
+        "--debug",
+        "--report",
+        "json",
+      ],
       {
         // ignore stdout
-        stdio: ["inherit", "ignore", "inherit"],
+        // stdio: ["inherit", "ignore", "inherit"],
+
         cwd: cwd,
       }
     );
+    const fullOutputPath = cwd ? path.join(cwd, outputPath) : outputPath;
+    if (await fs.fileExists(fullOutputPath)) {
+      await fsPromises.unlink(fullOutputPath, {
+        force: true /* ignore errors if file doesn't exist */,
+      });
     }
+    let commandOutput = "";
+
+    subprocess.stderr.on("data", function (data) {
+      commandOutput += data;
+    });
+
+    subprocess.on("close", async (code) => {
+      if (code == 0 && (await fs.fileExists(fullOutputPath))) {
+        resolve();
+      } else {
+        process.exitCode = 1;
+        reject(restoreColor(JSON.parse(commandOutput).errors));
+      }
+    });
+  });
 }
 
 /**
@@ -234,7 +237,7 @@ function runElm(options, elmEntrypointPath, outputPath, cwd) {
 async function runTerser(filePath) {
   console.log("Running terser");
   const minifiedElm = await terser.minify(
-    (await fs.readFile(filePath)).toString(),
+    (await fsPromises.readFile(filePath)).toString(),
     {
       ecma: 5,
 
@@ -268,16 +271,18 @@ async function runTerser(filePath) {
     }
   );
   if (minifiedElm.code) {
-    await fs.writeFile(filePath, minifiedElm.code);
+    await fsPromises.writeFile(filePath, minifiedElm.code);
   } else {
     throw "Error running terser.";
   }
 }
 
 async function copyAssets() {
-  fs.writeFile(
+  await fsPromises.writeFile(
     "dist/elm-pages.js",
-    fs.readFileSync(path.join(__dirname, "../static-code/elm-pages.js"))
+    await fsPromises.readFile(
+      path.join(__dirname, "../static-code/elm-pages.js")
+    )
   );
   fs.copyDirFlat("public", "dist");
 }
@@ -290,13 +295,25 @@ async function compileCliApp(options) {
     "./elm-stuff/elm-pages"
   );
 
-  const elmFileContent = await fs.readFile(ELM_FILE_PATH, "utf-8");
+  const elmFileContent = await fsPromises.readFile(ELM_FILE_PATH, "utf-8");
-  await fs.writeFile(
+  await fsPromises.writeFile(
     ELM_FILE_PATH,
-    elmFileContent.replace(
+    elmFileContent
+      .replace(
         /return \$elm\$json\$Json\$Encode\$string\(.REPLACE_ME_WITH_JSON_STRINGIFY.\)/g,
         "return " + (options.debug ? "_Json_wrap(x)" : "x")
       )
+      .replace(
+        "return ports ? { ports: ports } : {};",
+        `const die = function() {
+        managers = null
+        model = null
+        stepper = null
+        ports = null
+      }
+
+      return ports ? { ports: ports, die: die } : { die: die };`
+      )
   );
 }
 
@@ -316,14 +333,3 @@ async function compileCliApp(options) {
  */
 
 module.exports = { run };
-
-/**
- * @param {string} string
- */
-function jsonOrNull(string) {
-  try {
-    return JSON.parse(string);
-  } catch (e) {
-    return { invalidJson: e.toString() };
-  }
-}

generator/src/cli.js

@@ -18,8 +18,14 @@ async function main() {
   program
     .command("build")
     .option("--debug", "Skip terser and run elm make with --debug")
+    .option(
+      "--base <basePath>",
+      "build site to be served under a base path",
+      "/"
+    )
     .description("run a full site build")
     .action(async (options) => {
+      options.base = normalizeUrl(options.base);
       await build.run(options);
     });
 
@@ -27,8 +33,9 @@ async function main() {
     .command("dev")
     .description("start a dev server")
     .option("--port <number>", "serve site at localhost:<port>", "1234")
+    .option("--base <basePath>", "serve site under a base path", "/")
     .action(async (options) => {
-      console.log({ options });
+      options.base = normalizeUrl(options.base);
       await dev.start(options);
     });
 
@@ -51,7 +58,7 @@ async function main() {
     .description("open the docs for locally generated modules")
     .option("--port <number>", "serve site at localhost:<port>", "8000")
     .action(async (options) => {
-      await codegen.generate();
+      await codegen.generate("/");
       const DocServer = require("elm-doc-preview");
       const server = new DocServer({
         port: options.port,
@@ -65,4 +72,17 @@ async function main() {
   program.parse(process.argv);
 }
 
+/**
+ * @param {string} pagePath
+ */
+function normalizeUrl(pagePath) {
+  if (!pagePath.startsWith("/")) {
+    pagePath = "/" + pagePath;
+  }
+  if (!pagePath.endsWith("/")) {
+    pagePath = pagePath + "/";
+  }
+  return pagePath;
+}
+
 main();

generator/src/codegen.js

@@ -8,64 +8,66 @@ const path = require("path");
 const { ensureDirSync, deleteIfExists } = require("./file-helpers.js");
 global.builtAt = new Date();
 
-async function generate() {
+/**
-  await writeFiles();
+ * @param {string} basePath
-}
+ */
-
+async function generate(basePath) {
-async function writeFiles() {
+  const cliCode = generateTemplateModuleConnector(basePath, "cli");
-  const cliCode = generateTemplateModuleConnector("cli");
+  const browserCode = generateTemplateModuleConnector(basePath, "browser");
-  const browserCode = generateTemplateModuleConnector("browser");
   ensureDirSync("./elm-stuff");
   ensureDirSync("./.elm-pages");
   ensureDirSync("./elm-stuff/elm-pages/.elm-pages");
 
-  fs.copyFileSync(path.join(__dirname, `./Page.elm`), `./.elm-pages/Page.elm`);
+  const uiFileContent = elmPagesUiFile();
-  fs.copyFileSync(
+  await Promise.all([
+    fs.promises.copyFile(
+      path.join(__dirname, `./Page.elm`),
+      `./.elm-pages/Page.elm`
+    ),
+    fs.promises.copyFile(
       path.join(__dirname, `./elm-application.json`),
       `./elm-stuff/elm-pages/elm-application.json`
-  );
+    ),
-  fs.copyFileSync(
+    fs.promises.copyFile(
       path.join(__dirname, `./Page.elm`),
       `./elm-stuff/elm-pages/.elm-pages/Page.elm`
-  );
+    ),
-  fs.copyFileSync(
+    fs.promises.copyFile(
       path.join(__dirname, `./SharedTemplate.elm`),
       `./.elm-pages/SharedTemplate.elm`
-  );
+    ),
-  fs.copyFileSync(
+    fs.promises.copyFile(
       path.join(__dirname, `./SharedTemplate.elm`),
       `./elm-stuff/elm-pages/.elm-pages/SharedTemplate.elm`
-  );
+    ),
-  fs.copyFileSync(
+    fs.promises.copyFile(
       path.join(__dirname, `./SiteConfig.elm`),
       `./.elm-pages/SiteConfig.elm`
-  );
+    ),
-  fs.copyFileSync(
+    fs.promises.copyFile(
       path.join(__dirname, `./SiteConfig.elm`),
       `./elm-stuff/elm-pages/.elm-pages/SiteConfig.elm`
-  );
+    ),
-
+    fs.promises.writeFile("./.elm-pages/Pages.elm", uiFileContent),
-  const uiFileContent = elmPagesUiFile();
-  fs.writeFileSync("./.elm-pages/Pages.elm", uiFileContent);
-
     // write `Pages.elm` with cli interface
-  fs.writeFileSync(
+    fs.promises.writeFile(
       "./elm-stuff/elm-pages/.elm-pages/Pages.elm",
       elmPagesCliFile()
-  );
+    ),
-  fs.writeFileSync(
+    fs.promises.writeFile(
       "./elm-stuff/elm-pages/.elm-pages/TemplateModulesBeta.elm",
       cliCode.mainModule
-  );
+    ),
-  fs.writeFileSync(
+    fs.promises.writeFile(
       "./elm-stuff/elm-pages/.elm-pages/Route.elm",
       cliCode.routesModule
-  );
+    ),
-  fs.writeFileSync(
+    fs.promises.writeFile(
       "./.elm-pages/TemplateModulesBeta.elm",
       browserCode.mainModule
-  );
+    ),
-  fs.writeFileSync("./.elm-pages/Route.elm", browserCode.routesModule);
+    fs.promises.writeFile("./.elm-pages/Route.elm", browserCode.routesModule),
+  ]);
 
   // write modified elm.json to elm-stuff/elm-pages/
   copyModifiedElmJson();

generator/src/compile-elm.js

@@ -37,7 +37,10 @@ async function spawnElmMake(elmEntrypointPath, outputPath, cwd) {
 
 async function compileElmForBrowser() {
   await runElm("./.elm-pages/TemplateModulesBeta.elm", pathToClientElm);
-  return inject(await fs.promises.readFile(pathToClientElm, "utf-8"));
+  return fs.promises.writeFile(
+    "./.elm-pages/cache/elm.js",
+    inject(await fs.promises.readFile(pathToClientElm, "utf-8"))
+  );
 }
 
 /**

generator/src/dev-server.js

@@ -1,8 +1,6 @@
 const path = require("path");
 const fs = require("fs");
 const chokidar = require("chokidar");
-const compiledElmPath = path.join(process.cwd(), "elm-stuff/elm-pages/elm.js");
-const renderer = require("../../generator/src/render");
 const { spawnElmMake, compileElmForBrowser } = require("./compile-elm.js");
 const http = require("http");
 const codegen = require("./codegen.js");
@@ -10,14 +8,26 @@ const kleur = require("kleur");
 const serveStatic = require("serve-static");
 const connect = require("connect");
 const { restoreColor } = require("./error-formatter");
-let Elm;
+const { Worker, SHARE_ENV } = require("worker_threads");
+const os = require("os");
+const { ensureDirSync } = require("./file-helpers.js");
+const baseMiddleware = require("./basepath-middleware.js");
 
+/**
+ * @param {{ port: string; base: string }} options
+ */
 async function start(options) {
+  let threadReadyQueue = [];
+  let pool = [];
+  ensureDirSync(path.join(process.cwd(), ".elm-pages", "http-response-cache"));
+  const cpuCount = os.cpus().length;
+
   const port = options.port;
-  global.staticHttpCache = {};
   let elmMakeRunning = true;
 
   const serve = serveStatic("public/", { index: false });
+  fs.mkdirSync(".elm-pages/cache", { recursive: true });
+  const serveCachedFiles = serveStatic(".elm-pages/cache", { index: false });
   const generatedFilesDirectory = "elm-stuff/elm-pages/generated-files";
   fs.mkdirSync(generatedFilesDirectory, { recursive: true });
   const serveStaticCode = serveStatic(
@@ -33,17 +43,15 @@ async function start(options) {
     ignored: [/\.swp$/],
     ignoreInitial: true,
   });
-  watchElmSourceDirs();
 
-  await codegen.generate();
+  await codegen.generate(options.base);
   let clientElmMakeProcess = compileElmForBrowser();
   let pendingCliCompile = compileCliApp();
+  watchElmSourceDirs(true);
 
   async function setup() {
-    await codegen.generate();
     await Promise.all([clientElmMakeProcess, pendingCliCompile])
       .then(() => {
-        console.log("Dev server ready");
         elmMakeRunning = false;
       })
       .catch(() => {
@@ -54,24 +62,33 @@ async function start(options) {
         `<http://localhost:${port}>`
       )}`
     );
+    const poolSize = Math.max(1, cpuCount / 2 - 1);
+    for (let index = 0; index < poolSize; index++) {
+      pool.push(initWorker(options.base));
+    }
+    runPendingWork();
   }
 
   setup();
 
-  function watchElmSourceDirs() {
+  /**
+   * @param {boolean} initialRun
+   */
+  async function watchElmSourceDirs(initialRun) {
+    if (initialRun) {
+    } else {
       console.log("elm.json changed - reloading watchers");
       watcher.removeAllListeners();
-    const sourceDirs = JSON.parse(fs.readFileSync("./elm.json").toString())[
+    }
-      "source-directories"
+    const sourceDirs = JSON.parse(
-    ];
+      (await fs.promises.readFile("./elm.json")).toString()
-    console.log("Watching...", { sourceDirs });
+    )["source-directories"].filter(
+      (sourceDir) => path.resolve(sourceDir) !== path.resolve(".elm-pages")
+    );
+
     watcher.add(sourceDirs);
     watcher.add("./public/*.css");
-  }
+    watcher.add("./port-data-source.js");
-
-  function requireUncached() {
-    delete require.cache[require.resolve(compiledElmPath)];
-    Elm = require(compiledElmPath);
   }
 
   async function compileCliApp() {
@@ -80,11 +97,13 @@ async function start(options) {
       "elm.js",
       "elm-stuff/elm-pages/"
     );
-    requireUncached();
   }
 
   const app = connect()
     .use(timeMiddleware())
+    .use(baseMiddleware(options.base))
+    .use(awaitElmMiddleware)
+    .use(serveCachedFiles)
     .use(serveStaticCode)
     .use(serve)
     .use(processRequest);
@@ -95,27 +114,17 @@ async function start(options) {
    * @param {connect.NextHandleFunction} next
    */
   async function processRequest(request, response, next) {
-    if (request.url && request.url.startsWith("/elm.js")) {
+    if (request.url && request.url.startsWith("/stream")) {
-      try {
-        await pendingCliCompile;
-        const clientElmJs = await clientElmMakeProcess;
-        response.writeHead(200, { "Content-Type": "text/javascript" });
-        response.end(clientElmJs);
-      } catch (elmCompilerError) {
-        response.writeHead(500, { "Content-Type": "application/json" });
-        response.end(elmCompilerError);
-      }
-    } else if (request.url && request.url.startsWith("/stream")) {
       handleStream(request, response);
     } else {
-      handleNavigationRequest(request, response, next);
+      await handleNavigationRequest(request, response, next);
     }
   }
 
   watcher.on("all", async function (eventName, pathThatChanged) {
-    console.log({ pathThatChanged });
+    // console.log({ pathThatChanged });
     if (pathThatChanged === "elm.json") {
-      watchElmSourceDirs();
+      watchElmSourceDirs(false);
     } else if (pathThatChanged.endsWith(".css")) {
       clients.forEach((client) => {
         client.response.write(`data: style.css\n\n`);
@@ -126,7 +135,7 @@ async function start(options) {
         let codegenError = null;
         if (needToRerunCodegen(eventName, pathThatChanged)) {
           try {
-            await codegen.generate();
+            await codegen.generate(options.base);
             clientElmMakeProcess = compileElmForBrowser();
             pendingCliCompile = compileCliApp();
 
@@ -156,7 +165,6 @@ async function start(options) {
           clientElmMakeProcess = compileElmForBrowser();
           pendingCliCompile = compileCliApp();
         }
-        let timestamp = Date.now();
 
         Promise.all([clientElmMakeProcess, pendingCliCompile])
           .then(() => {
@@ -170,22 +178,23 @@ async function start(options) {
         });
       }
     } else {
-      const changedPathRelative = path.relative(process.cwd(), pathThatChanged);
+      // TODO use similar logic in the workers? Or don't use cache at all?
-
+      // const changedPathRelative = path.relative(process.cwd(), pathThatChanged);
-      Object.keys(global.staticHttpCache).forEach((dataSourceKey) => {
+      //
-        if (dataSourceKey.includes(`file://${changedPathRelative}`)) {
+      // Object.keys(global.staticHttpCache).forEach((dataSourceKey) => {
-          delete global.staticHttpCache[dataSourceKey];
+      //   if (dataSourceKey.includes(`file://${changedPathRelative}`)) {
-        } else if (
+      //     delete global.staticHttpCache[dataSourceKey];
-          (eventName === "add" ||
+      //   } else if (
-            eventName === "unlink" ||
+      //     (eventName === "add" ||
-            eventName === "change" ||
+      //       eventName === "unlink" ||
-            eventName === "addDir" ||
+      //       eventName === "change" ||
-            eventName === "unlinkDir") &&
+      //       eventName === "addDir" ||
-          dataSourceKey.startsWith("glob://")
+      //       eventName === "unlinkDir") &&
-        ) {
+      //     dataSourceKey.startsWith("glob://")
-          delete global.staticHttpCache[dataSourceKey];
+      //   ) {
-        }
+      //     delete global.staticHttpCache[dataSourceKey];
-      });
+      //   }
+      // });
       clients.forEach((client) => {
         client.response.write(`data: content.json\n\n`);
       });
@@ -219,6 +228,55 @@ async function start(options) {
     );
   }
 
+  /**
+   * @param {string} pathname
+   * @param {((value: any) => any) | null | undefined} onOk
+   * @param {((reason: any) => PromiseLike<never>) | null | undefined} onErr
+   */
+  function runRenderThread(pathname, onOk, onErr) {
+    let cleanUpThread = () => {};
+    return new Promise(async (resolve, reject) => {
+      const readyThread = await waitForThread();
+      console.log(`Rendering ${pathname}`, readyThread.worker.threadId);
+      cleanUpThread = () => {
+        cleanUp(readyThread);
+      };
+
+      readyThread.ready = false;
+      readyThread.worker.postMessage({
+        mode: "dev-server",
+        pathname,
+      });
+      readyThread.worker.on("message", (message) => {
+        if (message.tag === "done") {
+          resolve(message.data);
+        } else if (message.tag === "watch") {
+          // console.log("@@@ WATCH", message.data);
+          message.data.forEach((pattern) => watcher.add(pattern));
+        } else if (message.tag === "error") {
+          reject(message.data);
+        } else {
+          throw `Unhandled message: ${message}`;
+        }
+      });
+      readyThread.worker.on("error", (error) => {
+        reject(error.context);
+      });
+    })
+      .then(onOk)
+      .catch(onErr)
+      .finally(() => {
+        cleanUpThread();
+      });
+  }
+
+  function cleanUp(thread) {
+    thread.worker.removeAllListeners("message");
+    thread.worker.removeAllListeners("error");
+    thread.ready = true;
+    runPendingWork();
+  }
+
   /**
    * @param {http.IncomingMessage} req
    * @param {http.ServerResponse} res
@@ -229,15 +287,9 @@ async function start(options) {
     const pathname = urlParts.pathname || "";
     try {
       await pendingCliCompile;
-      const renderResult = await renderer(
+      await runRenderThread(
-        Elm,
         pathname,
-        req,
+        function (renderResult) {
-        function (pattern) {
-          console.log(`Watching data source ${pattern}`);
-          watcher.add(pattern);
-        }
-      );
           const is404 = renderResult.is404;
           switch (renderResult.kind) {
             case "json": {
@@ -257,7 +309,9 @@ async function start(options) {
             case "api-response": {
               let mimeType = serveStatic.mime.lookup(pathname || "text/html");
               mimeType =
-            mimeType === "application/octet-stream" ? "text/html" : mimeType;
+                mimeType === "application/octet-stream"
+                  ? "text/html"
+                  : mimeType;
               res.writeHead(renderResult.statusCode, {
                 "Content-Type": mimeType,
               });
@@ -267,8 +321,10 @@ async function start(options) {
               break;
             }
           }
-    } catch (error) {
+        },
-      console.log(restoreColor(error));
+
+        function (error) {
+          console.log(restoreColor(error.errorsJson));
 
           if (req.url.includes("content.json")) {
             res.writeHead(500, { "Content-Type": "application/json" });
@@ -278,6 +334,74 @@ async function start(options) {
             res.end(errorHtml());
           }
         }
+      );
+    } catch (error) {
+      console.log(restoreColor(error.errorsJson));
+
+      if (req.url.includes("content.json")) {
+        res.writeHead(500, { "Content-Type": "application/json" });
+        res.end(JSON.stringify(error.errorsJson));
+      } else {
+        res.writeHead(500, { "Content-Type": "text/html" });
+        res.end(errorHtml());
+      }
+    }
+  }
+  async function awaitElmMiddleware(req, res, next) {
+    if (req.url && req.url.startsWith("/elm.js")) {
+      try {
+        await pendingCliCompile;
+        await clientElmMakeProcess;
+        next();
+      } catch (elmCompilerError) {
+        res.writeHead(500, { "Content-Type": "application/json" });
+        res.end(elmCompilerError);
+      }
+    } else {
+      next();
+    }
+  }
+
+  /**
+   * @returns {Promise<{ ready:boolean; worker: Worker }>}
+   * */
+  function waitForThread() {
+    return new Promise((resolve, reject) => {
+      threadReadyQueue.push(resolve);
+      runPendingWork();
+    });
+  }
+
+  function runPendingWork() {
+    const readyThreads = pool.filter((thread) => thread.ready);
+    readyThreads.forEach((readyThread) => {
+      const startTask = threadReadyQueue.shift();
+      if (startTask) {
+        // if we don't use setImmediate here, the remaining work will be done sequentially by a single worker
+        // using setImmediate delegates a ready thread to each pending task until it runs out of ready workers
+        // so the delegation is done sequentially, and the actual work is then executed
+        setImmediate(() => {
+          startTask(readyThread);
+        });
+      }
+    });
+  }
+
+  /**
+   * @param {string} basePath
+   */
+  function initWorker(basePath) {
+    let newWorker = {
+      worker: new Worker(path.join(__dirname, "./render-worker.js"), {
+        env: SHARE_ENV,
+        workerData: { basePath },
+      }),
+      ready: false,
+    };
+    newWorker.worker.once("online", () => {
+      newWorker.ready = true;
+    });
+    return newWorker;
   }
 }
 

generator/src/dir-helpers.js

@@ -2,6 +2,7 @@ const util = require("util");
 const fsSync = require("fs");
 const fs = {
   writeFile: util.promisify(fsSync.writeFile),
+  writeFileSync: fsSync.writeFileSync,
   rm: util.promisify(fsSync.unlinkSync),
   mkdir: util.promisify(fsSync.mkdir),
   readFile: util.promisify(fsSync.readFile),
@@ -21,6 +22,22 @@ async function tryMkdir(dirName) {
   }
 }
 
+function fileExists(file) {
+  return fsSync.promises
+    .access(file, fsSync.constants.F_OK)
+    .then(() => true)
+    .catch(() => false);
+}
+
+/**
+ * @param {string} filePath
+ * @param {string} data
+ */
+function writeFileSyncSafe(filePath, data) {
+  fsSync.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, data);
+}
+
 const path = require("path");
 
 /**
@@ -61,14 +78,17 @@ async function copyDirNested(src, dest) {
 
 module.exports = {
   writeFile: fs.writeFile,
+  writeFileSync: fs.writeFileSync,
   readFile: fs.readFile,
   readFileSync: fsSync.readFileSync,
   copyFile: fs.copyFile,
   exists: fs.exists,
+  writeFileSyncSafe,
   tryMkdir,
   copyDirFlat,
   copyDirNested,
   rmSync: fs.rm,
   rm: fs.rm,
   existsSync: fs.existsSync,
+  fileExists: fileExists,
 };

generator/src/error-formatter.js

@@ -24,7 +24,9 @@ function parseMsg(msg) {
   if (typeof msg === "string") {
     return msg;
   } else {
-    if (msg.underline) {
+    if (msg.underline && msg.color) {
+      return kleur[msg.color.toLowerCase()]().underline(msg.string);
+    } else if (msg.underline) {
       return kleur.underline(msg.string);
     } else if (msg.color) {
       return kleur[msg.color.toLowerCase()](msg.string);
@@ -39,11 +41,10 @@ function parseMsg(msg) {
  *
  * This function takes in the array of compiler errors and maps over them to generate a formatted compiler error
  **/
-const restoreColor = (error) => {
+const restoreColor = (errors) => {
-  console.log(error);
   try {
-    return JSON.parse(error)
+    return errors
-      .errors.map(({ problems, path }) =>
+      .map(({ problems, path }) =>
         problems.map(restoreProblem(path)).join("\n\n\n")
       )
       .join("\n\n\n\n\n");
@@ -57,9 +58,7 @@ const restoreColor = (error) => {
  *
  * This function takes in the array of compiler errors and maps over them to generate a formatted compiler error
  **/
-const restoreProblem =
+const restoreProblem = (path) => ({ title, message }) =>
-  (path) =>
-  ({ title, message }) =>
   [parseHeader(title, path), ...message.map(parseMsg)].join("");
 
 module.exports = { restoreColor };

generator/src/generate-template-module-connector.js

@@ -4,9 +4,10 @@ const mm = require("micromatch");
 const routeHelpers = require("./route-codegen-helpers");
 
 /**
+ * @param {string} basePath
  * @param {'browser' | 'cli'} phase
  */
-function generateTemplateModuleConnector(phase) {
+function generateTemplateModuleConnector(basePath, phase) {
   const templates = globby.sync(["src/Page/**/*.elm"], {}).map((file) => {
     const captures = mm.capture("src/Page/**/*.elm", file);
     if (captures) {
@@ -128,7 +129,7 @@ type PageData
 
 view :
     { path : Path
-    , frontmatter : Maybe Route
+    , route : Maybe Route
     }
     -> Maybe PageUrl
     -> Shared.Data
@@ -138,7 +139,7 @@ view :
         , head : List Head.Tag
         }
 view page maybePageUrl globalData pageData =
-    case ( page.frontmatter, pageData ) of
+    case ( page.route, pageData ) of
         ${templates
           .map(
             (name) =>
@@ -350,7 +351,7 @@ update sharedData pageData navigationKey msg model =
                                 |> (\\( a, b, c ) ->
                                         case c of
                                             Just sharedMsg ->
-                                                ( a, b, Shared.template.update (Shared.template.sharedMsg sharedMsg) model.global )
+                                                ( a, b, Shared.template.update sharedMsg model.global )
 
                                             Nothing ->
                                                 ( a, b, ( model.global, Cmd.none ) )
@@ -430,8 +431,13 @@ main =
         , fromJsPort = fromJsPort identity
         , data = dataForRoute
         , sharedData = Shared.template.data
-        , apiRoutes = \\htmlToString -> routePatterns :: manifestHandler :: Api.routes getStaticRoutes htmlToString
+        , apiRoutes = \\htmlToString -> pathsToGenerateHandler :: routePatterns :: manifestHandler :: Api.routes getStaticRoutes htmlToString
         , pathPatterns = routePatterns3
+        , basePath = [ ${basePath
+          .split("/")
+          .filter((segment) => segment !== "")
+          .map((segment) => `"${segment}"`)
+          .join(", ")} ]
         }
 
 dataForRoute : Maybe Route -> DataSource PageData
@@ -593,6 +599,37 @@ getStaticRoutes =
         |> DataSource.map List.concat
 
 
+pathsToGenerateHandler : ApiRoute.Done ApiRoute.Response
+pathsToGenerateHandler =
+    ApiRoute.succeed
+        (DataSource.map2
+            (\\pageRoutes apiRoutes ->
+                { body =
+                    (pageRoutes ++ (apiRoutes |> List.map (\\api -> "/" ++ api)))
+                        |> Json.Encode.list Json.Encode.string
+                        |> Json.Encode.encode 0
+                }
+            )
+            (DataSource.map
+                (List.map
+                    (\\route ->
+                        route
+                            |> Route.toPath
+                            |> Path.toAbsolute
+                    )
+                )
+                getStaticRoutes
+            )
+            ((manifestHandler :: Api.routes getStaticRoutes (\\_ -> ""))
+                |> List.map ApiRoute.getBuildTimeRoutes
+                |> DataSource.combine
+                |> DataSource.map List.concat
+            )
+        )
+        |> ApiRoute.literal "all-paths.json"
+        |> ApiRoute.single
+
+
 manifestHandler : ApiRoute.Done ApiRoute.Response
 manifestHandler =
     ApiRoute.succeed
@@ -655,9 +692,29 @@ type Route
 {-| -}
 urlToRoute : { url | path : String } -> Maybe Route
 urlToRoute url =
-    Router.firstMatch matchers url.path
+    url.path
+    |> withoutBaseUrl 
+    |> Router.firstMatch matchers 
 
 
+baseUrl : String
+baseUrl =
+    "${basePath}"
+
+
+baseUrlAsPath : List String
+baseUrlAsPath =
+    baseUrl
+    |> String.split "/"
+    |> List.filter (not << String.isEmpty)
+
+
+withoutBaseUrl path =
+    if (path |> String.startsWith baseUrl) then
+      String.dropLeft (String.length baseUrl) path
+    else
+      path
+
 {-| -}
 matchers : List (Router.Matcher Route)
 matchers =
@@ -711,13 +768,20 @@ routeToPath route =
 {-| -}
 toPath : Route -> Path
 toPath route =
-    route |> routeToPath |> String.join "/" |> Path.fromString
+    (baseUrlAsPath ++ (route |> routeToPath)) |> String.join "/" |> Path.fromString
+
+
+{-| -}
+toString : Route -> String
+toString route =
+    route |> toPath |> Path.toAbsolute
+
 
 {-| -}
 toLink : (List (Attribute msg) -> tag) -> Route -> tag
 toLink toAnchorTag route =
     toAnchorTag
-        [ Attr.href ("/" ++ (routeToPath route |> String.join "/"))
+        [ route |> toString |> Attr.href
         , Attr.attribute "elm-pages:prefetch" ""
         ]
 

generator/src/pre-render-html.js

@@ -1,18 +1,20 @@
 const cliVersion = require("../../package.json").version;
 const seo = require("./seo-renderer.js");
 const elmPagesJsMinified = require("./elm-pages-js-minified.js");
+const path = require("path");
 
 /** @typedef { { head: any[]; errors: any[]; contentJson: any[]; html: string; route: string; title: string; } } Arg */
 /** @typedef { { tag : 'PageProgress'; args : Arg[] } } PageProgress */
 
 module.exports =
   /**
+   * @param {string} basePath
    * @param {Arg} fromElm
    * @param {string} contentJsonString
    * @param {boolean} devServer
    * @returns {string}
    */
-  function wrapHtml(fromElm, contentJsonString, devServer) {
+  function wrapHtml(basePath, fromElm, contentJsonString, devServer) {
     const devServerOnly = (/** @type {string} */ devServerOnlyString) =>
       devServer ? devServerOnlyString : "";
     const seoData = seo.gather(fromElm.head);
@@ -20,24 +22,29 @@ module.exports =
     return `<!DOCTYPE html>
   ${seoData.rootElement}
   <head>
-    <link rel="stylesheet" href="/style.css">
+    <link rel="stylesheet" href="${path.join(basePath, "style.css")}">
     ${devServerOnly(devServerStyleTag())}
-    <link rel="preload" href="/elm.js" as="script">
+    <link rel="preload" href="${path.join(basePath, "elm.js")}" as="script">
-    <link rel="modulepreload" href="/index.js">
+    <link rel="modulepreload" href="${path.join(basePath, "index.js")}">
     ${devServerOnly(
-      /* html */ `<script defer="defer" src="/hmr.js" type="text/javascript"></script>`
+      /* html */ `<script defer="defer" src="${path.join(
+        basePath,
+        "hmr.js"
+      )}" type="text/javascript"></script>`
     )}
-    <script defer="defer" src="/elm.js" type="text/javascript"></script>
+    <script defer="defer" src="${path.join(
-    <base href="${baseRoute(fromElm.route)}">
+      basePath,
+      "elm.js"
+    )}" type="text/javascript"></script>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width,initial-scale=1">
     <script type="module">
-import userInit from"/index.js";
+import userInit from"${path.join(basePath, "index.js")}";
 ${elmPagesJsMinified}
     </script>
     <title>${fromElm.title}</title>
     <meta name="generator" content="elm-pages v${cliVersion}">
-    <link rel="manifest" href="manifest.json">
+    <link rel="manifest" href="/manifest.json">
     <meta name="mobile-web-app-capable" content="yes">
     <meta name="theme-color" content="#ffffff">
     <meta name="apple-mobile-web-app-capable" content="yes">
@@ -53,14 +60,6 @@ ${elmPagesJsMinified}
   `;
   };
 
-/**
- * @param {string} route
- */
-function baseRoute(route) {
-  const cleanedRoute = cleanRoute(route);
-  return cleanedRoute === "" ? "./" : pathToRoot(route);
-}
-
 /**
  * @param {string} route
  */

generator/src/render-worker.js

@@ -0,0 +1,97 @@
+const renderer = require("../../generator/src/render");
+const path = require("path");
+const fs = require("./dir-helpers.js");
+const compiledElmPath = path.join(process.cwd(), "elm-stuff/elm-pages/elm.js");
+const { parentPort, threadId, workerData } = require("worker_threads");
+let Elm;
+
+global.staticHttpCache = {};
+
+async function run({ mode, pathname }) {
+  console.time(`${threadId} ${pathname}`);
+  try {
+    const req = null;
+    const renderResult = await renderer(
+      workerData.basePath,
+      requireElm(mode),
+      mode,
+      pathname,
+      req,
+      function (patterns) {
+        if (mode === "dev-server" && patterns.size > 0) {
+          parentPort.postMessage({ tag: "watch", data: [...patterns] });
+        }
+      }
+    );
+
+    if (mode === "dev-server") {
+      parentPort.postMessage({ tag: "done", data: renderResult });
+    } else if (mode === "build") {
+      outputString(renderResult, pathname);
+    } else {
+      throw `Unknown mode ${mode}`;
+    }
+  } catch (error) {
+    parentPort.postMessage({ tag: "error", data: error });
+  }
+  console.timeEnd(`${threadId} ${pathname}`);
+}
+
+function requireElm(mode) {
+  if (mode === "build") {
+    if (!Elm) {
+      const warnOriginal = console.warn;
+      console.warn = function () {};
+
+      Elm = require(compiledElmPath);
+      console.warn = warnOriginal;
+    }
+    return Elm;
+  } else {
+    delete require.cache[require.resolve(compiledElmPath)];
+    return require(compiledElmPath);
+  }
+}
+
+async function outputString(
+  /** @type { { kind: 'page'; data: PageProgress } | { kind: 'api'; data: Object }  } */ fromElm,
+  /** @type string */ pathname
+) {
+  switch (fromElm.kind) {
+    case "html": {
+      const args = fromElm;
+      console.log(`Pre-rendered /${args.route}`);
+      const normalizedRoute = args.route.replace(/index$/, "");
+      await fs.tryMkdir(`./dist/${normalizedRoute}`);
+      const contentJsonString = JSON.stringify({
+        is404: args.is404,
+        staticData: args.contentJson,
+        path: args.route,
+      });
+      fs.writeFileSync(`dist/${normalizedRoute}/index.html`, args.htmlString);
+      fs.writeFileSync(
+        `dist/${normalizedRoute}/content.json`,
+        contentJsonString
+      );
+      // parentPort.postMessage({ tag: "done" });
+      parentPort.postMessage({ tag: "done" });
+      break;
+    }
+    case "api-response": {
+      const body = fromElm.body;
+      console.log(`Generated ${pathname}`);
+      fs.writeFileSyncSafe(path.join("dist", pathname), body);
+      if (pathname === "/all-paths.json") {
+        parentPort.postMessage({ tag: "all-paths", data: body });
+      } else {
+        parentPort.postMessage({ tag: "done" });
+      }
+
+      break;
+    }
+  }
+}
+
+parentPort.on("message", run);
+
+/** @typedef { { tag : 'PageProgress'; args : Arg[] } } PageProgress */

generator/src/render.js

@@ -5,25 +5,45 @@ const matter = require("gray-matter");
 const globby = require("globby");
 const fsPromises = require("fs").promises;
 const preRenderHtml = require("./pre-render-html.js");
+const { lookupOrPerform } = require("./request-cache.js");
+const kleur = require("kleur");
+kleur.enabled = true;
 
-let foundErrors = false;
 process.on("unhandledRejection", (error) => {
   console.error(error);
 });
+let foundErrors;
+let pendingDataSourceResponses;
+let pendingDataSourceCount;
 
 module.exports =
   /**
    *
+   * @param {string} basePath
    * @param {Object} elmModule
    * @param {string} path
    * @param {import('aws-lambda').APIGatewayProxyEvent} request
    * @param {(pattern: string) => void} addDataSourceWatcher
    * @returns
    */
-  async function run(elmModule, path, request, addDataSourceWatcher) {
+  async function run(
-    XMLHttpRequest = require("xhr2");
+    basePath,
-    const result = await runElmApp(
     elmModule,
+    mode,
+    path,
+    request,
+    addDataSourceWatcher
+  ) {
+    foundErrors = false;
+    pendingDataSourceResponses = [];
+    pendingDataSourceCount = 0;
+    // since init/update are never called in pre-renders, and DataSource.Http is called using undici
+    // we can provide a fake HTTP instead of xhr2 (which is otherwise needed for Elm HTTP requests from Node)
+    XMLHttpRequest = {};
+    const result = await runElmApp(
+      basePath,
+      elmModule,
+      mode,
       path,
       request,
       addDataSourceWatcher
@@ -32,27 +52,36 @@ module.exports =
   };
 
 /**
+ * @param {string} basePath
  * @param {Object} elmModule
  * @param {string} pagePath
+ * @param {string} mode
  * @param {import('aws-lambda').APIGatewayProxyEvent} request
  * @param {(pattern: string) => void} addDataSourceWatcher
  * @returns {Promise<({is404: boolean} & ( { kind: 'json'; contentJson: string} | { kind: 'html'; htmlString: string } | { kind: 'api-response'; body: string; }) )>}
  */
-function runElmApp(elmModule, pagePath, request, addDataSourceWatcher) {
+function runElmApp(
+  basePath,
+  elmModule,
+  mode,
+  pagePath,
+  request,
+  addDataSourceWatcher
+) {
+  const isDevServer = mode !== "build";
+  let patternsToWatch = new Set();
   let app = null;
   let killApp;
   return new Promise((resolve, reject) => {
     const isJson = pagePath.match(/content\.json\/?$/);
     const route = pagePath.replace(/content\.json\/?$/, "");
 
-    const mode = "elm-to-html-beta";
     const modifiedRequest = { ...request, path: route };
     // console.log("StaticHttp cache keys", Object.keys(global.staticHttpCache));
     app = elmModule.Elm.TemplateModulesBeta.init({
       flags: {
         secrets: process.env,
-        mode,
+        staticHttpCache: global.staticHttpCache || {},
-        staticHttpCache: global.staticHttpCache,
         request: {
           payload: modifiedRequest,
           kind: "single-page",
@@ -62,7 +91,7 @@ function runElmApp(elmModule, pagePath, request, addDataSourceWatcher) {
     });
 
     killApp = () => {
-      // app.ports.toJsPort.unsubscribe(portHandler);
+      app.ports.toJsPort.unsubscribe(portHandler);
       app.die();
       app = null;
       // delete require.cache[require.resolve(compiledElmPath)];
@@ -71,13 +100,11 @@ function runElmApp(elmModule, pagePath, request, addDataSourceWatcher) {
     async function portHandler(/** @type { FromElm }  */ fromElm) {
       if (fromElm.command === "log") {
         console.log(fromElm.value);
-      } else if (fromElm.tag === "InitialData") {
-        const args = fromElm.args[0];
-        // console.log(`InitialData`, args);
-        writeGeneratedFiles(args.filesToGenerate);
       } else if (fromElm.tag === "ApiResponse") {
         const args = fromElm.args[0];
+        if (mode === "build") {
           global.staticHttpCache = args.staticHttpCache;
+        }
 
         resolve({
           kind: "api-response",
@@ -87,9 +114,10 @@ function runElmApp(elmModule, pagePath, request, addDataSourceWatcher) {
         });
       } else if (fromElm.tag === "PageProgress") {
         const args = fromElm.args[0];
+        if (mode === "build") {
           global.staticHttpCache = args.staticHttpCache;
+        }
 
-        // delete require.cache[require.resolve(compiledElmPath)];
         if (isJson) {
           resolve({
             kind: "json",
@@ -100,50 +128,29 @@ function runElmApp(elmModule, pagePath, request, addDataSourceWatcher) {
             }),
           });
         } else {
-          resolve(outputString(fromElm));
+          resolve(outputString(basePath, fromElm, isDevServer));
         }
       } else if (fromElm.tag === "ReadFile") {
         const filePath = fromElm.args[0];
         try {
-          addDataSourceWatcher(filePath);
+          patternsToWatch.add(filePath);
 
-          const fileContents = (
+          runJob(app, filePath);
-            await fsPromises.readFile(path.join(process.cwd(), filePath))
-          ).toString();
-          const parsedFile = matter(fileContents);
-          app.ports.fromJsPort.send({
-            tag: "GotFile",
-            data: {
-              filePath,
-              parsedFrontmatter: parsedFile.data,
-              withoutFrontmatter: parsedFile.content,
-              rawFile: fileContents,
-              jsonFile: jsonOrNull(fileContents),
-            },
-          });
         } catch (error) {
-          app.ports.fromJsPort.send({
+          sendError(app, {
-            tag: "BuildError",
+            title: "DataSource.File Error",
-            data: { filePath },
+            message: `A DataSource.File read failed because I couldn't find this file: ${kleur.yellow(
+              filePath
+            )}`,
           });
         }
+      } else if (fromElm.tag === "DoHttp") {
+        const requestToPerform = fromElm.args[0];
+        runHttpJob(app, mode, requestToPerform);
       } else if (fromElm.tag === "Glob") {
         const globPattern = fromElm.args[0];
-        addDataSourceWatcher(globPattern);
+        patternsToWatch.add(globPattern);
-        const matchedPaths = await globby(globPattern);
+        runGlobJob(app, globPattern);
-
-        app.ports.fromJsPort.send({
-          tag: "GotGlob",
-          data: { pattern: globPattern, result: matchedPaths },
-        });
-      } else if (fromElm.tag === "Port") {
-        const portName = fromElm.args[0];
-        console.log({ portName });
-
-        app.ports.fromJsPort.send({
-          tag: "GotPort",
-          data: { portName, portResponse: "Hello from ports!" },
-        });
       } else if (fromElm.tag === "Errors") {
         foundErrors = true;
         reject(fromElm.args[0]);
@@ -153,23 +160,34 @@ function runElmApp(elmModule, pagePath, request, addDataSourceWatcher) {
     }
     app.ports.toJsPort.subscribe(portHandler);
   }).finally(() => {
+    addDataSourceWatcher(patternsToWatch);
     killApp();
     killApp = null;
   });
 }
-
+/**
-async function outputString(/** @type { PageProgress } */ fromElm) {
+ * @param {string} basePath
+ * @param {PageProgress} fromElm
+ * @param {boolean} isDevServer
+ */
+async function outputString(
+  basePath,
+  /** @type { PageProgress } */ fromElm,
+  isDevServer
+) {
   const args = fromElm.args[0];
   let contentJson = {};
   contentJson["staticData"] = args.contentJson;
   contentJson["is404"] = args.is404;
+  contentJson["path"] = args.route;
   const normalizedRoute = args.route.replace(/index$/, "");
   const contentJsonString = JSON.stringify(contentJson);
 
   return {
     is404: args.is404,
     route: normalizedRoute,
-    htmlString: preRenderHtml(args, contentJsonString, true),
+    htmlString: preRenderHtml(basePath, args, contentJsonString, isDevServer),
+    contentJson: args.contentJson,
     kind: "html",
   };
 }
@@ -193,3 +211,206 @@ function jsonOrNull(string) {
     return { invalidJson: e.toString() };
   }
 }
+
+async function runJob(app, filePath) {
+  pendingDataSourceCount += 1;
+  try {
+    const fileContents = (
+      await fsPromises.readFile(path.join(process.cwd(), filePath))
+    ).toString();
+    const parsedFile = matter(fileContents);
+
+    pendingDataSourceResponses.push({
+      request: {
+        masked: {
+          url: `file://${filePath}`,
+          method: "GET",
+          headers: [],
+          body: { tag: "EmptyBody", args: [] },
+        },
+        unmasked: {
+          url: `file://${filePath}`,
+          method: "GET",
+          headers: [],
+          body: { tag: "EmptyBody", args: [] },
+        },
+      },
+      response: JSON.stringify({
+        parsedFrontmatter: parsedFile.data,
+        withoutFrontmatter: parsedFile.content,
+        rawFile: fileContents,
+        jsonFile: jsonOrNull(fileContents),
+      }),
+    });
+  } catch (e) {
+    sendError(app, {
+      title: "Error reading file",
+      message: `A DataSource.File read failed because I couldn't find this file: ${kleur.yellow(
+        filePath
+      )}`,
+    });
+  } finally {
+    pendingDataSourceCount -= 1;
+    flushIfDone(app);
+  }
+}
+
+async function runHttpJob(app, mode, requestToPerform) {
+  pendingDataSourceCount += 1;
+  try {
+    const responseFilePath = await lookupOrPerform(
+      mode,
+      requestToPerform.unmasked
+    );
+
+    pendingDataSourceResponses.push({
+      request: requestToPerform,
+      response: (
+        await fsPromises.readFile(responseFilePath, "utf8")
+      ).toString(),
+    });
+  } catch (error) {
+    sendError(app, error);
+  } finally {
+    pendingDataSourceCount -= 1;
+    flushIfDone(app);
+  }
+}
+
+async function runGlobJob(app, globPattern) {
+  try {
+    // if (pendingDataSourceCount > 0) {
+    //   console.log(`Waiting for ${pendingDataSourceCount} pending data sources`);
+    // }
+    pendingDataSourceCount += 1;
+
+    pendingDataSourceResponses.push(await globTask(globPattern));
+  } catch (error) {
+    console.log(`Error running glob pattern ${globPattern}`);
+    throw error;
+  } finally {
+    pendingDataSourceCount -= 1;
+    flushIfDone(app);
+  }
+}
+
+function flushIfDone(app) {
+  if (foundErrors) {
+    pendingDataSourceResponses = [];
+  } else if (pendingDataSourceCount === 0) {
+    // console.log(
+    //   `Flushing ${pendingDataSourceResponses.length} items in ${timeUntilThreshold}ms`
+    // );
+
+    flushQueue(app);
+  }
+}
+
+function flushQueue(app) {
+  const temp = pendingDataSourceResponses;
+  pendingDataSourceResponses = [];
+  // console.log("@@@ FLUSHING", temp.length);
+  app.ports.fromJsPort.send({
+    tag: "GotBatch",
+    data: temp,
+  });
+}
+
+/**
+ * @param {string} filePath
+ * @returns {Promise<Object>}
+ */
+async function readFileTask(app, filePath) {
+  // console.log(`Read file ${filePath}`);
+  try {
+    const fileContents = (
+      await fsPromises.readFile(path.join(process.cwd(), filePath))
+    ).toString();
+    // console.log(`DONE reading file ${filePath}`);
+    const parsedFile = matter(fileContents);
+
+    return {
+      request: {
+        masked: {
+          url: `file://${filePath}`,
+          method: "GET",
+          headers: [],
+          body: { tag: "EmptyBody", args: [] },
+        },
+        unmasked: {
+          url: `file://${filePath}`,
+          method: "GET",
+          headers: [],
+          body: { tag: "EmptyBody", args: [] },
+        },
+      },
+      response: JSON.stringify({
+        parsedFrontmatter: parsedFile.data,
+        withoutFrontmatter: parsedFile.content,
+        rawFile: fileContents,
+        jsonFile: jsonOrNull(fileContents),
+      }),
+    };
+  } catch (e) {
+    sendError(app, {
+      title: "Error reading file",
+      message: `A DataSource.File read failed because I couldn't find this file: ${kleur.yellow(
+        filePath
+      )}`,
+    });
+  }
+}
+
+/**
+ * @param {string} globPattern
+ * @returns {Promise<Object>}
+ */
+async function globTask(globPattern) {
+  try {
+    const matchedPaths = await globby(globPattern);
+    // console.log("Got glob path", matchedPaths);
+
+    return {
+      request: {
+        masked: {
+          url: `glob://${globPattern}`,
+          method: "GET",
+          headers: [],
+          body: { tag: "EmptyBody", args: [] },
+        },
+        unmasked: {
+          url: `glob://${globPattern}`,
+          method: "GET",
+          headers: [],
+          body: { tag: "EmptyBody", args: [] },
+        },
+      },
+      response: JSON.stringify(matchedPaths),
+    };
+  } catch (e) {
+    console.log(`Error performing glob '${globPattern}'`);
+    throw e;
+  }
+}
+
+function requireUncached(mode, filePath) {
+  if (mode === "dev-server") {
+    // for the build command, we can skip clearing the cache because it won't change while the build is running
+    // in the dev server, we want to clear the cache to get a the latest code each time it runs
+    delete require.cache[require.resolve(filePath)];
+  }
+  return require(filePath);
+}
+
+/**
+ * @param {{ ports: { fromJsPort: { send: (arg0: { tag: string; data: any; }) => void; }; }; }} app
+ * @param {{ message: string; title: string; }} error
+ */
+function sendError(app, error) {
+  foundErrors = true;
+
+  app.ports.fromJsPort.send({
+    tag: "BuildError",
+    data: error,
+  });
+}

generator/src/request-cache.js

@@ -0,0 +1,180 @@
+const path = require("path");
+const undici = require("undici");
+const fs = require("fs");
+const objectHash = require("object-hash");
+const kleur = require("kleur");
+
+/**
+ * To cache HTTP requests on disk with quick lookup and insertion, we store the hashed request.
+ * This uses SHA1 hashes. They are uni-directional hashes, which works for this use case. Most importantly,
+ * they're unique enough and can be expressed in a case-insensitive way so it works on Windows filesystems.
+ * And they are 40 hex characters, so the length won't be too long no matter what the request payload.
+ * @param {Object} request
+ */
+function requestToString(request) {
+  return objectHash(request);
+}
+/**
+ * @param {Object} request
+ */
+function fullPath(request) {
+  return path.join(
+    process.cwd(),
+    ".elm-pages",
+    "http-response-cache",
+    requestToString(request)
+  );
+}
+
+/**
+ * @param {string} mode
+ * @param {{url: string; headers: {[x: string]: string}; method: string; body: Body } } rawRequest
+ * @returns {Promise<string>}
+ */
+function lookupOrPerform(mode, rawRequest) {
+  return new Promise(async (resolve, reject) => {
+    const request = toRequest(rawRequest);
+    const responsePath = fullPath(request);
+
+    if (fs.existsSync(responsePath)) {
+      // console.log("Skipping request, found file.");
+      resolve(responsePath);
+    } else {
+      let portDataSource = {};
+      let portDataSourceFound = false;
+      try {
+        portDataSource = requireUncached(
+          mode,
+          path.join(process.cwd(), "port-data-source.js")
+        );
+        portDataSourceFound = true;
+      } catch (e) {}
+
+      if (request.url.startsWith("port://")) {
+        try {
+          const portName = request.url.replace(/^port:\/\//, "");
+          // console.time(JSON.stringify(request.url));
+          if (!portDataSource[portName]) {
+            if (portDataSourceFound) {
+              throw `DataSource.Port.send "${portName}" is not defined. Be sure to export a function with that name from port-data-source.js`;
+            } else {
+              throw `DataSource.Port.send "${portName}" was called, but I couldn't find the port definitions file 'port-data-source.js'.`;
+            }
+          } else if (typeof portDataSource[portName] !== "function") {
+            throw `DataSource.Port.send "${portName}" is not a function. Be sure to export a function with that name from port-data-source.js`;
+          }
+          await fs.promises.writeFile(
+            responsePath,
+            JSON.stringify(
+              await portDataSource[portName](rawRequest.body.args[0])
+            )
+          );
+          resolve(responsePath);
+
+          // console.timeEnd(JSON.stringify(requestToPerform.masked));
+        } catch (error) {
+          console.trace(error);
+          reject({
+            title: "DataSource.Port Error",
+            message: error.toString(),
+          });
+        }
+      } else {
+        undici
+          .stream(
+            request.url,
+            {
+              method: request.method,
+              body: request.body,
+              headers: {
+                "User-Agent": "request",
+                ...request.headers,
+              },
+            },
+            (response) => {
+              const writeStream = fs.createWriteStream(responsePath);
+              writeStream.on("finish", async () => {
+                resolve(responsePath);
+              });
+
+              return writeStream;
+            }
+          )
+          .catch((error) => {
+            let errorMessage = error.toString();
+            if (error.code === "ENOTFOUND") {
+              errorMessage = `Could not reach URL.`;
+            }
+            reject({
+              title: "DataSource.Http Error",
+              message: `${kleur
+                .yellow()
+                .underline(request.url)} ${errorMessage}`,
+            });
+          });
+      }
+    }
+  });
+}
+
+/**
+ * @param {{url: string; headers: {[x: string]: string}; method: string; body: Body } } elmRequest
+ */
+function toRequest(elmRequest) {
+  const elmHeaders = Object.fromEntries(elmRequest.headers);
+  let contentType = toContentType(elmRequest.body);
+  let headers = { ...contentType, ...elmHeaders };
+  return {
+    url: elmRequest.url,
+    method: elmRequest.method,
+    headers,
+    body: toBody(elmRequest.body),
+  };
+}
+
+/**
+ * @param {Body} body
+ */
+function toBody(body) {
+  switch (body.tag) {
+    case "EmptyBody": {
+      return null;
+    }
+    case "StringBody": {
+      return body.args[1];
+    }
+    case "JsonBody": {
+      return JSON.stringify(body.args[0]);
+    }
+  }
+}
+
+/**
+ * @param {Body} body
+ * @returns Object
+ */
+function toContentType(body) {
+  switch (body.tag) {
+    case "EmptyBody": {
+      return {};
+    }
+    case "StringBody": {
+      return { "Content-Type": body.args[0] };
+    }
+    case "JsonBody": {
+      return { "Content-Type": "application/json" };
+    }
+  }
+}
+
+/** @typedef { { tag: 'EmptyBody'} | { tag: 'StringBody'; args: [string, string] } | {tag: 'JsonBody'; args: [ Object ] } } Body  */
+function requireUncached(mode, filePath) {
+  if (mode === "dev-server") {
+    // for the build command, we can skip clearing the cache because it won't change while the build is running
+    // in the dev server, we want to clear the cache to get a the latest code each time it runs
+    delete require.cache[require.resolve(filePath)];
+  }
+  return require(filePath);
+}
+
+module.exports = { lookupOrPerform };

generator/src/rewrite-elm-json.js

@@ -1,10 +1,13 @@
 const fs = require("fs");
 
-module.exports = function () {
+module.exports = async function () {
-  var elmJson = JSON.parse(fs.readFileSync("./elm.json").toString());
+  var elmJson = JSON.parse(
+    (await fs.promises.readFile("./elm.json")).toString()
+  );
 
   // write new elm.json
-  fs.writeFileSync(
+
+  await writeFileIfChanged(
     "./elm-stuff/elm-pages/elm.json",
     JSON.stringify(rewriteElmJson(elmJson))
   );
@@ -28,3 +31,18 @@ function rewriteElmJson(elmJson) {
   elmJson["source-directories"].push(".elm-pages");
   return elmJson;
 }
+
+async function writeFileIfChanged(filePath, content) {
+  if (
+    !(await fileExists(filePath)) ||
+    (await fs.promises.readFile(filePath, "utf8")) !== content
+  ) {
+    await fs.promises.writeFile(filePath, content);
+  }
+}
+function fileExists(file) {
+  return fs.promises
+    .access(file, fs.constants.F_OK)
+    .then(() => true)
+    .catch(() => false);
+}

generator/static-code/elm-pages.js

@@ -14,7 +14,6 @@ function loadContentAndInitializeApp() {
   const app = Elm.TemplateModulesBeta.init({
     flags: {
       secrets: null,
-      baseUrl: document.baseURI,
       isPrerendering: false,
       isDevServer: false,
       isElmDebugMode: false,
@@ -45,7 +44,6 @@ function prefetchIfNeeded(/** @type {HTMLAnchorElement} */ target) {
     !prefetchedPages.includes(target.pathname)
   ) {
     prefetchedPages.push(target.pathname);
-    console.log("Preloading...", target.pathname);
     const link = document.createElement("link");
     link.setAttribute("as", "fetch");
 

generator/static-code/hmr.js

@@ -41,6 +41,7 @@ async function handleEvent(sendContentJsonPort, evt) {
     showCompiling("");
     elmJsFetch().then(thenApplyHmr);
   } else if (evt.data === "style.css") {
+    // https://stackoverflow.com/a/43161591
     const links = document.getElementsByTagName("link");
     for (var i = 0; i < links.length; i++) {
       const link = links[i];

generator/template/src/Shared.elm

@@ -6,11 +6,12 @@ import Html exposing (Html)
 import Pages.Flags
 import Pages.PageUrl exposing (PageUrl)
 import Path exposing (Path)
+import Route exposing (Route)
 import SharedTemplate exposing (SharedTemplate)
 import View exposing (View)
 
 
-template : SharedTemplate Msg Model Data SharedMsg msg
+template : SharedTemplate Msg Model Data msg
 template =
     { init = init
     , update = update
@@ -18,7 +19,6 @@ template =
     , data = data
     , subscriptions = subscriptions
     , onPageChange = Just OnPageChange
-    , sharedMsg = SharedMsg
     }
 
 
@@ -88,7 +88,7 @@ view :
     Data
     ->
         { path : Path
-        , frontmatter : route
+        , route : Maybe Route
         }
     -> Model
     -> (Msg -> msg)

package.json

@@ -22,35 +22,36 @@
   "author": "Dillon Kearns",
   "license": "BSD-3-Clause",
   "dependencies": {
-    "chokidar": "^3.5.1",
+    "chokidar": "3.5.2",
-    "commander": "^7.2.0",
+    "commander": "8.0.0",
     "connect": "^3.7.0",
     "cross-spawn": "7.0.3",
     "elm-doc-preview": "^5.0.5",
     "elm-hot": "^1.1.6",
     "elm-optimize-level-2": "^0.1.5",
     "fs-extra": "^10.0.0",
-    "globby": "^11.0.3",
+    "globby": "11.0.4",
     "gray-matter": "^4.0.3",
     "kleur": "^4.1.4",
     "micromatch": "^4.0.4",
+    "object-hash": "^2.2.0",
     "serve-static": "^1.14.1",
-    "terser": "^5.7.0",
+    "terser": "5.7.1",
-    "xhr2": "^0.2.1"
+    "undici": "4.2.1"
   },
   "devDependencies": {
     "@types/cross-spawn": "^6.0.2",
-    "@types/fs-extra": "^9.0.11",
+    "@types/fs-extra": "9.0.12",
     "@types/micromatch": "^4.0.1",
     "@types/node": "12.20.12",
-    "@types/serve-static": "^1.13.9",
+    "@types/serve-static": "1.13.10",
-    "cypress": "^7.4.0",
+    "cypress": "^8.0.0",
     "elm-review": "^2.5.3",
     "elm-test": "^0.19.1-revision7",
     "elm-tooling": "^1.3.0",
     "elm-verify-examples": "^5.0.0",
     "mocha": "^8.4.0",
-    "typescript": "^4.2.4"
+    "typescript": "4.3.5"
   },
   "files": [
     "generator/src/",

plugins/Shiki.elm

@@ -0,0 +1,102 @@
+module Shiki exposing (Highlighted, decoder, view)
+
+import Html exposing (Html)
+import Html.Attributes as Attr exposing (class)
+import Html.Lazy
+import OptimizedDecoder as Decode exposing (Decoder)
+
+
+type alias ShikiToken =
+    { content : String
+    , color : Maybe String
+    , fontStyle : Maybe ( String, String )
+    }
+
+
+type alias Highlighted =
+    { lines : List (List ShikiToken)
+    , fg : String
+    , bg : String
+    }
+
+
+decoder : Decoder Highlighted
+decoder =
+    Decode.map3 Highlighted
+        (Decode.field "tokens" (Decode.list (Decode.list shikiTokenDecoder)))
+        (Decode.field "fg" Decode.string)
+        (Decode.field "bg" Decode.string)
+
+
+shikiTokenDecoder : Decode.Decoder ShikiToken
+shikiTokenDecoder =
+    Decode.map3 ShikiToken
+        (Decode.field "content" Decode.string)
+        (Decode.optionalField "color" Decode.string)
+        (Decode.optionalField "fontStyle" fontStyleDecoder |> Decode.map (Maybe.andThen identity))
+
+
+fontStyleDecoder : Decoder (Maybe ( String, String ))
+fontStyleDecoder =
+    Decode.int
+        |> Decode.map
+            (\styleNumber ->
+                case styleNumber of
+                    1 ->
+                        Just ( "font-style", "italic" )
+
+                    2 ->
+                        Just ( "font-style", "bold" )
+
+                    4 ->
+                        Just ( "font-style", "underline" )
+
+                    _ ->
+                        Nothing
+            )
+
+
+{-| <https://github.com/shikijs/shiki/blob/2a31dc50f4fbdb9a63990ccd15e08cccc9c1566a/packages/shiki/src/renderer.ts#L16>
+-}
+view : List (Html.Attribute msg) -> Highlighted -> Html msg
+view attrs highlighted =
+    highlighted.lines
+        |> List.indexedMap
+            (\lineIndex line ->
+                let
+                    isLastLine =
+                        List.length highlighted.lines == (lineIndex + 1)
+                in
+                Html.span [ class "line" ]
+                    ((line
+                        |> List.map
+                            (\token ->
+                                Html.span
+                                    [ Attr.style "color" (token.color |> Maybe.withDefault highlighted.fg)
+                                    , token.fontStyle
+                                        |> Maybe.map
+                                            (\( key, value ) ->
+                                                Attr.style key value
+                                            )
+                                        |> Maybe.withDefault (Attr.title "")
+                                    ]
+                                    [ Html.text token.content ]
+                            )
+                     )
+                        ++ [ if isLastLine then
+                                Html.text ""
+
+                             else
+                                Html.text "\n"
+                           ]
+                    )
+            )
+        |> Html.code []
+        |> List.singleton
+        |> Html.pre
+            ([ Attr.style "background-color" highlighted.bg
+             , Attr.style "white-space" "pre-wrap"
+             , Attr.style "overflow-wrap" "break-word"
+             ]
+                ++ attrs
+            )

plugins/Timestamps.elm

@@ -0,0 +1,64 @@
+module Timestamps exposing (Timestamps, data, format)
+
+import DataSource exposing (DataSource)
+import DataSource.Port
+import DateFormat
+import Json.Encode
+import List.Extra
+import OptimizedDecoder as Decode exposing (Decoder)
+import Result.Extra
+import Time
+
+
+type alias Timestamps =
+    { updated : Time.Posix
+    , created : Time.Posix
+    }
+
+
+data : String -> DataSource Timestamps
+data filePath =
+    DataSource.Port.send "gitTimestamps"
+        (Json.Encode.string filePath)
+        (Decode.string
+            |> Decode.map (String.trim >> String.split "\n")
+            |> Decode.map (List.map secondsStringToPosix)
+            |> Decode.map Result.Extra.combine
+            |> Decode.andThen Decode.fromResult
+            |> Decode.map (firstAndLast Timestamps >> Result.fromMaybe "Error")
+            |> Decode.andThen Decode.fromResult
+        )
+
+
+firstAndLast : (a -> a -> b) -> List a -> Maybe b
+firstAndLast constructor list =
+    Maybe.map2 constructor
+        (List.head list)
+        (List.Extra.last list)
+
+
+secondsStringToPosix : String -> Result String Time.Posix
+secondsStringToPosix posixTime =
+    posixTime
+        |> String.trim
+        |> String.toInt
+        |> Maybe.map (\unixTimeInSeconds -> (unixTimeInSeconds * 1000) |> Time.millisToPosix)
+        |> Result.fromMaybe "Expected int"
+
+
+format : Time.Posix -> String
+format posix =
+    DateFormat.format
+        [ DateFormat.monthNameFull
+        , DateFormat.text " "
+        , DateFormat.dayOfMonthNumber
+        , DateFormat.text ", "
+        , DateFormat.yearNumber
+        ]
+        pacificZone
+        posix
+
+
+pacificZone : Time.Zone
+pacificZone =
+    Time.customZone (-60 * 7) []

review/elm.json

@@ -9,13 +9,13 @@
             "elm/core": "1.0.5",
             "elm/json": "1.1.3",
             "elm/project-metadata-utils": "1.0.2",
-            "jfmengels/elm-review": "2.4.2",
+            "jfmengels/elm-review": "2.5.0",
             "jfmengels/elm-review-common": "1.0.4",
             "jfmengels/elm-review-debug": "1.0.6",
-            "jfmengels/elm-review-performance": "1.0.0",
+            "jfmengels/elm-review-performance": "1.0.1",
-            "jfmengels/elm-review-unused": "1.1.11",
+            "jfmengels/elm-review-unused": "1.1.15",
             "sparksp/elm-review-imports": "1.0.1",
-            "stil4m/elm-syntax": "7.2.5"
+            "stil4m/elm-syntax": "7.2.6"
         },
         "indirect": {
             "elm/html": "1.0.0",
@@ -23,7 +23,7 @@
             "elm/random": "1.0.0",
             "elm/time": "1.0.0",
             "elm/virtual-dom": "1.0.2",
-            "elm-community/list-extra": "8.3.0",
+            "elm-community/list-extra": "8.3.1",
             "elm-explorations/test": "1.2.2",
             "miniBill/elm-unicode": "1.0.2",
             "rtfeldman/elm-hex": "1.0.0",

src/ApiRoute.elm

@@ -1,4 +1,7 @@
-module ApiRoute exposing (Done, Handler, Response, buildTimeRoutes, capture, int, literal, single, slash, succeed)
+module ApiRoute exposing
+    ( Done, Handler, Response, buildTimeRoutes, capture, int, literal, single, slash, succeed
+    , getBuildTimeRoutes
+    )
 
 {-|
 
@@ -104,13 +107,8 @@ slash (Handler pattern handler toString constructor) =
 
 {-| -}
 capture :
-    Handler
+    Handler (String -> a) constructor
-        (String -> a)
+    -> Handler a (String -> constructor)
-        constructor
-    ->
-        Handler
-            a
-            (String -> constructor)
 capture (Handler pattern previousHandler toString constructor) =
     Handler
         (pattern ++ "(.*)")
@@ -138,13 +136,8 @@ capture (Handler pattern previousHandler toString constructor) =
 
 {-| -}
 int :
-    Handler
+    Handler (Int -> a) constructor
-        (Int -> a)
+    -> Handler a (Int -> constructor)
-        constructor
-    ->
-        Handler
-            a
-            (Int -> constructor)
 int (Handler pattern previousHandler toString constructor) =
     Handler
         (pattern ++ "(\\d+)")
@@ -170,6 +163,11 @@ int (Handler pattern previousHandler toString constructor) =
         )
 
 
+getBuildTimeRoutes : Done response -> DataSource (List String)
+getBuildTimeRoutes (Done handler) =
+    handler.buildTimeRoutes
+
+
 
 --captureRest : Handler (List String -> a) b -> Handler a b
 --captureRest previousHandler =

src/BuildError.elm

@@ -29,7 +29,7 @@ errorToString error =
 banner : String -> List Terminal.Text
 banner title =
     [ Terminal.cyan <|
-        Terminal.text ("-- " ++ String.toUpper title ++ " ----------------------------------------------------- elm-pages")
+        ("-- " ++ String.toUpper title ++ " ----------------------------------------------------- elm-pages")
     , Terminal.text "\n\n"
     ]
 

src/DataSource.elm

@@ -418,7 +418,7 @@ resolve =
 -}
 combine : List (DataSource value) -> DataSource (List value)
 combine =
-    List.foldl (map2 (::)) (succeed [])
+    List.foldr (map2 (::)) (succeed [])
 
 
 {-| Like map, but it takes in two `Request`s.

src/DataSource/File.elm

@@ -59,7 +59,7 @@ frontmatter frontmatterDecoder =
     import DataSource.File as File
     import OptimizedDecoder as Decode exposing (Decoder)
 
-    blogPost : DataSource ( String, BlogPostMetadata )
+    blogPost : DataSource BlogPostMetadata
     blogPost =
         File.bodyWithFrontmatter blogPostDecoder
             "blog/hello-world.md"
@@ -70,7 +70,7 @@ frontmatter frontmatterDecoder =
         , tags : List String
         }
 
-    blogPostDecoder : Decoder BlogPostMetadata
+    blogPostDecoder : String -> Decoder BlogPostMetadata
     blogPostDecoder body =
         Decode.map2 (BlogPostMetadata body)
             (Decode.field "title" Decode.string)

src/DataSource/Port.elm

@@ -0,0 +1,20 @@
+module DataSource.Port exposing (send)
+
+import DataSource
+import DataSource.Http
+import Json.Encode
+import OptimizedDecoder exposing (Decoder)
+import Secrets
+
+
+send : String -> Json.Encode.Value -> Decoder b -> DataSource.DataSource b
+send portName input decoder =
+    DataSource.Http.request
+        (Secrets.succeed
+            { url = "port://" ++ portName
+            , method = "GET"
+            , headers = []
+            , body = DataSource.jsonBody input
+            }
+        )
+        decoder

src/NotFoundReason.elm

@@ -148,14 +148,17 @@ prerenderedOptionsView moduleContext routes =
                                 Html.li
                                     [ Attr.style "list-style" "inside"
                                     ]
-                                    [ Html.a
+                                    [ --Html.a
-                                        [ Attr.href "/blog/extensible-markdown-parsing-in-elm"
+                                      --    [-- Attr.href "/blog/extensible-markdown-parsing-in-elm"
-                                        ]
+                                      --     -- TODO get href data
-                                        [ Html.code
+                                      --    ]
+                                      --    [
+                                      Html.code
                                         []
                                         [ Html.text (recordToString record)
                                         ]
-                                        ]
+
+                                    --]
                                     ]
                             )
                     )

src/Pages/ContentCache.elm

@@ -53,26 +53,22 @@ type alias Path =
 
 
 init :
-    Maybe ( { currentUrl : Url, baseUrl : Url }, ContentJson )
+    Maybe ( Path, ContentJson )
     -> ContentCache
 init maybeInitialPageContent =
-    Dict.fromList []
-        |> (\dict ->
     case maybeInitialPageContent of
         Nothing ->
-                        dict
+            Dict.empty
 
         Just ( urls, contentJson ) ->
-                        dict
+            Dict.singleton urls (Parsed contentJson)
-                            |> Dict.insert (pathForUrl urls) (Parsed contentJson)
-           )
 
 
 {-| Get from the Cache... if it's not already parsed, it will
 parse it before returning it and store the parsed version in the Cache
 -}
 lazyLoad :
-    { currentUrl : Url, baseUrl : Url }
+    { currentUrl : Url, basePath : List String }
     -> ContentCache
     -> Task Http.Error ( Url, ContentJson, ContentCache )
 lazyLoad urls cache =
@@ -141,6 +137,7 @@ httpTask url =
 type alias ContentJson =
     { staticData : RequestsAndPending
     , is404 : Bool
+    , path : Maybe String
     , notFoundReason : Maybe NotFoundReason.Payload
     }
 
@@ -151,9 +148,10 @@ contentJsonDecoder =
         |> Decode.andThen
             (\is404Value ->
                 if is404Value then
-                    Decode.map3 ContentJson
+                    Decode.map4 ContentJson
                         (Decode.succeed Dict.empty)
                         (Decode.succeed is404Value)
+                        (Decode.field "path" Decode.string |> Decode.map Just)
                         (Decode.at [ "staticData", "notFoundReason" ]
                             (Decode.string
                                 |> Decode.andThen
@@ -176,16 +174,17 @@ contentJsonDecoder =
                         )
 
                 else
-                    Decode.map3 ContentJson
+                    Decode.map4 ContentJson
                         (Decode.field "staticData" RequestsAndPending.decoder)
                         (Decode.succeed is404Value)
                         (Decode.succeed Nothing)
+                        (Decode.succeed Nothing)
             )
 
 
 update :
     ContentCache
-    -> { currentUrl : Url, baseUrl : Url }
+    -> { currentUrl : Url, basePath : List String }
     -> ContentJson
     -> ContentCache
 update cache urls rawContent =
@@ -199,6 +198,7 @@ update cache urls rawContent =
                 Nothing ->
                     { staticData = rawContent.staticData
                     , is404 = rawContent.is404
+                    , path = rawContent.path
                     , notFoundReason = rawContent.notFoundReason
                     }
                         |> Parsed
@@ -207,18 +207,18 @@ update cache urls rawContent =
         cache
 
 
-pathForUrl : { currentUrl : Url, baseUrl : Url } -> Path
+pathForUrl : { currentUrl : Url, basePath : List String } -> Path
-pathForUrl { currentUrl, baseUrl } =
+pathForUrl { currentUrl, basePath } =
     currentUrl.path
-        |> String.dropLeft (String.length baseUrl.path)
         |> String.chopForwardSlashes
         |> String.split "/"
         |> List.filter ((/=) "")
+        |> List.drop (List.length basePath)
 
 
 is404 :
     ContentCache
-    -> { currentUrl : Url, baseUrl : Url }
+    -> { currentUrl : Url, basePath : List String }
     -> Bool
 is404 dict urls =
     dict
@@ -234,7 +234,7 @@ is404 dict urls =
 
 notFoundReason :
     ContentCache
-    -> { currentUrl : Url, baseUrl : Url }
+    -> { currentUrl : Url, basePath : List String }
     -> Maybe NotFoundReason.Payload
 notFoundReason dict urls =
     dict

src/Pages/Internal/Platform.elm

@@ -19,7 +19,6 @@ import Pages.ProgramConfig exposing (ProgramConfig)
 import Pages.StaticHttpRequest as StaticHttpRequest
 import Path exposing (Path)
 import QueryParams
-import RequestsAndPending exposing (RequestsAndPending)
 import Task
 import Url exposing (Url)
 
@@ -34,10 +33,10 @@ mainView :
     -> { title : String, body : Html userMsg }
 mainView config model =
     let
-        urls : { currentUrl : Url, baseUrl : Url }
+        urls : { currentUrl : Url, basePath : List String }
         urls =
             { currentUrl = model.url
-            , baseUrl = model.baseUrl
+            , basePath = config.basePath
             }
     in
     case ContentCache.notFoundReason model.contentCache urls of
@@ -49,7 +48,7 @@ mainView config model =
                 Ok pageData ->
                     (config.view
                         { path = ContentCache.pathForUrl urls |> Path.join
-                        , frontmatter = config.urlToRoute model.url
+                        , route = config.urlToRoute model.url
                         }
                         Nothing
                         pageData.sharedData
@@ -65,27 +64,15 @@ mainView config model =
                     }
 
 
-urlToPath : Url -> Url -> Path
-urlToPath url baseUrl =
-    url.path
-        |> String.dropLeft (String.length baseUrl.path)
-        |> String.chopForwardSlashes
-        |> String.split "/"
-        |> List.filter ((/=) "")
-        |> Path.join
-
-
 urlsToPagePath :
-    { currentUrl : Url
+    { currentUrl : Url, basePath : List String }
-    , baseUrl : Url
-    }
     -> Path
 urlsToPagePath urls =
     urls.currentUrl.path
-        |> String.dropLeft (String.length urls.baseUrl.path)
         |> String.chopForwardSlashes
         |> String.split "/"
         |> List.filter ((/=) "")
+        |> List.drop (List.length urls.basePath)
         |> Path.join
 
 
@@ -137,33 +124,37 @@ init config flags url key =
             ContentCache.init
                 (Maybe.map
                     (\cj ->
-                        -- TODO parse the page path to a list here
+                        ( currentPath
-                        ( urls
                         , cj
                         )
                     )
                     contentJson
                 )
 
+        currentPath : List String
+        currentPath =
+            flags
+                |> Decode.decodeValue
+                    (Decode.at [ "contentJson", "path" ]
+                        (Decode.string
+                            |> Decode.map Path.fromString
+                            |> Decode.map Path.toSegments
+                        )
+                    )
+                |> Result.mapError Decode.errorToString
+                |> Result.toMaybe
+                |> Maybe.withDefault []
+
         contentJson : Maybe ContentJson
         contentJson =
             flags
                 |> Decode.decodeValue (Decode.field "contentJson" contentJsonDecoder)
                 |> Result.toMaybe
 
-        baseUrl : Url
+        urls : { currentUrl : Url, basePath : List String }
-        baseUrl =
-            flags
-                |> Decode.decodeValue (Decode.field "baseUrl" Decode.string)
-                |> Result.toMaybe
-                |> Maybe.andThen Url.fromString
-                |> Maybe.withDefault url
-
-        urls : { currentUrl : Url, baseUrl : Url }
         urls =
-            -- @@@
+            { currentUrl = url
-            { currentUrl = url -- |> normalizeUrl baseUrl
+            , basePath = config.basePath
-            , baseUrl = baseUrl
             }
     in
     case contentJson |> Maybe.map .staticData of
@@ -231,10 +222,10 @@ init config flags url key =
                                 |> List.filterMap identity
                                 |> Cmd.batch
 
+                        initialModel : Model userModel pageData sharedData
                         initialModel =
                             { key = key
                             , url = url
-                            , baseUrl = baseUrl
                             , contentCache = contentCache
                             , pageData =
                                 Ok
@@ -243,6 +234,7 @@ init config flags url key =
                                     , userModel = userModel
                                     }
                             , ariaNavigationAnnouncement = ""
+                            , userFlags = flags
                             }
                     in
                     ( { initialModel
@@ -254,10 +246,10 @@ init config flags url key =
                 Err error ->
                     ( { key = key
                       , url = url
-                      , baseUrl = baseUrl
                       , contentCache = contentCache
                       , pageData = BuildError.errorToString error |> Err
                       , ariaNavigationAnnouncement = "Error"
+                      , userFlags = flags
                       }
                     , Cmd.none
                     )
@@ -265,10 +257,10 @@ init config flags url key =
         Nothing ->
             ( { key = key
               , url = url
-              , baseUrl = baseUrl
               , contentCache = contentCache
               , pageData = Err "TODO"
               , ariaNavigationAnnouncement = "Error"
+              , userFlags = flags
               }
             , Cmd.none
             )
@@ -288,16 +280,15 @@ type Msg userMsg
 type alias Model userModel pageData sharedData =
     { key : Browser.Navigation.Key
     , url : Url
-    , baseUrl : Url
     , contentCache : ContentCache
     , ariaNavigationAnnouncement : String
     , pageData :
-        Result
+        Result String
-            String
             { userModel : userModel
             , pageData : pageData
             , sharedData : sharedData
             }
+    , userFlags : Decode.Value
     }
 
 
@@ -335,10 +326,10 @@ update config appMsg model =
                 navigatingToSamePage =
                     (url.path == model.url.path) && (url /= model.url)
 
-                urls : { currentUrl : Url, baseUrl : Url }
+                urls : { currentUrl : Url, basePath : List String }
                 urls =
                     { currentUrl = url
-                    , baseUrl = model.baseUrl
+                    , basePath = config.basePath
                     }
             in
             if navigatingToSamePage then
@@ -368,7 +359,7 @@ update config appMsg model =
                                             { protocol = model.url.protocol
                                             , host = model.url.host
                                             , port_ = model.url.port_
-                                            , path = urlToPath url model.baseUrl
+                                            , path = urlPathToPath config urls.currentUrl
                                             , query = url.query
                                             , fragment = url.fragment
                                             , metadata = config.urlToRoute url
@@ -449,7 +440,12 @@ update config appMsg model =
                             StaticHttpRequest.resolve ApplicationType.Browser
                                 (config.data (config.urlToRoute url))
                                 contentJson.staticData
-                                |> Result.mapError (\_ -> "Http error")
+                                |> Result.mapError
+                                    (\error ->
+                                        error
+                                            |> StaticHttpRequest.toBuildError ""
+                                            |> BuildError.errorToString
+                                    )
 
                         ( userModel, userCmd ) =
                             config.update
@@ -460,7 +456,7 @@ update config appMsg model =
                                     { protocol = model.url.protocol
                                     , host = model.url.host
                                     , port_ = model.url.port_
-                                    , path = urlToPath url model.baseUrl
+                                    , path = url |> urlPathToPath config
                                     , query = url.query
                                     , fragment = url.fragment
                                     , metadata = config.urlToRoute url
@@ -484,18 +480,36 @@ update config appMsg model =
                         ]
                     )
 
-                Err _ ->
+                Err error ->
-                    -- TODO handle error
+                    {-
-                    ( { model | url = url }, Cmd.none )
+                       When there is an error loading the content.json, we are either
+                       1) in the dev server, and should show the relevant DataSource error for the page
+                          we're navigating to. This could be done more cleanly, but it's simplest to just
+                          do a fresh page load and use the code path for presenting an error for a fresh page.
+                       2) In a production app. That means we had a successful build, so there were no DataSource failures,
+                          so the app must be stale (unless it's in some unexpected state from a bug). In the future,
+                          it probably makes sense to include some sort of hash of the app version we are fetching, match
+                          it with the current version that's running, and perform this logic when we see there is a mismatch.
+                          But for now, if there is any error we do a full page load (not a single-page navigation), which
+                          gives us a fresh version of the app to make sure things are in sync.
+
+                    -}
+                    ( model
+                    , url
+                        |> Url.toString
+                        |> Browser.Navigation.load
+                    )
 
         PageScrollComplete ->
             ( model, Cmd.none )
 
         HotReloadComplete contentJson ->
             let
-                urls : { currentUrl : Url, baseUrl : Url }
+                urls : { currentUrl : Url, basePath : List String }
                 urls =
-                    { currentUrl = model.url, baseUrl = model.baseUrl }
+                    { currentUrl = model.url
+                    , basePath = config.basePath
+                    }
 
                 pageDataResult : Result BuildError pageData
                 pageDataResult =
@@ -536,7 +550,7 @@ update config appMsg model =
                                                 { protocol = model.url.protocol
                                                 , host = model.url.host
                                                 , port_ = model.url.port_
-                                                , path = urlToPath model.url model.baseUrl
+                                                , path = model.url |> urlPathToPath config
                                                 , query = model.url.query
                                                 , fragment = model.url.fragment
                                                 , metadata = config.urlToRoute model.url
@@ -554,7 +568,15 @@ update config appMsg model =
                     case updateResult of
                         Just ( userModel, userCmd ) ->
                             ( { model
-                                | contentCache = ContentCache.init (Just ( urls, contentJson ))
+                                | contentCache =
+                                    ContentCache.init
+                                        (Just
+                                            ( urls.currentUrl
+                                                |> config.urlToRoute
+                                                |> config.routeToPath
+                                            , contentJson
+                                            )
+                                        )
                                 , pageData =
                                     Ok
                                         { pageData = pageData
@@ -568,9 +590,49 @@ update config appMsg model =
                             )
 
                         Nothing ->
+                            let
+                                pagePath : Path
+                                pagePath =
+                                    urlsToPagePath urls
+
+                                userFlags : Pages.Flags.Flags
+                                userFlags =
+                                    model.userFlags
+                                        |> Decode.decodeValue
+                                            (Decode.field "userFlags" Decode.value)
+                                        |> Result.withDefault Json.Encode.null
+                                        |> Pages.Flags.BrowserFlags
+
+                                ( userModel, userCmd ) =
+                                    Just
+                                        { path =
+                                            { path = pagePath
+                                            , query = model.url.query
+                                            , fragment = model.url.fragment
+                                            }
+                                        , metadata = config.urlToRoute model.url
+                                        , pageUrl =
+                                            Just
+                                                { protocol = model.url.protocol
+                                                , host = model.url.host
+                                                , port_ = model.url.port_
+                                                , path = pagePath
+                                                , query = model.url.query |> Maybe.map QueryParams.fromString
+                                                , fragment = model.url.fragment
+                                                }
+                                        }
+                                        |> config.init userFlags sharedData pageData (Just model.key)
+                            in
                             ( { model
                                 | contentCache =
-                                    ContentCache.init (Just ( urls, contentJson ))
+                                    ContentCache.init
+                                        (Just
+                                            ( urls.currentUrl
+                                                |> config.urlToRoute
+                                                |> config.routeToPath
+                                            , contentJson
+                                            )
+                                        )
                                 , pageData =
                                     model.pageData
                                         |> Result.map
@@ -580,14 +642,27 @@ update config appMsg model =
                                                 , userModel = previousPageData.userModel
                                                 }
                                             )
+                                        |> Result.withDefault
+                                            { pageData = pageData
+                                            , sharedData = sharedData
+                                            , userModel = userModel
                                             }
-                            , Cmd.none
+                                        |> Ok
+                              }
+                            , userCmd |> Cmd.map UserMsg
                             )
 
                 Err error ->
                     ( { model
                         | contentCache =
-                            ContentCache.init (Just ( urls, contentJson ))
+                            ContentCache.init
+                                (Just
+                                    ( urls.currentUrl
+                                        |> config.urlToRoute
+                                        |> config.routeToPath
+                                    , contentJson
+                                    )
+                                )
                       }
                     , Cmd.none
                     )
@@ -609,18 +684,16 @@ application config =
         , subscriptions =
             \model ->
                 let
-                    urls : { currentUrl : Url, baseUrl : Url }
+                    urls : { currentUrl : Url }
                     urls =
-                        { currentUrl = model.url, baseUrl = model.baseUrl }
+                        { currentUrl = model.url }
-
-                    pagePath : Path
-                    pagePath =
-                        urlsToPagePath urls
                 in
                 case model.pageData of
                     Ok pageData ->
                         Sub.batch
-                            [ config.subscriptions (model.url |> config.urlToRoute) pagePath pageData.userModel
+                            [ config.subscriptions (model.url |> config.urlToRoute)
+                                (urls.currentUrl |> config.urlToRoute |> config.routeToPath |> Path.join)
+                                pageData.userModel
                                 |> Sub.map UserMsg
                             , config.fromJsPort
                                 |> Sub.map
@@ -650,3 +723,8 @@ application config =
         , onUrlChange = UrlChanged
         , onUrlRequest = LinkClicked
         }
+
+
+urlPathToPath : ProgramConfig userMsg userModel route siteData pageData sharedData -> Url -> Path
+urlPathToPath config urls =
+    urls.path |> Path.fromString

src/Pages/Internal/Platform/Cli.elm

@@ -28,11 +28,11 @@ import Pages.Flags
 import Pages.Http
 import Pages.Internal.ApplicationType as ApplicationType
 import Pages.Internal.Platform.Effect as Effect exposing (Effect)
-import Pages.Internal.Platform.Mode as Mode exposing (Mode)
 import Pages.Internal.Platform.StaticResponses as StaticResponses exposing (StaticResponses)
-import Pages.Internal.Platform.ToJsPayload as ToJsPayload exposing (ToJsSuccessPayload)
+import Pages.Internal.Platform.ToJsPayload as ToJsPayload
 import Pages.Internal.StaticHttpBody as StaticHttpBody
 import Pages.ProgramConfig exposing (ProgramConfig)
+import Pages.StaticHttp.Request
 import Pages.StaticHttpRequest as StaticHttpRequest
 import Path exposing (Path)
 import RenderRequest exposing (RenderRequest)
@@ -51,7 +51,6 @@ type alias Model route =
     , secrets : SecretsDict
     , errors : List BuildError
     , allRawResponses : Dict String (Maybe String)
-    , mode : Mode
     , pendingRequests : List { masked : RequestDetails, unmasked : RequestDetails }
     , unprocessedPages : List ( Path, route )
     , staticRoutes : Maybe (List ( Path, route ))
@@ -60,11 +59,13 @@ type alias Model route =
 
 
 type Msg
-    = GotStaticHttpResponse { request : { masked : RequestDetails, unmasked : RequestDetails }, response : Result Pages.Http.Error String }
+    = GotDataBatch
-    | GotPortResponse ( String, Decode.Value )
+        (List
-    | GotStaticFile ( String, Decode.Value )
+            { request : { masked : RequestDetails, unmasked : RequestDetails }
+            , response : String
+            }
+        )
     | GotBuildError BuildError
-    | GotGlob ( String, Decode.Value )
     | Continue
 
 
@@ -88,7 +89,7 @@ cliApplication config =
                     renderRequest : RenderRequest (Maybe route)
                     renderRequest =
                         Decode.decodeValue (RenderRequest.decoder config) flags
-                            |> Result.withDefault RenderRequest.FullBuild
+                            |> Result.withDefault RenderRequest.default
                 in
                 init renderRequest contentCache config flags
                     |> Tuple.mapSecond (perform renderRequest config config.toJsPort)
@@ -112,36 +113,36 @@ cliApplication config =
                                                 case tag of
                                                     "BuildError" ->
                                                         Decode.field "data"
-                                                            (Decode.field "filePath" Decode.string
+                                                            (Decode.map2
-                                                                |> Decode.map
+                                                                (\message title ->
-                                                                    (\filePath ->
+                                                                    { title = title
-                                                                        { title = "File not found"
+                                                                    , message = message
-                                                                        , message =
-                                                                            [ Terminal.text "A DataSource.File read failed because I couldn't find this file: "
-                                                                            , Terminal.yellow <| Terminal.text filePath
-                                                                            ]
                                                                     , fatal = True
                                                                     , path = "" -- TODO wire in current path here
                                                                     }
                                                                 )
+                                                                (Decode.field "message" Decode.string |> Decode.map Terminal.fromAnsiString)
+                                                                (Decode.field "title" Decode.string)
                                                             )
                                                             |> Decode.map GotBuildError
 
-                                                    "GotFile" ->
+                                                    "GotBatch" ->
-                                                        gotStaticFileDecoder
-                                                            |> Decode.map GotStaticFile
-
-                                                    "GotPort" ->
-                                                        gotPortDecoder
-                                                            |> Decode.map GotPortResponse
-
-                                                    "GotGlob" ->
                                                         Decode.field "data"
-                                                            (Decode.map2 Tuple.pair
+                                                            (Decode.list
-                                                                (Decode.field "pattern" Decode.string)
+                                                                (Decode.map2
-                                                                (Decode.field "result" Decode.value)
+                                                                    (\requests response ->
+                                                                        { request =
+                                                                            { masked = requests.masked
+                                                                            , unmasked = requests.unmasked
+                                                                            }
+                                                                        , response = response
+                                                                        }
                                                                     )
-                                                            |> Decode.map GotGlob
+                                                                    (Decode.field "request" requestDecoder)
+                                                                    (Decode.field "response" Decode.string)
+                                                                )
+                                                            )
+                                                            |> Decode.map GotDataBatch
 
                                                     _ ->
                                                         Decode.fail "Unhandled msg"
@@ -154,6 +155,16 @@ cliApplication config =
         }
 
 
+requestDecoder : Decode.Decoder { masked : Pages.StaticHttp.Request.Request, unmasked : Pages.StaticHttp.Request.Request }
+requestDecoder =
+    (Codec.object (\masked unmasked -> { masked = masked, unmasked = unmasked })
+        |> Codec.field "masked" .masked Pages.StaticHttp.Request.codec
+        |> Codec.field "unmasked" .unmasked Pages.StaticHttp.Request.codec
+        |> Codec.buildObject
+    )
+        |> Codec.decoder
+
+
 gotStaticFileDecoder : Decode.Decoder ( String, Decode.Value )
 gotStaticFileDecoder =
     Decode.field "data"
@@ -189,12 +200,6 @@ perform renderRequest config toJsPort effect =
         Effect.NoEffect ->
             Cmd.none
 
-        Effect.SendJsData value ->
-            value
-                |> Codec.encoder ToJsPayload.toJsCodec
-                |> toJsPort
-                |> Cmd.map never
-
         Effect.Batch list ->
             list
                 |> List.map (perform renderRequest config toJsPort)
@@ -202,17 +207,53 @@ perform renderRequest config toJsPort effect =
 
         Effect.FetchHttp ({ unmasked, masked } as requests) ->
             if unmasked.url == "$$elm-pages$$headers" then
-                Cmd.batch
+                case
-                    [ Task.succeed
-                        { request = requests
-                        , response =
                     renderRequest
                         |> RenderRequest.maybeRequestPayload
                         |> Maybe.map (Json.Encode.encode 0)
                         |> Result.fromMaybe (Pages.Http.BadUrl "$$elm-pages$$headers is only available on server-side request (not on build).")
+                of
+                    Ok okResponse ->
+                        Task.succeed
+                            [ { request = requests
+                              , response = okResponse
                               }
-                        |> Task.perform GotStaticHttpResponse
                             ]
+                            |> Task.perform GotDataBatch
+
+                    Err error ->
+                        { title = "Static HTTP Error"
+                        , message =
+                            [ Terminal.text "I got an error making an HTTP request to this URL: "
+
+                            -- TODO include HTTP method, headers, and body
+                            , Terminal.yellow requests.masked.url
+                            , Terminal.text <| Json.Encode.encode 2 <| StaticHttpBody.encode requests.masked.body
+                            , Terminal.text "\n\n"
+                            , case error of
+                                Pages.Http.BadStatus metadata body ->
+                                    Terminal.text <|
+                                        String.join "\n"
+                                            [ "Bad status: " ++ String.fromInt metadata.statusCode
+                                            , "Status message: " ++ metadata.statusText
+                                            , "Body: " ++ body
+                                            ]
+
+                                Pages.Http.BadUrl _ ->
+                                    -- TODO include HTTP method, headers, and body
+                                    Terminal.text <| "Invalid url: " ++ requests.masked.url
+
+                                Pages.Http.Timeout ->
+                                    Terminal.text "Timeout"
+
+                                Pages.Http.NetworkError ->
+                                    Terminal.text "Network error"
+                            ]
+                        , fatal = True
+                        , path = "" -- TODO wire in current path here
+                        }
+                            |> Task.succeed
+                            |> Task.perform GotBuildError
 
             else if unmasked.url |> String.startsWith "file://" then
                 let
@@ -236,53 +277,12 @@ perform renderRequest config toJsPort effect =
                     |> toJsPort
                     |> Cmd.map never
 
-            else if unmasked.url |> String.startsWith "port://" then
+            else
-                let
+                ToJsPayload.DoHttp { masked = masked, unmasked = unmasked }
-                    portName : String
-                    portName =
-                        String.dropLeft 7 unmasked.url
-                in
-                ToJsPayload.Port portName
                     |> Codec.encoder (ToJsPayload.successCodecNew2 canonicalSiteUrl "")
                     |> toJsPort
                     |> Cmd.map never
 
-            else
-                Cmd.batch
-                    [ Http.request
-                        { method = unmasked.method
-                        , url = unmasked.url
-                        , headers = unmasked.headers |> List.map (\( key, value ) -> Http.header key value)
-                        , body =
-                            case unmasked.body of
-                                StaticHttpBody.EmptyBody ->
-                                    Http.emptyBody
-
-                                StaticHttpBody.StringBody contentType string ->
-                                    Http.stringBody contentType string
-
-                                StaticHttpBody.JsonBody value ->
-                                    Http.jsonBody value
-                        , expect =
-                            Pages.Http.expectString
-                                (\response ->
-                                    GotStaticHttpResponse
-                                        { request = requests
-                                        , response = response
-                                        }
-                                )
-                        , timeout = Nothing
-                        , tracker = Nothing
-                        }
-                    , toJsPort
-                        (Json.Encode.object
-                            [ ( "command", Json.Encode.string "log" )
-                            , ( "value", Json.Encode.string ("Fetching " ++ masked.url) )
-                            ]
-                        )
-                        |> Cmd.map never
-                    ]
-
         Effect.SendSinglePage done info ->
             let
                 currentPagePath : String
@@ -326,19 +326,16 @@ perform renderRequest config toJsPort effect =
 flagsDecoder :
     Decode.Decoder
         { secrets : SecretsDict
-        , mode : Mode
         , staticHttpCache : Dict String (Maybe String)
         }
 flagsDecoder =
-    Decode.map3
+    Decode.map2
-        (\secrets mode staticHttpCache ->
+        (\secrets staticHttpCache ->
             { secrets = secrets
-            , mode = mode
             , staticHttpCache = staticHttpCache
             }
         )
         (Decode.field "secrets" SecretsDict.decoder)
-        (Decode.field "mode" Mode.modeDecoder)
         (Decode.field "staticHttpCache"
             (Decode.dict
                 (Decode.string
@@ -356,8 +353,8 @@ init :
     -> ( Model route, Effect )
 init renderRequest contentCache config flags =
     case Decode.decodeValue flagsDecoder flags of
-        Ok { secrets, mode, staticHttpCache } ->
+        Ok { secrets, staticHttpCache } ->
-            initLegacy renderRequest { secrets = secrets, mode = mode, staticHttpCache = staticHttpCache } contentCache config flags
+            initLegacy renderRequest { secrets = secrets, staticHttpCache = staticHttpCache } contentCache config flags
 
         Err error ->
             updateAndSendPortIfDone
@@ -373,7 +370,6 @@ init renderRequest contentCache config flags =
                       }
                     ]
                 , allRawResponses = Dict.empty
-                , mode = Mode.Dev
                 , pendingRequests = []
                 , unprocessedPages = []
                 , staticRoutes = Just []
@@ -383,12 +379,12 @@ init renderRequest contentCache config flags =
 
 initLegacy :
     RenderRequest route
-    -> { a | secrets : SecretsDict, mode : Mode, staticHttpCache : Dict String (Maybe String) }
+    -> { a | secrets : SecretsDict, staticHttpCache : Dict String (Maybe String) }
     -> ContentCache
     -> ProgramConfig userMsg userModel route siteData pageData sharedData
     -> Decode.Value
     -> ( Model route, Effect )
-initLegacy renderRequest { secrets, mode, staticHttpCache } contentCache config flags =
+initLegacy renderRequest { secrets, staticHttpCache } contentCache config flags =
     let
         staticResponses : StaticResponses
         staticResponses =
@@ -412,9 +408,6 @@ initLegacy renderRequest { secrets, mode, staticHttpCache } contentCache config
                             StaticResponses.renderApiRequest
                                 (DataSource.succeed [])
 
-                RenderRequest.FullBuild ->
-                    StaticResponses.init config
-
         unprocessedPages : List ( Path, route )
         unprocessedPages =
             case renderRequest of
@@ -429,9 +422,6 @@ initLegacy renderRequest { secrets, mode, staticHttpCache } contentCache config
                         RenderRequest.NotFound path ->
                             []
 
-                RenderRequest.FullBuild ->
-                    []
-
         unprocessedPagesState : Maybe (List ( Path, route ))
         unprocessedPagesState =
             case renderRequest of
@@ -446,16 +436,12 @@ initLegacy renderRequest { secrets, mode, staticHttpCache } contentCache config
                         RenderRequest.NotFound path ->
                             Just []
 
-                RenderRequest.FullBuild ->
-                    Nothing
-
         initialModel : Model route
         initialModel =
             { staticResponses = staticResponses
             , secrets = secrets
             , errors = []
             , allRawResponses = staticHttpCache
-            , mode = mode
             , pendingRequests = []
             , unprocessedPages = unprocessedPages
             , staticRoutes = unprocessedPagesState
@@ -489,103 +475,26 @@ update :
     -> ( Model route, Effect )
 update contentCache config msg model =
     case msg of
-        GotStaticHttpResponse { request, response } ->
+        GotDataBatch batch ->
             let
-                updatedModel : Model route
-                updatedModel =
-                    (case response of
-                        Ok _ ->
-                            { model
-                                | pendingRequests =
-                                    model.pendingRequests
-                                        |> List.filter (\pending -> pending /= request)
-                            }
-
-                        Err error ->
-                            { model
-                                | errors =
-                                    List.append
-                                        model.errors
-                                        [ { title = "Static HTTP Error"
-                                          , message =
-                                                [ Terminal.text "I got an error making an HTTP request to this URL: "
-
-                                                -- TODO include HTTP method, headers, and body
-                                                , Terminal.yellow <| Terminal.text request.masked.url
-                                                , Terminal.text <| Json.Encode.encode 2 <| StaticHttpBody.encode request.masked.body
-                                                , Terminal.text "\n\n"
-                                                , case error of
-                                                    Pages.Http.BadStatus metadata body ->
-                                                        Terminal.text <|
-                                                            String.join "\n"
-                                                                [ "Bad status: " ++ String.fromInt metadata.statusCode
-                                                                , "Status message: " ++ metadata.statusText
-                                                                , "Body: " ++ body
-                                                                ]
-
-                                                    Pages.Http.BadUrl _ ->
-                                                        -- TODO include HTTP method, headers, and body
-                                                        Terminal.text <| "Invalid url: " ++ request.masked.url
-
-                                                    Pages.Http.Timeout ->
-                                                        Terminal.text "Timeout"
-
-                                                    Pages.Http.NetworkError ->
-                                                        Terminal.text "Network error"
-                                                ]
-                                          , fatal = True
-                                          , path = "" -- TODO wire in current path here
-                                          }
-                                        ]
-                            }
-                    )
-                        |> StaticResponses.update
-                            -- TODO for hash pass in RequestDetails here
-                            { request = request
-                            , response = Result.mapError (\_ -> ()) response
-                            }
-            in
-            StaticResponses.nextStep config
-                updatedModel
-                Nothing
-                |> nextStepToEffect contentCache config updatedModel
-
-        GotStaticFile ( filePath, fileContent ) ->
-            let
-                --_ =
-                --    Debug.log "GotStaticFile"
-                --        { filePath = filePath
-                --        , pendingRequests = model.pendingRequests
-                --        }
-                updatedModel : Model route
                 updatedModel =
+                    (case batch of
+                        [ single ] ->
                             { model
                                 | pendingRequests =
                                     model.pendingRequests
                                         |> List.filter
                                             (\pending ->
-                                        pending.unmasked.url
+                                                pending /= single.request
-                                            == ("file://" ++ filePath)
                                             )
                             }
-                        |> StaticResponses.update
+
-                            -- TODO for hash pass in RequestDetails here
+                        _ ->
-                            { request =
+                            { model
-                                { masked =
+                                | pendingRequests = [] -- TODO is it safe to clear it entirely?
-                                    { url = "file://" ++ filePath
-                                    , method = "GET"
-                                    , headers = []
-                                    , body = StaticHttpBody.EmptyBody
-                                    }
-                                , unmasked =
-                                    { url = "file://" ++ filePath
-                                    , method = "GET"
-                                    , headers = []
-                                    , body = StaticHttpBody.EmptyBody
-                                    }
-                                }
-                            , response = Ok (Json.Encode.encode 0 fileContent)
                             }
+                    )
+                        |> StaticResponses.batchUpdate batch
             in
             StaticResponses.nextStep config
                 updatedModel
@@ -603,40 +512,6 @@ update contentCache config msg model =
                 Nothing
                 |> nextStepToEffect contentCache config updatedModel
 
-        GotGlob ( globPattern, globResult ) ->
-            let
-                updatedModel : Model route
-                updatedModel =
-                    { model
-                        | pendingRequests =
-                            model.pendingRequests
-                                |> List.filter
-                                    (\pending -> pending.unmasked.url == ("glob://" ++ globPattern))
-                    }
-                        |> StaticResponses.update
-                            -- TODO for hash pass in RequestDetails here
-                            { request =
-                                { masked =
-                                    { url = "glob://" ++ globPattern
-                                    , method = "GET"
-                                    , headers = []
-                                    , body = StaticHttpBody.EmptyBody
-                                    }
-                                , unmasked =
-                                    { url = "glob://" ++ globPattern
-                                    , method = "GET"
-                                    , headers = []
-                                    , body = StaticHttpBody.EmptyBody
-                                    }
-                                }
-                            , response = Ok (Json.Encode.encode 0 globResult)
-                            }
-            in
-            StaticResponses.nextStep config
-                updatedModel
-                Nothing
-                |> nextStepToEffect contentCache config updatedModel
-
         GotBuildError buildError ->
             let
                 updatedModel : Model route
@@ -651,40 +526,6 @@ update contentCache config msg model =
                 Nothing
                 |> nextStepToEffect contentCache config updatedModel
 
-        GotPortResponse ( portName, portResponse ) ->
-            let
-                updatedModel : Model route
-                updatedModel =
-                    { model
-                        | pendingRequests =
-                            model.pendingRequests
-                                |> List.filter
-                                    (\pending -> pending.unmasked.url == ("port://" ++ portName))
-                    }
-                        |> StaticResponses.update
-                            -- TODO for hash pass in RequestDetails here
-                            { request =
-                                { masked =
-                                    { url = "port://" ++ portName
-                                    , method = "GET"
-                                    , headers = []
-                                    , body = StaticHttpBody.EmptyBody
-                                    }
-                                , unmasked =
-                                    { url = "port://" ++ portName
-                                    , method = "GET"
-                                    , headers = []
-                                    , body = StaticHttpBody.EmptyBody
-                                    }
-                                }
-                            , response = Ok (Json.Encode.encode 0 portResponse)
-                            }
-            in
-            StaticResponses.nextStep config
-                updatedModel
-                Nothing
-                |> nextStepToEffect contentCache config updatedModel
-
 
 nextStepToEffect :
     ContentCache
@@ -767,39 +608,8 @@ nextStepToEffect contentCache config model ( updatedStaticResponsesModel, nextSt
                 )
 
         StaticResponses.Finish toJsPayload ->
-            case model.mode of
-                Mode.ElmToHtmlBeta ->
-                    let
-                        sendManifestIfNeeded : Effect
-                        sendManifestIfNeeded =
-                            if
-                                List.length model.unprocessedPages
-                                    == (model.staticRoutes
-                                            |> Maybe.map List.length
-                                            |> Maybe.withDefault -1
-                                       )
-                                    && model.maybeRequestJson
-                                    == RenderRequest.FullBuild
-                            then
             case toJsPayload of
-                                    ToJsPayload.Success value ->
+                StaticResponses.ApiResponse ->
-                                        Effect.SendSinglePage True
-                                            (ToJsPayload.InitialData
-                                                { filesToGenerate = value.filesToGenerate
-                                                }
-                                            )
-
-                                    ToJsPayload.Errors _ ->
-                                        Effect.SendJsData toJsPayload
-
-                                    ToJsPayload.ApiResponse ->
-                                        Effect.NoEffect
-
-                            else
-                                Effect.NoEffect
-                    in
-                    case toJsPayload of
-                        ToJsPayload.ApiResponse ->
                     let
                         apiResponse : Effect
                         apiResponse =
@@ -837,7 +647,7 @@ nextStepToEffect contentCache config model ( updatedStaticResponsesModel, nextSt
                                                             Err error ->
                                                                 [ error ]
                                                                     |> ToJsPayload.Errors
-                                                                            |> Effect.SendJsData
+                                                                    |> Effect.SendSinglePage True
                                                    )
 
                                         RenderRequest.Page payload ->
@@ -894,7 +704,7 @@ nextStepToEffect contentCache config model ( updatedStaticResponsesModel, nextSt
                                                                                                     , query = Nothing
                                                                                                     , fragment = Nothing
                                                                                                     }
-                                                                                                        , metadata = currentPage.frontmatter
+                                                                                                , metadata = currentPage.route
                                                                                                 , pageUrl = Nothing
                                                                                                 }
                                                                                             )
@@ -917,9 +727,9 @@ nextStepToEffect contentCache config model ( updatedStaticResponsesModel, nextSt
                                                             --    |> Maybe.withDefault Dict.empty
                                                             Dict.empty
 
-                                                                currentPage : { path : Path, frontmatter : route }
+                                                        currentPage : { path : Path, route : route }
                                                         currentPage =
-                                                                    { path = payload.path, frontmatter = config.urlToRoute currentUrl }
+                                                            { path = payload.path, route = config.urlToRoute currentUrl }
 
                                                         pageDataResult : Result BuildError pageData
                                                         pageDataResult =
@@ -961,61 +771,46 @@ nextStepToEffect contentCache config model ( updatedStaticResponsesModel, nextSt
                                                                 |> Effect.SendSinglePage False
 
                                                         Err error ->
-                                                                    [ error ] |> ToJsPayload.Errors |> Effect.SendJsData
+                                                            [ error ] |> ToJsPayload.Errors |> Effect.SendSinglePage True
 
                                                 Ok (Just notFoundReason) ->
                                                     render404Page config model payload.path notFoundReason
 
                                                 Err error ->
-                                                            [] |> ToJsPayload.Errors |> Effect.SendJsData
+                                                    [ error ] |> ToJsPayload.Errors |> Effect.SendSinglePage True
 
                                         RenderRequest.NotFound path ->
                                             render404Page config model path NotFoundReason.NoMatchingRoute
-
-                                        RenderRequest.FullBuild ->
-                                            [] |> ToJsPayload.Errors |> Effect.SendJsData
                     in
                     ( { model | staticRoutes = Just [] }
                     , apiResponse
                     )
 
-                        _ ->
+                StaticResponses.Page contentJson ->
-                            model.unprocessedPages
+                    case model.unprocessedPages |> List.head of
-                                |> List.take 1
+                        Just pageAndMetadata ->
-                                |> List.filterMap
-                                    (\pageAndMetadata ->
-                                        case toJsPayload of
-                                            ToJsPayload.Success value ->
-                                                sendSinglePageProgress value config model pageAndMetadata
-                                                    |> Just
-
-                                            ToJsPayload.Errors errors ->
-                                                errors |> ToJsPayload.Errors |> Effect.SendJsData |> Just
-
-                                            ToJsPayload.ApiResponse ->
-                                                Nothing
-                                    )
-                                |> (\cmds ->
                             ( model
-                                            |> popProcessedRequest
+                            , sendSinglePageProgress contentJson config model pageAndMetadata
-                                        , Effect.Batch
-                                            (sendManifestIfNeeded
-                                                :: cmds
-                                            )
-                                        )
                             )
 
-                _ ->
+                        Nothing ->
-                    ( model, Effect.SendJsData toJsPayload )
+                            ( model
+                            , [] |> ToJsPayload.Errors |> Effect.SendSinglePage True
+                            )
+
+                StaticResponses.Errors errors ->
+                    ( model
+                    , errors |> ToJsPayload.Errors |> Effect.SendSinglePage True
+                    )
 
 
 sendSinglePageProgress :
-    ToJsSuccessPayload
+    Dict String String
     -> ProgramConfig userMsg userModel route siteData pageData sharedData
     -> Model route
     -> ( Path, route )
     -> Effect
-sendSinglePageProgress toJsPayload config model =
+sendSinglePageProgress contentJson config model =
     \( page, route ) ->
         case model.maybeRequestJson of
             RenderRequest.SinglePage includeHtml _ _ ->
@@ -1059,7 +854,7 @@ sendSinglePageProgress toJsPayload config model =
                                                                 , query = Nothing
                                                                 , fragment = Nothing
                                                                 }
-                                                            , metadata = currentPage.frontmatter
+                                                            , metadata = currentPage.route
                                                             , pageUrl = Nothing
                                                             }
                                                         )
@@ -1085,35 +880,29 @@ sendSinglePageProgress toJsPayload config model =
                         , fragment = Nothing
                         }
 
-                    staticData : Dict String String
+                    currentPage : { path : Path, route : route }
-                    staticData =
-                        toJsPayload.pages
-                            |> Dict.get (Path.toRelative page)
-                            |> Maybe.withDefault Dict.empty
-
-                    currentPage : { path : Path, frontmatter : route }
                     currentPage =
-                        { path = page, frontmatter = config.urlToRoute currentUrl }
+                        { path = page, route = config.urlToRoute currentUrl }
 
                     pageDataResult : Result BuildError pageData
                     pageDataResult =
                         StaticHttpRequest.resolve ApplicationType.Browser
                             (config.data (config.urlToRoute currentUrl))
-                            (staticData |> Dict.map (\_ v -> Just v))
+                            (contentJson |> Dict.map (\_ v -> Just v))
                             |> Result.mapError (StaticHttpRequest.toBuildError currentUrl.path)
 
                     sharedDataResult : Result BuildError sharedData
                     sharedDataResult =
                         StaticHttpRequest.resolve ApplicationType.Browser
                             config.sharedData
-                            (staticData |> Dict.map (\_ v -> Just v))
+                            (contentJson |> Dict.map (\_ v -> Just v))
                             |> Result.mapError (StaticHttpRequest.toBuildError currentUrl.path)
 
                     siteDataResult : Result BuildError siteData
                     siteDataResult =
                         StaticHttpRequest.resolve ApplicationType.Cli
                             (config.site allRoutes |> .data)
-                            (staticData |> Dict.map (\_ v -> Just v))
+                            (contentJson |> Dict.map (\_ v -> Just v))
                             |> Result.mapError (StaticHttpRequest.toBuildError "Site.elm")
                 in
                 case Result.map3 (\a b c -> ( a, b, c )) pageFoundResult renderedResult siteDataResult of
@@ -1121,10 +910,7 @@ sendSinglePageProgress toJsPayload config model =
                         case maybeNotFoundReason of
                             Nothing ->
                                 { route = page |> Path.toRelative
-                                , contentJson =
+                                , contentJson = contentJson
-                                    toJsPayload.pages
-                                        |> Dict.get (Path.toRelative page)
-                                        |> Maybe.withDefault Dict.empty
                                 , html = rendered.view
                                 , errors = []
                                 , head = rendered.head ++ (config.site allRoutes |> .head) siteData
@@ -1132,7 +918,8 @@ sendSinglePageProgress toJsPayload config model =
                                 , staticHttpCache = model.allRawResponses |> Dict.Extra.filterMap (\_ v -> v)
                                 , is404 = False
                                 }
-                                    |> sendProgress
+                                    |> ToJsPayload.PageProgress
+                                    |> Effect.SendSinglePage True
 
                             Just notFoundReason ->
                                 render404Page config model page notFoundReason
@@ -1140,115 +927,7 @@ sendSinglePageProgress toJsPayload config model =
                     Err error ->
                         [ error ]
                             |> ToJsPayload.Errors
-                            |> Effect.SendJsData
+                            |> Effect.SendSinglePage True
-
-            RenderRequest.FullBuild ->
-                let
-                    staticData : Dict String String
-                    staticData =
-                        toJsPayload.pages
-                            |> Dict.get (Path.toRelative page)
-                            |> Maybe.withDefault Dict.empty
-
-                    currentPage : { path : Path, frontmatter : route }
-                    currentPage =
-                        { path = page, frontmatter = config.urlToRoute currentUrl }
-
-                    pageDataResult : Result BuildError pageData
-                    pageDataResult =
-                        StaticHttpRequest.resolve ApplicationType.Browser
-                            (config.data (config.urlToRoute currentUrl))
-                            (staticData |> Dict.map (\_ v -> Just v))
-                            |> Result.mapError (StaticHttpRequest.toBuildError currentUrl.path)
-
-                    sharedDataResult : Result BuildError sharedData
-                    sharedDataResult =
-                        StaticHttpRequest.resolve ApplicationType.Browser
-                            config.sharedData
-                            (staticData |> Dict.map (\_ v -> Just v))
-                            |> Result.mapError (StaticHttpRequest.toBuildError currentUrl.path)
-
-                    allRoutes : List route
-                    allRoutes =
-                        -- TODO
-                        []
-
-                    currentUrl : { protocol : Url.Protocol, host : String, port_ : Maybe Int, path : String, query : Maybe String, fragment : Maybe String }
-                    currentUrl =
-                        { protocol = Url.Https
-                        , host = config.site allRoutes |> .canonicalUrl
-                        , port_ = Nothing
-                        , path = page |> Path.toRelative
-                        , query = Nothing
-                        , fragment = Nothing
-                        }
-
-                    siteDataResult : Result BuildError siteData
-                    siteDataResult =
-                        StaticHttpRequest.resolve ApplicationType.Cli
-                            (config.site allRoutes |> .data)
-                            (staticData |> Dict.map (\_ v -> Just v))
-                            |> Result.mapError (StaticHttpRequest.toBuildError "Site.elm")
-                in
-                case Result.map3 (\a b c -> ( a, b, c )) sharedDataResult pageDataResult siteDataResult of
-                    Ok ( sharedData, pageData, siteData ) ->
-                        let
-                            pageModel : userModel
-                            pageModel =
-                                config.init
-                                    Pages.Flags.PreRenderFlags
-                                    sharedData
-                                    pageData
-                                    Nothing
-                                    (Just
-                                        { path =
-                                            { path = currentPage.path
-                                            , query = Nothing
-                                            , fragment = Nothing
-                                            }
-                                        , metadata = currentPage.frontmatter
-                                        , pageUrl = Nothing
-                                        }
-                                    )
-                                    |> Tuple.first
-
-                            viewValue : { title : String, body : Html userMsg }
-                            viewValue =
-                                (config.view currentPage Nothing sharedData pageData |> .view) pageModel
-
-                            headTags : List Head.Tag
-                            headTags =
-                                (config.view currentPage Nothing sharedData pageData |> .head)
-                                    ++ (siteData |> (config.site allRoutes |> .head))
-                        in
-                        { route = page |> Path.toRelative
-                        , contentJson =
-                            toJsPayload.pages
-                                |> Dict.get (Path.toRelative page)
-                                |> Maybe.withDefault Dict.empty
-                        , html = viewValue.body |> HtmlPrinter.htmlToString
-                        , errors = []
-                        , head = headTags
-                        , title = viewValue.title
-                        , staticHttpCache = model.allRawResponses |> Dict.Extra.filterMap (\_ v -> v)
-                        , is404 = False
-                        }
-                            |> sendProgress
-
-                    Err error ->
-                        [ error ]
-                            |> ToJsPayload.Errors
-                            |> Effect.SendJsData
-
-
-popProcessedRequest : Model route -> Model route
-popProcessedRequest model =
-    { model | unprocessedPages = List.drop 1 model.unprocessedPages }
-
-
-sendProgress : ToJsPayload.ToJsSuccessPayloadNew -> Effect
-sendProgress singlePage =
-    singlePage |> ToJsPayload.PageProgress |> Effect.SendSinglePage False
 
 
 render404Page :
@@ -1277,6 +956,7 @@ render404Page config model path notFoundReason =
                         }
                     )
               )
+            , ( "path", Path.toAbsolute path )
             ]
 
     -- TODO include the needed info for content.json?

src/Pages/Internal/Platform/Effect.elm

@@ -1,12 +1,11 @@
 module Pages.Internal.Platform.Effect exposing (Effect(..))
 
 import DataSource.Http exposing (RequestDetails)
-import Pages.Internal.Platform.ToJsPayload exposing (ToJsPayload, ToJsSuccessPayloadNewCombined)
+import Pages.Internal.Platform.ToJsPayload exposing (ToJsSuccessPayloadNewCombined)
 
 
 type Effect
     = NoEffect
-    | SendJsData ToJsPayload
     | FetchHttp { masked : RequestDetails, unmasked : RequestDetails }
     | ReadFile String
     | GetGlob String

src/Pages/Internal/Platform/Mode.elm

@@ -1,25 +0,0 @@
-module Pages.Internal.Platform.Mode exposing (Mode(..), modeDecoder)
-
-import Json.Decode as Decode exposing (Decoder)
-
-
-type Mode
-    = Prod
-    | Dev
-    | ElmToHtmlBeta
-
-
-modeDecoder : Decoder Mode
-modeDecoder =
-    Decode.string
-        |> Decode.andThen
-            (\mode ->
-                if mode == "prod" then
-                    Decode.succeed Prod
-
-                else if mode == "elm-to-html-beta" then
-                    Decode.succeed ElmToHtmlBeta
-
-                else
-                    Decode.succeed Dev
-            )

src/Pages/Internal/Platform/StaticResponses.elm

@@ -1,4 +1,4 @@
-module Pages.Internal.Platform.StaticResponses exposing (NextStep(..), StaticResponses, error, init, nextStep, renderApiRequest, renderSingleRoute, update)
+module Pages.Internal.Platform.StaticResponses exposing (FinishKind(..), NextStep(..), StaticResponses, batchUpdate, error, nextStep, renderApiRequest, renderSingleRoute)
 
 import ApiRoute
 import BuildError exposing (BuildError)
@@ -11,8 +11,6 @@ import HtmlPrinter exposing (htmlToString)
 import Internal.ApiRoute exposing (Done(..))
 import NotFoundReason exposing (NotFoundReason)
 import Pages.Internal.ApplicationType as ApplicationType
-import Pages.Internal.Platform.Mode exposing (Mode)
-import Pages.Internal.Platform.ToJsPayload as ToJsPayload exposing (ToJsPayload)
 import Pages.SiteConfig exposing (SiteConfig)
 import Pages.StaticHttp.Request as HashRequest
 import Pages.StaticHttpRequest as StaticHttpRequest
@@ -25,8 +23,7 @@ import TerminalText as Terminal
 
 
 type StaticResponses
-    = GettingInitialData StaticHttpResult
+    = ApiRequest StaticHttpResult
-    | ApiRequest StaticHttpResult
     | StaticResponses (Dict String StaticHttpResult)
     | CheckIfHandled (DataSource (Maybe NotFoundReason)) StaticHttpResult (Dict String StaticHttpResult)
 
@@ -40,32 +37,6 @@ error =
     StaticResponses Dict.empty
 
 
-init :
-    { config
-        | getStaticRoutes : DataSource (List route)
-        , site : SiteConfig route siteData
-        , data : route -> DataSource pageData
-        , sharedData : DataSource sharedData
-        , apiRoutes :
-            (Html Never -> String) -> List (ApiRoute.Done ApiRoute.Response)
-    }
-    -> StaticResponses
-init config =
-    NotFetched
-        (DataSource.map3 (\_ _ _ -> ())
-            (config.getStaticRoutes
-                |> DataSource.andThen
-                    (\resolvedRoutes ->
-                        config.site resolvedRoutes |> .data
-                    )
-            )
-            (buildTimeFilesRequest config)
-            config.sharedData
-        )
-        Dict.empty
-        |> GettingInitialData
-
-
 buildTimeFilesRequest :
     { config
         | apiRoutes :
@@ -136,10 +107,11 @@ renderApiRequest request =
         )
 
 
-update :
+batchUpdate :
+    List
         { request :
             { masked : RequestDetails, unmasked : RequestDetails }
-    , response : Result () String
+        , response : String
         }
     ->
         { model
@@ -151,23 +123,33 @@ update :
             | staticResponses : StaticResponses
             , allRawResponses : Dict String (Maybe String)
         }
-update newEntry model =
+batchUpdate newEntries model =
     let
+        newResponses =
+            newEntries
+                |> List.map
+                    (\newEntry ->
+                        ( HashRequest.hash newEntry.request.masked, newEntry.response )
+                    )
+                |> Dict.fromList
+
         updatedAllResponses : Dict String (Maybe String)
         updatedAllResponses =
-            -- @@@@@@@@@ TODO handle errors here, change Dict to have `Result` instead of `Maybe`
+            Dict.merge
-            Dict.insert
+                (\key a -> Dict.insert key (Just a))
-                (HashRequest.hash newEntry.request.masked)
+                (\key a _ -> Dict.insert key (Just a))
-                (Just <| Result.withDefault "TODO" newEntry.response)
+                (\key b -> Dict.insert key b)
+                newResponses
                 model.allRawResponses
+                Dict.empty
     in
     { model
         | allRawResponses = updatedAllResponses
     }
 
 
-encode : RequestsAndPending -> Mode -> Dict String StaticHttpResult -> Result (List BuildError) (Dict String (Dict String String))
+encode : RequestsAndPending -> Dict String StaticHttpResult -> Result (List BuildError) (Dict String (Dict String String))
-encode requestsAndPending _ staticResponses =
+encode requestsAndPending staticResponses =
     staticResponses
         |> Dict.filter
             (\key _ ->
@@ -191,7 +173,13 @@ cliDictKey =
 
 type NextStep route
     = Continue (Dict String (Maybe String)) (List { masked : RequestDetails, unmasked : RequestDetails }) (Maybe (List route))
-    | Finish ToJsPayload
+    | Finish (FinishKind route)
+
+
+type FinishKind route
+    = ApiResponse
+    | Errors (List BuildError)
+    | Page (Dict String String)
 
 
 nextStep :
@@ -209,11 +197,10 @@ nextStep :
             , secrets : SecretsDict
             , errors : List BuildError
             , allRawResponses : Dict String (Maybe String)
-            , mode : Mode
         }
     -> Maybe (List route)
     -> ( StaticResponses, NextStep route )
-nextStep config ({ mode, secrets, allRawResponses, errors } as model) maybeRoutes =
+nextStep config ({ secrets, allRawResponses, errors } as model) maybeRoutes =
     let
         staticResponses : Dict String StaticHttpResult
         staticResponses =
@@ -221,9 +208,6 @@ nextStep config ({ mode, secrets, allRawResponses, errors } as model) maybeRoute
                 StaticResponses s ->
                     s
 
-                GettingInitialData initialData ->
-                    Dict.singleton cliDictKey initialData
-
                 ApiRequest staticHttpResult ->
                     Dict.singleton cliDictKey staticHttpResult
 
@@ -240,19 +224,6 @@ nextStep config ({ mode, secrets, allRawResponses, errors } as model) maybeRoute
                 (buildTimeFilesRequest config)
                 (allRawResponses |> Dict.Extra.filterMap (\_ value -> Just value))
 
-        generatedOkayFiles : List { path : List String, content : String }
-        generatedOkayFiles =
-            generatedFiles
-                |> List.filterMap
-                    (\result ->
-                        case result of
-                            Ok ok ->
-                                Just ok
-
-                            Err _ ->
-                                Nothing
-                    )
-
         generatedFileErrors : List BuildError
         generatedFileErrors =
             generatedFiles
@@ -429,73 +400,10 @@ nextStep config ({ mode, secrets, allRawResponses, errors } as model) maybeRoute
                 ( model.staticResponses, Continue newAllRawResponses newThing maybeRoutes )
 
             Err error_ ->
-                ( model.staticResponses, Finish (ToJsPayload.Errors <| (error_ ++ failedRequests ++ errors)) )
+                ( model.staticResponses, Finish (Errors <| (error_ ++ failedRequests ++ errors)) )
 
     else
         case model.staticResponses of
-            GettingInitialData (NotFetched _ _) ->
-                let
-                    resolvedRoutes : Result StaticHttpRequest.Error (List route)
-                    resolvedRoutes =
-                        StaticHttpRequest.resolve ApplicationType.Cli
-                            (DataSource.map3
-                                (\routes _ _ ->
-                                    routes
-                                )
-                                config.getStaticRoutes
-                                (buildTimeFilesRequest config)
-                                config.sharedData
-                            )
-                            (allRawResponses |> Dict.Extra.filterMap (\_ value -> Just value))
-                in
-                case resolvedRoutes of
-                    Ok staticRoutes ->
-                        let
-                            newState : StaticResponses
-                            newState =
-                                staticRoutes
-                                    |> List.map
-                                        (\route ->
-                                            let
-                                                entry : StaticHttpResult
-                                                entry =
-                                                    NotFetched
-                                                        (DataSource.map2 (\_ _ -> ())
-                                                            config.sharedData
-                                                            (config.data route)
-                                                        )
-                                                        Dict.empty
-                                            in
-                                            ( config.routeToPath route |> String.join "/"
-                                            , entry
-                                            )
-                                        )
-                                    |> Dict.fromList
-                                    |> StaticResponses
-
-                            newThing : List { masked : RequestDetails, unmasked : RequestDetails }
-                            newThing =
-                                []
-                        in
-                        ( newState
-                        , Continue allRawResponses newThing (Just staticRoutes)
-                        )
-
-                    Err error_ ->
-                        ( model.staticResponses
-                        , Finish
-                            (ToJsPayload.Errors <|
-                                ([ StaticHttpRequest.toBuildError
-                                    -- TODO give more fine-grained error reference
-                                    "get static routes"
-                                    error_
-                                 ]
-                                    ++ failedRequests
-                                    ++ errors
-                                )
-                            )
-                        )
-
             StaticResponses _ ->
                 --let
                 --    siteStaticData =
@@ -517,29 +425,27 @@ nextStep config ({ mode, secrets, allRawResponses, errors } as model) maybeRoute
                 --
                 --    Ok okSiteStaticData ->
                 ( model.staticResponses
-                , case encode allRawResponses mode staticResponses of
+                , case encode allRawResponses staticResponses of
                     Ok encodedResponses ->
-                        ToJsPayload.toJsPayload
-                            encodedResponses
-                            generatedOkayFiles
-                            allRawResponses
-                            allErrors
                         -- TODO send all global head tags on initial call
+                        if List.length allErrors > 0 then
+                            allErrors
+                                |> Errors
+                                |> Finish
+
+                        else
+                            Page (encodedResponses |> Dict.values |> List.head |> Maybe.withDefault Dict.empty)
                                 |> Finish
 
                     Err buildErrors ->
-                        ToJsPayload.toJsPayload
-                            Dict.empty
-                            generatedOkayFiles
-                            allRawResponses
                         (allErrors ++ buildErrors)
-                            -- TODO send all global head tags on initial call
+                            |> Errors
                             |> Finish
                 )
 
             ApiRequest _ ->
                 ( model.staticResponses
-                , ToJsPayload.ApiResponse
+                , ApiResponse
                     |> Finish
                 )
 
@@ -557,14 +463,14 @@ nextStep config ({ mode, secrets, allRawResponses, errors } as model) maybeRoute
 
                     Ok (Just _) ->
                         ( StaticResponses Dict.empty
-                        , Finish ToJsPayload.ApiResponse
+                        , Finish ApiResponse
                           -- TODO should there be a new type for 404response? Or something else?
                         )
 
                     Err error_ ->
                         ( model.staticResponses
                         , Finish
-                            (ToJsPayload.Errors <|
+                            (Errors <|
                                 ([ StaticHttpRequest.toBuildError
                                     -- TODO give more fine-grained error reference
                                     "get static routes"

src/Pages/Internal/Platform/ToJsPayload.elm

@@ -1,13 +1,7 @@
 module Pages.Internal.Platform.ToJsPayload exposing
-    ( FileToGenerate
+    ( ToJsSuccessPayloadNew
-    , InitialDataRecord
-    , ToJsPayload(..)
-    , ToJsSuccessPayload
-    , ToJsSuccessPayloadNew
     , ToJsSuccessPayloadNewCombined(..)
     , successCodecNew2
-    , toJsCodec
-    , toJsPayload
     )
 
 import BuildError exposing (BuildError)
@@ -16,20 +10,7 @@ import Dict exposing (Dict)
 import Head
 import Json.Decode as Decode
 import Json.Encode
-
+import Pages.StaticHttp.Request
-
-type ToJsPayload
-    = Errors (List BuildError)
-    | Success ToJsSuccessPayload
-    | ApiResponse
-
-
-type alias ToJsSuccessPayload =
-    { pages : Dict String (Dict String String)
-    , filesToGenerate : List FileToGenerate
-    , staticHttpCache : Dict String String
-    , errors : List BuildError
-    }
 
 
 type alias ToJsSuccessPayloadNew =
@@ -44,60 +25,6 @@ type alias ToJsSuccessPayloadNew =
     }
 
 
-type alias FileToGenerate =
-    { path : List String
-    , content : String
-    }
-
-
-toJsPayload :
-    Dict String (Dict String String)
-    -> List FileToGenerate
-    -> Dict String (Maybe String)
-    -> List BuildError
-    -> ToJsPayload
-toJsPayload encodedStatic generated allRawResponses allErrors =
-    if allErrors |> List.filter .fatal |> List.isEmpty then
-        Success
-            (ToJsSuccessPayload
-                encodedStatic
-                generated
-                (allRawResponses
-                    |> Dict.toList
-                    |> List.filterMap
-                        (\( key, maybeValue ) ->
-                            maybeValue
-                                |> Maybe.map (\value -> ( key, value ))
-                        )
-                    |> Dict.fromList
-                )
-                allErrors
-            )
-
-    else
-        Errors <| allErrors
-
-
-toJsCodec : Codec ToJsPayload
-toJsCodec =
-    Codec.custom
-        (\errorsTag success vApiResponse value ->
-            case value of
-                Errors errorList ->
-                    errorsTag errorList
-
-                Success { pages, filesToGenerate, errors, staticHttpCache } ->
-                    success (ToJsSuccessPayload pages filesToGenerate staticHttpCache errors)
-
-                ApiResponse ->
-                    vApiResponse
-        )
-        |> Codec.variant1 "Errors" Errors errorCodec
-        |> Codec.variant1 "Success" Success successCodec
-        |> Codec.variant0 "ApiResponse" ApiResponse
-        |> Codec.buildCustom
-
-
 errorCodec : Codec (List BuildError)
 errorCodec =
     Codec.object (\errorString _ -> errorString)
@@ -117,39 +44,6 @@ errorCodec =
         |> Codec.buildObject
 
 
-successCodec : Codec ToJsSuccessPayload
-successCodec =
-    Codec.object ToJsSuccessPayload
-        |> Codec.field "pages"
-            .pages
-            (Codec.dict (Codec.dict Codec.string))
-        |> Codec.field "filesToGenerate"
-            .filesToGenerate
-            (Codec.build
-                (\list ->
-                    list
-                        |> Json.Encode.list
-                            (\item ->
-                                Json.Encode.object
-                                    [ ( "path", item.path |> String.join "/" |> Json.Encode.string )
-                                    , ( "content", item.content |> Json.Encode.string )
-                                    ]
-                            )
-                )
-                (Decode.list
-                    (Decode.map2 (\path content -> { path = path, content = content })
-                        (Decode.string |> Decode.map (String.split "/") |> Decode.field "path")
-                        (Decode.string |> Decode.field "content")
-                    )
-                )
-            )
-        |> Codec.field "staticHttpCache"
-            .staticHttpCache
-            (Codec.dict Codec.string)
-        |> Codec.field "errors" .errors errorCodec
-        |> Codec.buildObject
-
-
 successCodecNew : String -> String -> Codec ToJsSuccessPayloadNew
 successCodecNew canonicalSiteUrl currentPagePath =
     Codec.object ToJsSuccessPayloadNew
@@ -180,45 +74,56 @@ headCodec canonicalSiteUrl currentPagePath =
 
 type ToJsSuccessPayloadNewCombined
     = PageProgress ToJsSuccessPayloadNew
-    | InitialData InitialDataRecord
     | SendApiResponse { body : String, staticHttpCache : Dict String String, statusCode : Int }
     | ReadFile String
     | Glob String
+    | DoHttp { masked : Pages.StaticHttp.Request.Request, unmasked : Pages.StaticHttp.Request.Request }
     | Port String
-
+    | Errors (List BuildError)
-
+    | ApiResponse
-type alias InitialDataRecord =
-    { filesToGenerate : List FileToGenerate
-    }
 
 
 successCodecNew2 : String -> String -> Codec ToJsSuccessPayloadNewCombined
 successCodecNew2 canonicalSiteUrl currentPagePath =
     Codec.custom
-        (\success initialData vReadFile vGlob vSendApiResponse vPort value ->
+        (\errorsTag vApiResponse success vReadFile vGlob vDoHttp vSendApiResponse vPort value ->
             case value of
+                ApiResponse ->
+                    vApiResponse
+
+                Errors errorList ->
+                    errorsTag errorList
+
                 PageProgress payload ->
                     success payload
 
-                InitialData payload ->
-                    initialData payload
-
                 ReadFile filePath ->
                     vReadFile filePath
 
                 Glob globPattern ->
                     vGlob globPattern
 
+                DoHttp requestUrl ->
+                    vDoHttp requestUrl
+
                 SendApiResponse record ->
                     vSendApiResponse record
 
                 Port string ->
                     vPort string
         )
+        |> Codec.variant1 "Errors" Errors errorCodec
+        |> Codec.variant0 "ApiResponse" ApiResponse
         |> Codec.variant1 "PageProgress" PageProgress (successCodecNew canonicalSiteUrl currentPagePath)
-        |> Codec.variant1 "InitialData" InitialData initialDataCodec
         |> Codec.variant1 "ReadFile" ReadFile Codec.string
         |> Codec.variant1 "Glob" Glob Codec.string
+        |> Codec.variant1 "DoHttp"
+            DoHttp
+            (Codec.object (\masked unmasked -> { masked = masked, unmasked = unmasked })
+                |> Codec.field "masked" .masked Pages.StaticHttp.Request.codec
+                |> Codec.field "unmasked" .unmasked Pages.StaticHttp.Request.codec
+                |> Codec.buildObject
+            )
         |> Codec.variant1 "ApiResponse"
             SendApiResponse
             (Codec.object (\body staticHttpCache statusCode -> { body = body, staticHttpCache = staticHttpCache, statusCode = statusCode })
@@ -231,33 +136,3 @@ successCodecNew2 canonicalSiteUrl currentPagePath =
             )
         |> Codec.variant1 "Port" Port Codec.string
         |> Codec.buildCustom
-
-
-filesToGenerateCodec : Codec (List { path : List String, content : String })
-filesToGenerateCodec =
-    Codec.build
-        (\list ->
-            list
-                |> Json.Encode.list
-                    (\item ->
-                        Json.Encode.object
-                            [ ( "path", item.path |> String.join "/" |> Json.Encode.string )
-                            , ( "content", item.content |> Json.Encode.string )
-                            ]
-                    )
-        )
-        (Decode.list
-            (Decode.map2 (\path content -> { path = path, content = content })
-                (Decode.string |> Decode.map (String.split "/") |> Decode.field "path")
-                (Decode.string |> Decode.field "content")
-            )
-        )
-
-
-initialDataCodec : Codec InitialDataRecord
-initialDataCodec =
-    Codec.object InitialDataRecord
-        |> Codec.field "filesToGenerate"
-            .filesToGenerate
-            filesToGenerateCodec
-        |> Codec.buildObject

src/Pages/Internal/StaticHttpBody.elm

@@ -1,5 +1,6 @@
-module Pages.Internal.StaticHttpBody exposing (Body(..), encode)
+module Pages.Internal.StaticHttpBody exposing (Body(..), codec, encode)
 
+import Codec exposing (Codec)
 import Json.Encode as Encode
 
 
@@ -31,3 +32,23 @@ encodeWithType typeName otherFields =
     Encode.object <|
         ( "type", Encode.string typeName )
             :: otherFields
+
+
+codec : Codec Body
+codec =
+    Codec.custom
+        (\vEmpty vString vJson value ->
+            case value of
+                EmptyBody ->
+                    vEmpty
+
+                StringBody a b ->
+                    vString a b
+
+                JsonBody body ->
+                    vJson body
+        )
+        |> Codec.variant0 "EmptyBody" EmptyBody
+        |> Codec.variant2 "StringBody" StringBody Codec.string Codec.string
+        |> Codec.variant1 "JsonBody" JsonBody Codec.value
+        |> Codec.buildCustom

src/Pages/ProgramConfig.elm

@@ -39,7 +39,7 @@ type alias ProgramConfig userMsg userModel route siteData pageData sharedData =
     , data : route -> DataSource.DataSource pageData
     , view :
         { path : Path
-        , frontmatter : route
+        , route : route
         }
         -> Maybe PageUrl
         -> sharedData
@@ -69,4 +69,5 @@ type alias ProgramConfig userMsg userModel route siteData pageData sharedData =
         (Html Never -> String)
         -> List (ApiRoute.Done ApiRoute.Response)
     , pathPatterns : List RoutePattern
+    , basePath : List String
     }

src/Pages/StaticHttp/Request.elm

@@ -1,6 +1,8 @@
-module Pages.StaticHttp.Request exposing (Request, hash)
+module Pages.StaticHttp.Request exposing (Request, codec, hash)
 
+import Codec exposing (Codec)
 import Json.Encode as Encode
+import Murmur3
 import Pages.Internal.StaticHttpBody as StaticHttpBody exposing (Body)
 
 
@@ -21,8 +23,20 @@ hash requestDetails =
         , ( "body", StaticHttpBody.encode requestDetails.body )
         ]
         |> Encode.encode 0
+        |> Murmur3.hashString 0
+        |> String.fromInt
 
 
 hashHeader : ( String, String ) -> Encode.Value
 hashHeader ( name, value ) =
     Encode.string <| name ++ ": " ++ value
+
+
+codec : Codec Request
+codec =
+    Codec.object Request
+        |> Codec.field "url" .url Codec.string
+        |> Codec.field "method" .method Codec.string
+        |> Codec.field "headers" .headers (Codec.list (Codec.tuple Codec.string Codec.string))
+        |> Codec.field "body" .body StaticHttpBody.codec
+        |> Codec.buildObject

src/Pages/StaticHttpRequest.elm

@@ -16,11 +16,7 @@ import TerminalText as Terminal
 
 
 type RawRequest value
-    = Request
+    = Request (Dict String WhatToDo) ( List (Secrets.Value Pages.StaticHttp.Request.Request), KeepOrDiscard -> ApplicationType -> RequestsAndPending -> RawRequest value )
-        (Dict String WhatToDo)
-        ( List (Secrets.Value Pages.StaticHttp.Request.Request)
-        , KeepOrDiscard -> ApplicationType -> RequestsAndPending -> RawRequest value
-        )
     | RequestError Error
     | Done (Dict String WhatToDo) value
 
@@ -70,14 +66,14 @@ merge key whatToDo1 whatToDo2 =
                       , message =
                             [ Terminal.text "I encountered DataSource.distill with two matching keys that had differing encoded values.\n\n"
                             , Terminal.text "Look for "
-                            , Terminal.red <| Terminal.text "DataSource.distill"
+                            , Terminal.red <| "DataSource.distill"
                             , Terminal.text " with the key "
-                            , Terminal.red <| Terminal.text ("\"" ++ key ++ "\"")
+                            , Terminal.red <| ("\"" ++ key ++ "\"")
                             , Terminal.text "\n\n"
-                            , Terminal.yellow <| Terminal.text "The first encoded value was:\n"
+                            , Terminal.yellow <| "The first encoded value was:\n"
                             , Terminal.text <| Json.Encode.encode 2 distilled1
                             , Terminal.text "\n\n-------------------------------\n\n"
-                            , Terminal.yellow <| Terminal.text "The second encoded value was:\n"
+                            , Terminal.yellow <| "The second encoded value was:\n"
                             , Terminal.text <| Json.Encode.encode 2 distilled2
                             ]
                       , path = "" -- TODO wire in path here?
@@ -263,10 +259,6 @@ resolveUrlsHelp appType request rawResponses soFar =
         RequestError error ->
             case error of
                 MissingHttpResponse _ next ->
-                    let
-                        thing =
-                            next |> List.map Secrets.maskedLookup
-                    in
                     (soFar ++ next)
                         |> List.Extra.uniqueBy (Secrets.maskedLookup >> Pages.StaticHttp.Request.hash)
 
@@ -311,7 +303,7 @@ cacheRequestResolutionHelp foundUrls appType request rawResponses =
     case request of
         RequestError error ->
             case error of
-                MissingHttpResponse key _ ->
+                MissingHttpResponse _ _ ->
                     -- TODO do I need to pass through continuation URLs here? -- Incomplete (urlList ++ foundUrls)
                     Incomplete foundUrls
 

src/Path.elm

@@ -72,7 +72,7 @@ fromString path =
 {-| -}
 toSegments : Path -> List String
 toSegments (Path path) =
-    path |> String.split "/"
+    path |> String.split "/" |> List.filter ((/=) "")
 
 
 {-| Turn a Path to an absolute URL (with no trailing slash).

src/RenderRequest.elm

@@ -3,6 +3,7 @@ module RenderRequest exposing
     , RenderRequest(..)
     , RequestPayload(..)
     , decoder
+    , default
     , maybeRequestPayload
     )
 
@@ -10,6 +11,7 @@ import ApiRoute
 import HtmlPrinter
 import Internal.ApiRoute
 import Json.Decode as Decode
+import Json.Encode as Encode
 import Pages.ProgramConfig exposing (ProgramConfig)
 import Path exposing (Path)
 import Regex
@@ -24,15 +26,19 @@ type RequestPayload route
 
 type RenderRequest route
     = SinglePage IncludeHtml (RequestPayload route) Decode.Value
-    | FullBuild
+
+
+default : RenderRequest route
+default =
+    SinglePage
+        HtmlAndJson
+        (NotFound (Path.fromString "/error"))
+        Encode.null
 
 
 maybeRequestPayload : RenderRequest route -> Maybe Decode.Value
 maybeRequestPayload renderRequest =
     case renderRequest of
-        FullBuild ->
-            Nothing
-
         SinglePage _ _ rawJson ->
             Just rawJson
 
@@ -46,7 +52,7 @@ decoder :
     ProgramConfig userMsg userModel (Maybe route) siteData pageData sharedData
     -> Decode.Decoder (RenderRequest (Maybe route))
 decoder config =
-    optionalField "request"
+    Decode.field "request"
         (Decode.map3
             (\includeHtml requestThing payload ->
                 SinglePage includeHtml requestThing payload
@@ -73,15 +79,6 @@ decoder config =
             (requestPayloadDecoder config)
             (Decode.field "payload" Decode.value)
         )
-        |> Decode.map
-            (\maybeRequest ->
-                case maybeRequest of
-                    Just request ->
-                        request
-
-                    Nothing ->
-                        FullBuild
-            )
 
 
 

src/Router.elm

@@ -32,6 +32,7 @@ fromOptionalSplat : Maybe String -> List String
 fromOptionalSplat maybeMatch =
     maybeMatch
         |> Maybe.map (\match -> match |> String.split "/")
+        |> Maybe.map (List.filter (\item -> item /= ""))
         |> Maybe.withDefault []
 
 

src/Secrets.elm

@@ -50,11 +50,11 @@ buildError secretName secretsDict =
         [ Terminal.text "I expected to find this Secret in your environment variables but didn't find a match:\n\nSecrets.get \""
         , Terminal.text secretName
         , Terminal.text "\"\n             "
-        , Terminal.red <| Terminal.text (underlineText (secretName |> String.length))
+        , Terminal.red <| underlineText (secretName |> String.length)
         , Terminal.text "\n\nSo maybe "
-        , Terminal.yellow <| Terminal.text secretName
+        , Terminal.yellow <| secretName
         , Terminal.text " should be "
-        , Terminal.green <| Terminal.text (sortMatches secretName availableEnvironmentVariables |> List.head |> Maybe.withDefault "")
+        , Terminal.green <| (sortMatches secretName availableEnvironmentVariables |> List.head |> Maybe.withDefault "")
         ]
     , path = "" -- TODO wire in path here?
     , fatal = True

src/TerminalText.elm

@@ -1,13 +1,12 @@
 module TerminalText exposing
-    ( Color(..)
+    ( Text(..)
-    , Text(..)
     , ansi
     , ansiPrefix
     , blue
     , colorToString
     , cyan
     , encoder
-    , getString
+    , fromAnsiString
     , green
     , red
     , resetColors
@@ -17,50 +16,42 @@ module TerminalText exposing
     , yellow
     )
 
+import Ansi
 import Json.Encode as Encode
 
 
 type Text
-    = RawText String
+    = Style AnsiStyle String
-    | Style Color Text
-
-
-type Color
-    = Red
-    | Blue
-    | Green
-    | Yellow
-    | Cyan
 
 
 text : String -> Text
 text value =
-    RawText value
+    Style blankStyle value
 
 
-cyan : Text -> Text
+cyan : String -> Text
 cyan inner =
-    Style Cyan inner
+    Style { blankStyle | color = Just Ansi.Cyan } inner
 
 
-green : Text -> Text
+green : String -> Text
 green inner =
-    Style Green inner
+    Style { blankStyle | color = Just Ansi.Green } inner
 
 
-yellow : Text -> Text
+yellow : String -> Text
 yellow inner =
-    Style Yellow inner
+    Style { blankStyle | color = Just Ansi.Yellow } inner
 
 
-red : Text -> Text
+red : String -> Text
 red inner =
-    Style Red inner
+    Style { blankStyle | color = Just Ansi.Red } inner
 
 
-blue : Text -> Text
+blue : String -> Text
 blue inner =
-    Style Blue inner
+    Style { blankStyle | color = Just Ansi.Blue } inner
 
 
 resetColors : String
@@ -78,25 +69,29 @@ ansiPrefix =
     "\u{001B}"
 
 
-colorToString : Color -> String
+colorToString : Ansi.Color -> String
 colorToString color =
     ansi <|
         case color of
-            Red ->
+            Ansi.Red ->
                 "[31m"
 
-            Blue ->
+            Ansi.Blue ->
                 "[34m"
 
-            Green ->
+            Ansi.Green ->
                 "[32m"
 
-            Yellow ->
+            Ansi.Yellow ->
                 "[33m"
 
-            Cyan ->
+            Ansi.Cyan ->
                 "[36m"
 
+            _ ->
+                -- TODO
+                ""
+
 
 toString : List Text -> String
 toString list =
@@ -106,57 +101,142 @@ toString list =
 
 
 toString_ : Text -> String
-toString_ textValue =
+toString_ (Style ansiStyle innerText) =
-    -- elm-review: known-unoptimized-recursion
-    case textValue of
-        RawText content ->
-            content
-
-        Style color innerText ->
     String.concat
-                [ colorToString color
+        [ ansiStyle.color |> Maybe.withDefault Ansi.White |> colorToString
-                , toString_ innerText
+        , innerText
         , resetColors
         ]
 
 
+fromAnsiString : String -> List Text
+fromAnsiString ansiString =
+    Ansi.parseInto ( blankStyle, [] ) parseInto ansiString
+        |> Tuple.second
+        |> List.reverse
+
+
+type alias AnsiStyle =
+    { bold : Bool
+    , underline : Bool
+    , color : Maybe Ansi.Color
+    }
+
+
+blankStyle : AnsiStyle
+blankStyle =
+    { bold = False
+    , underline = False
+    , color = Nothing
+    }
+
+
+parseInto : Ansi.Action -> ( AnsiStyle, List Text ) -> ( AnsiStyle, List Text )
+parseInto action ( pendingStyle, soFar ) =
+    case action of
+        Ansi.Print string ->
+            ( blankStyle, Style pendingStyle string :: soFar )
+
+        Ansi.Remainder _ ->
+            ( pendingStyle, soFar )
+
+        Ansi.SetForeground maybeColor ->
+            case maybeColor of
+                Just newColor ->
+                    ( { pendingStyle
+                        | color = Just newColor
+                      }
+                    , soFar
+                    )
+
+                Nothing ->
+                    ( blankStyle, soFar )
+
+        Ansi.SetBold bool ->
+            ( { pendingStyle | bold = bool }, soFar )
+
+        Ansi.SetFaint _ ->
+            ( pendingStyle, soFar )
+
+        Ansi.SetItalic _ ->
+            ( pendingStyle, soFar )
+
+        Ansi.SetUnderline bool ->
+            ( { pendingStyle | underline = bool }, soFar )
+
+        Ansi.SetBackground _ ->
+            ( pendingStyle, soFar )
+
+        Ansi.Linebreak ->
+            case soFar of
+                next :: rest ->
+                    ( pendingStyle, Style blankStyle "\n" :: next :: rest )
+
+                [] ->
+                    ( pendingStyle, soFar )
+
+        _ ->
+            ( pendingStyle, soFar )
+
+
 encoder : Text -> Encode.Value
-encoder node =
+encoder (Style ansiStyle string) =
     Encode.object
-        [ ( "bold", Encode.bool False )
+        [ ( "bold", Encode.bool ansiStyle.bold )
-        , ( "underline", Encode.bool False )
+        , ( "underline", Encode.bool ansiStyle.underline )
         , ( "color"
           , Encode.string <|
-                case node of
+                case ansiStyle.color |> Maybe.withDefault Ansi.White of
-                    RawText _ ->
+                    Ansi.Red ->
-                        "WHITE"
-
-                    Style color _ ->
-                        case color of
-                            Red ->
                         "red"
 
-                            Blue ->
+                    Ansi.Blue ->
                         "blue"
 
-                            Green ->
+                    Ansi.Green ->
                         "green"
 
-                            Yellow ->
+                    Ansi.Yellow ->
                         "yellow"
 
-                            Cyan ->
+                    Ansi.Cyan ->
                         "cyan"
+
+                    Ansi.Black ->
+                        "black"
+
+                    Ansi.Magenta ->
+                        "magenta"
+
+                    Ansi.White ->
+                        "white"
+
+                    Ansi.BrightBlack ->
+                        "BLACK"
+
+                    Ansi.BrightRed ->
+                        "RED"
+
+                    Ansi.BrightGreen ->
+                        "GREEN"
+
+                    Ansi.BrightYellow ->
+                        "YELLOW"
+
+                    Ansi.BrightBlue ->
+                        "BLUE"
+
+                    Ansi.BrightMagenta ->
+                        "MAGENTA"
+
+                    Ansi.BrightCyan ->
+                        "CYAN"
+
+                    Ansi.BrightWhite ->
+                        "WHITE"
+
+                    Ansi.Custom _ _ _ ->
+                        ""
           )
-        , ( "string", Encode.string (getString node) )
+        , ( "string", Encode.string string )
         ]
-
-
-getString : Text -> String
-getString node =
-    case node of
-        RawText string ->
-            string
-
-        Style _ innerNode ->
-            getString innerNode

tests/StaticHttpRequestsTests.elm

@@ -11,13 +11,12 @@ import Expect
 import Html
 import Json.Decode as JD
 import Json.Encode as Encode
-import List.Extra
 import NotFoundReason
 import OptimizedDecoder as Decode exposing (Decoder)
 import Pages.ContentCache as ContentCache exposing (ContentCache)
 import Pages.Internal.Platform.Cli exposing (..)
 import Pages.Internal.Platform.Effect as Effect exposing (Effect)
-import Pages.Internal.Platform.ToJsPayload as ToJsPayload exposing (ToJsPayload)
+import Pages.Internal.Platform.ToJsPayload as ToJsPayload
 import Pages.Internal.StaticHttpBody as StaticHttpBody
 import Pages.Manifest as Manifest
 import Pages.ProgramConfig exposing (ProgramConfig)
@@ -28,12 +27,11 @@ import ProgramTest exposing (ProgramTest)
 import Regex
 import RenderRequest
 import Secrets
-import Serialize
 import SimulatedEffect.Cmd
 import SimulatedEffect.Http as Http
 import SimulatedEffect.Ports
 import SimulatedEffect.Task
-import Test exposing (Test, describe, only, test)
+import Test exposing (Test, describe, test)
 import Test.Http
 
 
@@ -52,13 +50,10 @@ all =
                         "https://api.github.com/repos/dillonkearns/elm-pages"
                         """{ "stargazer_count": 86 }"""
                     |> expectSuccess
-                        [ ( ""
+                        [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
-                          , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
                           , """{"stargazer_count":86}"""
                           )
                         ]
-                          )
-                        ]
         , test "StaticHttp request for initial are resolved" <|
             \() ->
                 start
@@ -76,13 +71,10 @@ all =
                         "https://api.github.com/repos/dillonkearns/elm-pages"
                         """{ "stargazer_count": 86 }"""
                     |> expectSuccess
-                        [ ( "post-1"
+                        [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
-                          , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
                           , """{"stargazer_count":86}"""
                           )
                         ]
-                          )
-                        ]
         , describe "single page renders"
             [ test "single pages that are pre-rendered" <|
                 \() ->
@@ -151,13 +143,10 @@ all =
                         "https://api.github.com/repos/dillonkearns/elm-pages"
                         """{ "stargazer_count": 86, "language": "Elm" }"""
                     |> expectSuccess
-                        [ ( "post-1"
+                        [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
-                          , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
                           , """{"stargazer_count":86,"language":"Elm"}"""
                           )
                         ]
-                          )
-                        ]
         , test "andThen" <|
             \() ->
                 start
@@ -178,16 +167,13 @@ all =
                         "NEXT-REQUEST"
                         """null"""
                     |> expectSuccess
-                        [ ( "elm-pages"
+                        [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
-                          , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
                           , """null"""
                           )
                         , ( get "NEXT-REQUEST"
                           , """null"""
                           )
                         ]
-                          )
-                        ]
         , test "andThen chain avoids repeat requests" <|
             \() ->
                 let
@@ -272,8 +258,7 @@ all =
                         "url10"
                         """{"image": "image10.jpg"}"""
                     |> expectSuccess
-                        [ ( "elm-pages"
+                        [ ( get "https://pokeapi.co/api/v2/pokemon/"
-                          , [ ( get "https://pokeapi.co/api/v2/pokemon/"
                           , """[{"url":"url1"},{"url":"url2"},{"url":"url3"},{"url":"url4"},{"url":"url5"},{"url":"url6"},{"url":"url7"},{"url":"url8"},{"url":"url9"},{"url":"url10"}]"""
                           )
                         , ( get "url1"
@@ -307,40 +292,39 @@ all =
                           , """{"image":"image10.jpg"}"""
                           )
                         ]
-                          )
+
-                        ]
+        --, test "port is sent out once all requests are finished" <|
-        , test "port is sent out once all requests are finished" <|
+        --    \() ->
-            \() ->
+        --        start
-                start
+        --            [ ( [ "elm-pages" ]
-                    [ ( [ "elm-pages" ]
+        --              , DataSource.Http.get (Secrets.succeed "https://api.github.com/repos/dillonkearns/elm-pages") starDecoder
-                      , DataSource.Http.get (Secrets.succeed "https://api.github.com/repos/dillonkearns/elm-pages") starDecoder
+        --              )
-                      )
+        --            , ( [ "elm-pages-starter" ]
-                    , ( [ "elm-pages-starter" ]
+        --              , DataSource.Http.get (Secrets.succeed "https://api.github.com/repos/dillonkearns/elm-pages-starter") starDecoder
-                      , DataSource.Http.get (Secrets.succeed "https://api.github.com/repos/dillonkearns/elm-pages-starter") starDecoder
+        --              )
-                      )
+        --            ]
-                    ]
+        --            |> ProgramTest.simulateHttpOk
-                    |> ProgramTest.simulateHttpOk
+        --                "GET"
-                        "GET"
+        --                "https://api.github.com/repos/dillonkearns/elm-pages"
-                        "https://api.github.com/repos/dillonkearns/elm-pages"
+        --                """{ "stargazer_count": 86 }"""
-                        """{ "stargazer_count": 86 }"""
+        --            |> ProgramTest.simulateHttpOk
-                    |> ProgramTest.simulateHttpOk
+        --                "GET"
-                        "GET"
+        --                "https://api.github.com/repos/dillonkearns/elm-pages-starter"
-                        "https://api.github.com/repos/dillonkearns/elm-pages-starter"
+        --                """{ "stargazer_count": 22 }"""
-                        """{ "stargazer_count": 22 }"""
+        --            |> expectSuccess
-                    |> expectSuccess
+        --                [ ( "elm-pages"
-                        [ ( "elm-pages"
+        --                  , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
-                          , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
+        --                      , """{"stargazer_count":86}"""
-                              , """{"stargazer_count":86}"""
+        --                      )
-                              )
+        --                    ]
-                            ]
+        --                  )
-                          )
+        --                , ( "elm-pages-starter"
-                        , ( "elm-pages-starter"
+        --                  , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages-starter"
-                          , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages-starter"
+        --                      , """{"stargazer_count":22}"""
-                              , """{"stargazer_count":22}"""
+        --                      )
-                              )
+        --                    ]
-                            ]
+        --                  )
-                          )
+        --                ]
-                        ]
         , test "reduced JSON is sent out" <|
             \() ->
                 start
@@ -353,13 +337,10 @@ all =
                         "https://api.github.com/repos/dillonkearns/elm-pages"
                         """{ "stargazer_count": 86, "unused_field": 123 }"""
                     |> expectSuccess
-                        [ ( ""
+                        [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
-                          , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
                           , """{"stargazer_count":86}"""
                           )
                         ]
-                          )
-                        ]
         , test "you can use elm/json decoders with StaticHttp.unoptimizedRequest" <|
             \() ->
                 start
@@ -382,13 +363,10 @@ all =
                         "https://api.github.com/repos/dillonkearns/elm-pages"
                         """{ "stargazer_count": 86, "unused_field": 123 }"""
                     |> expectSuccess
-                        [ ( ""
+                        [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
-                          , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
                           , """{ "stargazer_count": 86, "unused_field": 123 }"""
                           )
                         ]
-                          )
-                        ]
         , test "plain string" <|
             \() ->
                 start
@@ -409,13 +387,10 @@ all =
                         "https://example.com/file.txt"
                         "This is a raw text file."
                     |> expectSuccess
-                        [ ( ""
+                        [ ( get "https://example.com/file.txt"
-                          , [ ( get "https://example.com/file.txt"
                           , "This is a raw text file."
                           )
                         ]
-                          )
-                        ]
         , test "Err in String to Result function turns into decode error" <|
             \() ->
                 start
@@ -445,7 +420,7 @@ all =
                         "This is a raw text file."
                     |> ProgramTest.expectOutgoingPortValues
                         "toJsPort"
-                        (Codec.decoder ToJsPayload.toJsCodec)
+                        (Codec.decoder (ToJsPayload.successCodecNew2 "" ""))
                         (expectErrorsPort
                             """-- STATIC HTTP DECODING ERROR ----------------------------------------------------- elm-pages
 
@@ -473,8 +448,7 @@ String was not uppercased"""
                         "https://api.github.com/repos/dillonkearns/elm-pages"
                         """{ "stargazer_count": 86, "unused_field": 123 }"""
                     |> expectSuccess
-                        [ ( ""
+                        [ ( { method = "POST"
-                          , [ ( { method = "POST"
                             , url = "https://api.github.com/repos/dillonkearns/elm-pages"
                             , headers = []
                             , body = DataSource.emptyBody
@@ -482,8 +456,6 @@ String was not uppercased"""
                           , """{"stargazer_count":86}"""
                           )
                         ]
-                          )
-                        ]
         , test "json is reduced from andThen chains" <|
             \() ->
                 start
@@ -504,16 +476,13 @@ String was not uppercased"""
                         "https://api.github.com/repos/dillonkearns/elm-pages-starter"
                         """{ "stargazer_count": 50, "unused_field": 456 }"""
                     |> expectSuccess
-                        [ ( ""
+                        [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
-                          , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
                           , """{"stargazer_count":100}"""
                           )
                         , ( get "https://api.github.com/repos/dillonkearns/elm-pages-starter"
                           , """{"stargazer_count":50}"""
                           )
                         ]
-                          )
-                        ]
         , test "reduced json is preserved by StaticHttp.map2" <|
             \() ->
                 start
@@ -532,16 +501,13 @@ String was not uppercased"""
                         "https://api.github.com/repos/dillonkearns/elm-pages-starter"
                         """{ "stargazer_count": 50, "unused_field": 456 }"""
                     |> expectSuccess
-                        [ ( ""
+                        [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
-                          , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
                           , """{"stargazer_count":100}"""
                           )
                         , ( get "https://api.github.com/repos/dillonkearns/elm-pages-starter"
                           , """{"stargazer_count":50}"""
                           )
                         ]
-                          )
-                        ]
         , test "the port sends out even if there are no http requests" <|
             \() ->
                 start
@@ -549,7 +515,7 @@ String was not uppercased"""
                       , DataSource.succeed ()
                       )
                     ]
-                    |> expectSuccess [ ( "", [] ) ]
+                    |> expectSuccess []
         , test "the port sends out when there are duplicate http requests for the same page" <|
             \() ->
                 start
@@ -564,13 +530,10 @@ String was not uppercased"""
                         "http://example.com"
                         """null"""
                     |> expectSuccess
-                        [ ( ""
+                        [ ( get "http://example.com"
-                          , [ ( get "http://example.com"
                           , """null"""
                           )
                         ]
-                          )
-                        ]
         , test "an error is sent out for decoder failures" <|
             \() ->
                 start
@@ -584,7 +547,7 @@ String was not uppercased"""
                         """{ "stargazer_count": 86 }"""
                     |> ProgramTest.expectOutgoingPortValues
                         "toJsPort"
-                        (Codec.decoder ToJsPayload.toJsCodec)
+                        (Codec.decoder (ToJsPayload.successCodecNew2 "" ""))
                         (expectErrorsPort
                             """-- STATIC HTTP DECODING ERROR ----------------------------------------------------- elm-pages
 
@@ -627,7 +590,7 @@ I encountered some errors while decoding this JSON:
                         """ "continuation-url" """
                     |> ProgramTest.expectOutgoingPortValues
                         "toJsPort"
-                        (Codec.decoder ToJsPayload.toJsCodec)
+                        (Codec.decoder (ToJsPayload.successCodecNew2 "" ""))
                         (expectErrorsPort
                             """-- MISSING SECRET ----------------------------------------------------- elm-pages
 
@@ -656,21 +619,8 @@ So maybe MISSING should be API_KEY"""
                         )
                     |> ProgramTest.expectOutgoingPortValues
                         "toJsPort"
-                        (Codec.decoder ToJsPayload.toJsCodec)
+                        (Codec.decoder (ToJsPayload.successCodecNew2 "" ""))
                         (expectErrorsPort """-- STATIC HTTP ERROR ----------------------------------------------------- elm-pages
-
-I got an error making an HTTP request to this URL: https://api.github.com/repos/dillonkearns/elm-pages
-
-Bad status: 404
-Status message: TODO: if you need this, please report to https://github.com/avh4/elm-program-test/issues
-Body: 
-
--- STATIC HTTP DECODING ERROR ----------------------------------------------------- elm-pages
-
-
-
-Payload sent back invalid JSON
-TODO
 """)
         , test "uses real secrets to perform request and masked secrets to store and lookup response" <|
             \() ->
@@ -707,8 +657,7 @@ TODO
                             }
                         )
                     |> expectSuccess
-                        [ ( ""
+                        [ ( { method = "GET"
-                          , [ ( { method = "GET"
                             , url = "https://api.github.com/repos/dillonkearns/elm-pages?apiKey=<API_KEY>"
                             , headers =
                                 [ ( "Authorization", "Bearer <BEARER>" )
@@ -718,8 +667,6 @@ TODO
                           , """{}"""
                           )
                         ]
-                          )
-                        ]
         , describe "staticHttpCache"
             [ test "it doesn't perform http requests that are provided in the http cache flag" <|
                 \() ->
@@ -737,13 +684,10 @@ TODO
                           )
                         ]
                         |> expectSuccess
-                            [ ( ""
+                            [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
-                              , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
                               , """{"stargazer_count":86}"""
                               )
                             ]
-                              )
-                            ]
             , test "it ignores unused cache" <|
                 \() ->
                     startWithHttpCache
@@ -764,13 +708,10 @@ TODO
                             "https://api.github.com/repos/dillonkearns/elm-pages"
                             """{ "stargazer_count": 86 }"""
                         |> expectSuccess
-                            [ ( ""
+                            [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
-                              , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
                               , """{"stargazer_count":86}"""
                               )
                             ]
-                              )
-                            ]
             , test "validate DataSource is not stored for any pages" <|
                 \() ->
                     startWithRoutes [ "hello" ]
@@ -941,7 +882,25 @@ TODO
                                     [ {- ToJsPayload.Glob _, ToJsPayload.ReadFile _ -} ToJsPayload.PageProgress portData ] ->
                                         portData.contentJson
                                             |> Expect.equalDicts
-                                                (Dict.fromList [ ( "{\"method\":\"GET\",\"url\":\"file://content/glossary/hello.md\",\"headers\":[],\"body\":{\"type\":\"empty\"}}", "{\"withoutFrontmatter\":\"BODY\"}" ), ( "{\"method\":\"GET\",\"url\":\"glob://content/glossary/*.md\",\"headers\":[],\"body\":{\"type\":\"empty\"}}", "[\"content/glossary/hello.md\"]" ) ])
+                                                (Dict.fromList
+                                                    [ ( Request.hash
+                                                            { method = "GET"
+                                                            , url = "file://content/glossary/hello.md"
+                                                            , headers = []
+                                                            , body = DataSource.Http.emptyBody
+                                                            }
+                                                      , "{\"withoutFrontmatter\":\"BODY\"}"
+                                                      )
+                                                    , ( Request.hash
+                                                            { method = "GET"
+                                                            , url = "glob://content/glossary/*.md"
+                                                            , headers = []
+                                                            , body = DataSource.Http.emptyBody
+                                                            }
+                                                      , "[\"content/glossary/hello.md\"]"
+                                                      )
+                                                    ]
+                                                )
 
                                     _ ->
                                         Expect.fail <|
@@ -1009,7 +968,7 @@ TODO
                             """{ "stargazer_count": 123 }"""
                         |> ProgramTest.expectOutgoingPortValues
                             "toJsPort"
-                            (Codec.decoder ToJsPayload.toJsCodec)
+                            (Codec.decoder (ToJsPayload.successCodecNew2 "" ""))
                             (expectErrorsPort """-- NON-UNIQUE DISTILL KEYS ----------------------------------------------------- elm-pages
 I encountered DataSource.distill with two matching keys that had differing encoded values.
 
@@ -1022,91 +981,92 @@ The second encoded value was:
 
 123""")
             ]
-        , describe "generateFiles"
+
-            [ test "initial requests are sent out" <|
+        --, describe "generateFiles"
-                \() ->
+        --    [ test "initial requests are sent out" <|
-                    startLowLevel
+        --        \() ->
-                        [ ApiRoute.succeed
+        --            startLowLevel
-                            (DataSource.Http.get (Secrets.succeed "https://api.github.com/repos/dillonkearns/elm-pages")
+        --                [ ApiRoute.succeed
-                                (starDecoder
+        --                    (DataSource.Http.get (Secrets.succeed "https://api.github.com/repos/dillonkearns/elm-pages")
-                                    |> Decode.map
+        --                        (starDecoder
-                                        (\starCount ->
+        --                            |> Decode.map
-                                            { body = "Star count: " ++ String.fromInt starCount
+        --                                (\starCount ->
-                                            }
+        --                                    { body = "Star count: " ++ String.fromInt starCount
-                                        )
+        --                                    }
-                                )
+        --                                )
-                            )
+        --                        )
-                            |> ApiRoute.literal "test.txt"
+        --                    )
-                            |> ApiRoute.single
+        --                    |> ApiRoute.literal "test.txt"
-                        ]
+        --                    |> ApiRoute.single
-                        []
+        --                ]
-                        []
+        --                []
-                        |> ProgramTest.simulateHttpOk
+        --                []
-                            "GET"
+        --                |> ProgramTest.simulateHttpOk
-                            "https://api.github.com/repos/dillonkearns/elm-pages"
+        --                    "GET"
-                            """{ "stargazer_count": 86 }"""
+        --                    "https://api.github.com/repos/dillonkearns/elm-pages"
-                        |> expectSuccessNew
+        --                    """{ "stargazer_count": 86 }"""
-                            []
+        --                |> expectSuccessNew
-                            [ \success ->
+        --                    []
-                                success.filesToGenerate
+        --                    [ \success ->
-                                    |> Expect.equal
+        --                        success.filesToGenerate
-                                        [ { path = [ "test.txt" ]
+        --                            |> Expect.equal
-                                          , content = "Star count: 86"
+        --                                [ { path = [ "test.txt" ]
-                                          }
+        --                                  , content = "Star count: 86"
-                                        ]
+        --                                  }
-                            ]
+        --                                ]
-            , test "it sends success port when no HTTP requests are needed because they're all cached" <|
+        --                    ]
-                \() ->
+        --    , test "it sends success port when no HTTP requests are needed because they're all cached" <|
-                    startLowLevel
+        --        \() ->
-                        [ ApiRoute.succeed
+        --            startLowLevel
-                            (DataSource.Http.get (Secrets.succeed "https://api.github.com/repos/dillonkearns/elm-pages-starter")
+        --                [ ApiRoute.succeed
-                                (starDecoder
+        --                    (DataSource.Http.get (Secrets.succeed "https://api.github.com/repos/dillonkearns/elm-pages-starter")
-                                    |> Decode.map
+        --                        (starDecoder
-                                        (\starCount ->
+        --                            |> Decode.map
-                                            { body = "Star count: " ++ String.fromInt starCount
+        --                                (\starCount ->
-                                            }
+        --                                    { body = "Star count: " ++ String.fromInt starCount
-                                        )
+        --                                    }
-                                )
+        --                                )
-                            )
+        --                        )
-                            |> ApiRoute.literal "test.txt"
+        --                    )
-                            |> ApiRoute.single
+        --                    |> ApiRoute.literal "test.txt"
-                        ]
+        --                    |> ApiRoute.single
-                        [ ( { url = "https://api.github.com/repos/dillonkearns/elm-pages"
+        --                ]
-                            , method = "GET"
+        --                [ ( { url = "https://api.github.com/repos/dillonkearns/elm-pages"
-                            , headers = []
+        --                    , method = "GET"
-                            , body = StaticHttpBody.EmptyBody
+        --                    , headers = []
-                            }
+        --                    , body = StaticHttpBody.EmptyBody
-                          , """{"stargazer_count":86}"""
+        --                    }
-                          )
+        --                  , """{"stargazer_count":86}"""
-                        , ( { url = "https://api.github.com/repos/dillonkearns/elm-pages-starter"
+        --                  )
-                            , method = "GET"
+        --                , ( { url = "https://api.github.com/repos/dillonkearns/elm-pages-starter"
-                            , headers = []
+        --                    , method = "GET"
-                            , body = StaticHttpBody.EmptyBody
+        --                    , headers = []
-                            }
+        --                    , body = StaticHttpBody.EmptyBody
-                          , """{"stargazer_count":23}"""
+        --                    }
-                          )
+        --                  , """{"stargazer_count":23}"""
-                        ]
+        --                  )
-                        [ ( []
+        --                ]
-                          , DataSource.Http.get (Secrets.succeed "https://api.github.com/repos/dillonkearns/elm-pages") starDecoder
+        --                [ ( []
-                          )
+        --                  , DataSource.Http.get (Secrets.succeed "https://api.github.com/repos/dillonkearns/elm-pages") starDecoder
-                        ]
+        --                  )
-                        |> expectSuccessNew
+        --                ]
-                            [ ( ""
+        --                |> expectSuccessNew
-                              , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
+        --                    [ ( ""
-                                  , """{"stargazer_count":86}"""
+        --                      , [ ( get "https://api.github.com/repos/dillonkearns/elm-pages"
-                                  )
+        --                          , """{"stargazer_count":86}"""
-                                ]
+        --                          )
-                              )
+        --                        ]
-                            ]
+        --                      )
-                            [ \success ->
+        --                    ]
-                                success.filesToGenerate
+        --                    [ \success ->
-                                    |> Expect.equal
+        --                        success.filesToGenerate
-                                        [ { path = [ "test.txt" ]
+        --                            |> Expect.equal
-                                          , content = "Star count: 23"
+        --                                [ { path = [ "test.txt" ]
-                                          }
+        --                                  , content = "Star count: 23"
-                                        ]
+        --                                  }
-                            ]
+        --                                ]
-            ]
+        --                    ]
+        --    ]
         ]
 
 
@@ -1138,6 +1098,15 @@ startLowLevel apiRoutes staticHttpCache pages =
         contentCache =
             ContentCache.init Nothing
 
+        pageToLoad : List String
+        pageToLoad =
+            case pages |> List.head |> Maybe.map Tuple.first of
+                Just justPageToLoad ->
+                    justPageToLoad
+
+                Nothing ->
+                    Debug.todo "Error - no pages"
+
         config : ProgramConfig Msg () Route () () ()
         config =
             { toJsPort = toJsPort
@@ -1154,6 +1123,7 @@ startLowLevel apiRoutes staticHttpCache pages =
             , handleRoute = \_ -> DataSource.succeed Nothing
             , urlToRoute = .path >> Route
             , update = \_ _ _ _ _ -> ( (), Cmd.none )
+            , basePath = []
             , data =
                 \(Route pageRoute) ->
                     let
@@ -1239,7 +1209,19 @@ startLowLevel apiRoutes staticHttpCache pages =
        -> ( model, Effect pathKey )
     -}
     ProgramTest.createDocument
-        { init = init RenderRequest.FullBuild contentCache config
+        { init =
+            init
+                (RenderRequest.SinglePage
+                    RenderRequest.OnlyJson
+                    (RenderRequest.Page
+                        { path = Path.fromString (pageToLoad |> String.join "/")
+                        , frontmatter = Route (pageToLoad |> String.join "/")
+                        }
+                    )
+                    (Encode.object [])
+                )
+                contentCache
+                config
         , update = update contentCache config
         , view = \_ -> { title = "", body = [] }
         }
@@ -1284,6 +1266,7 @@ startWithRoutes pageToLoad staticRoutes staticHttpCache pages =
                         |> DataSource.succeed
             , urlToRoute = .path >> Route
             , update = \_ _ _ _ _ -> ( (), Cmd.none )
+            , basePath = []
             , data =
                 \(Route pageRoute) ->
                     let
@@ -1347,7 +1330,6 @@ startWithRoutes pageToLoad staticRoutes staticHttpCache pages =
                         |> Dict.fromList
                         |> Encode.dict identity Encode.string
                   )
-                , ( "mode", Encode.string "elm-to-html-beta" )
                 , ( "staticHttpCache", encodedStaticHttpCache )
                 ]
 
@@ -1410,20 +1392,13 @@ simulateEffects effect =
         Effect.NoEffect ->
             SimulatedEffect.Cmd.none
 
-        Effect.SendJsData value ->
-            SimulatedEffect.Ports.send "toJsPort" (value |> Codec.encoder ToJsPayload.toJsCodec)
-
         --            toJsPort value |> Cmd.map never
         Effect.Batch list ->
             list
                 |> List.map simulateEffects
                 |> SimulatedEffect.Cmd.batch
 
-        Effect.FetchHttp ({ unmasked } as requests) ->
+        Effect.FetchHttp { unmasked } ->
-            let
-                _ =
-                    Debug.log "Fetching " unmasked.url
-            in
             if unmasked.url |> String.startsWith "file://" then
                 let
                     filePath : String
@@ -1455,10 +1430,6 @@ simulateEffects effect =
                     |> SimulatedEffect.Cmd.map never
 
             else
-                let
-                    _ =
-                        Debug.log "Fetching" unmasked.url
-                in
                 Http.request
                     { method = unmasked.method
                     , url = unmasked.url
@@ -1476,9 +1447,23 @@ simulateEffects effect =
                     , expect =
                         PagesHttp.expectString
                             (\response ->
-                                GotStaticHttpResponse
+                                case response of
-                                    { request = requests
+                                    Ok okResponse ->
-                                    , response = response
+                                        GotDataBatch
+                                            [ { request =
+                                                    { unmasked = unmasked
+                                                    , masked = unmasked -- TODO use masked
+                                                    }
+                                              , response = okResponse
+                                              }
+                                            ]
+
+                                    Err _ ->
+                                        GotBuildError
+                                            { title = "Static HTTP Error"
+                                            , message = []
+                                            , fatal = True
+                                            , path = ""
                                             }
                             )
                     , timeout = Nothing
@@ -1508,7 +1493,7 @@ simulateEffects effect =
             SimulatedEffect.Cmd.none
 
 
-expectErrorsPort : String -> List ToJsPayload -> Expect.Expectation
+expectErrorsPort : String -> List ToJsPayload.ToJsSuccessPayloadNewCombined -> Expect.Expectation
 expectErrorsPort expectedPlainString actualPorts =
     case actualPorts of
         [ ToJsPayload.Errors actualRichTerminalString ] ->
@@ -1528,10 +1513,12 @@ normalizeErrorExpectEqual : String -> String -> Expect.Expectation
 normalizeErrorExpectEqual expectedPlainString actualRichTerminalString =
     actualRichTerminalString
         |> Regex.replace
-            (Regex.fromString "\u{001B}\\[[0-9;]+m"
+            -- strip out all possible ANSI sequences
+            (Regex.fromString "(\\x9B|\\x1B\\[)[0-?]*[ -/]*[@-~]"
                 |> Maybe.withDefault Regex.never
             )
             (\_ -> "")
+        |> String.replace "\u{001B}" ""
         |> normalizeNewlines
         |> Expect.equal
             (expectedPlainString |> normalizeNewlines)
@@ -1546,6 +1533,10 @@ normalizeNewlines string =
         |> Regex.replace
             (Regex.fromString "( )+" |> Maybe.withDefault Regex.never)
             (\_ -> " ")
+        |> String.replace "\u{000D}" ""
+        |> Regex.replace
+            (Regex.fromString "\\s" |> Maybe.withDefault Regex.never)
+            (\_ -> "")
 
 
 toJsPort : a -> Cmd msg
@@ -1573,40 +1564,37 @@ starDecoder =
     Decode.field "stargazer_count" Decode.int
 
 
-expectSuccess : List ( String, List ( Request.Request, String ) ) -> ProgramTest model msg effect -> Expect.Expectation
+expectSuccess : List ( Request.Request, String ) -> ProgramTest model msg effect -> Expect.Expectation
 expectSuccess expectedRequests previous =
     expectSuccessNew expectedRequests [] previous
 
 
-expectSuccessNew : List ( String, List ( Request.Request, String ) ) -> List (ToJsPayload.ToJsSuccessPayload -> Expect.Expectation) -> ProgramTest model msg effect -> Expect.Expectation
+expectSuccessNew : List ( Request.Request, String ) -> List (ToJsPayload.ToJsSuccessPayloadNew -> Expect.Expectation) -> ProgramTest model msg effect -> Expect.Expectation
-expectSuccessNew expectedRequests expectations previous =
+expectSuccessNew expectedRequest expectations previous =
     previous
         |> ProgramTest.expectOutgoingPortValues
             "toJsPort"
-            (Codec.decoder ToJsPayload.toJsCodec)
+            (Codec.decoder (ToJsPayload.successCodecNew2 "" ""))
             (\value ->
                 case value of
-                    (ToJsPayload.Success portPayload) :: _ ->
+                    (ToJsPayload.PageProgress portPayload) :: _ ->
-                        portPayload
+                        let
-                            |> Expect.all
+                            singleExpectation : ToJsPayload.ToJsSuccessPayloadNew -> Expect.Expectation
-                                ((\subject ->
+                            singleExpectation =
-                                    subject.pages
+                                \subject ->
-                                        |> Expect.equalDicts
+                                    subject.contentJson
-                                            (expectedRequests
+                                        |> Expect.equal
-                                                |> List.map
+                                            (expectedRequest
-                                                    (\( url, requests ) ->
-                                                        ( url
-                                                        , requests
                                                 |> List.map
                                                     (\( request, response ) ->
                                                         ( Request.hash request, response )
                                                     )
                                                 |> Dict.fromList
                                             )
-                                                    )
+                        in
-                                                |> Dict.fromList
+                        portPayload
-                                            )
+                            |> Expect.all
-                                 )
+                                (singleExpectation
                                     :: expectations
                                 )
 
@@ -1631,7 +1619,27 @@ simulateSubscriptions _ =
                                     (JD.field "pattern" JD.string)
                                     (JD.field "result" JD.value)
                                 )
-                                |> JD.map GotGlob
+                                |> JD.map
+                                    (\( globPattern, response ) ->
+                                        GotDataBatch
+                                            [ { request =
+                                                    { masked =
+                                                        { url = "glob://" ++ globPattern
+                                                        , method = "GET"
+                                                        , headers = []
+                                                        , body = StaticHttpBody.EmptyBody
+                                                        }
+                                                    , unmasked =
+                                                        { url = "glob://" ++ globPattern
+                                                        , method = "GET"
+                                                        , headers = []
+                                                        , body = StaticHttpBody.EmptyBody
+                                                        }
+                                                    }
+                                              , response = Encode.encode 0 response
+                                              }
+                                            ]
+                                    )
 
                         "GotFile" ->
                             JD.field "data"
@@ -1639,7 +1647,27 @@ simulateSubscriptions _ =
                                     (JD.field "filePath" JD.string)
                                     JD.value
                                 )
-                                |> JD.map GotStaticFile
+                                |> JD.map
+                                    (\( filePath, response ) ->
+                                        GotDataBatch
+                                            [ { request =
+                                                    { masked =
+                                                        { url = "file://" ++ filePath
+                                                        , method = "GET"
+                                                        , headers = []
+                                                        , body = StaticHttpBody.EmptyBody
+                                                        }
+                                                    , unmasked =
+                                                        { url = "file://" ++ filePath
+                                                        , method = "GET"
+                                                        , headers = []
+                                                        , body = StaticHttpBody.EmptyBody
+                                                        }
+                                                    }
+                                              , response = Encode.encode 0 response
+                                              }
+                                            ]
+                                    )
 
                         _ ->
                             JD.fail "Unexpected subscription tag."