Merge pull request #639 from NoRedInk/get-a-little-more-shakey

port build system to shake

.gitignore

@@ -242,3 +242,6 @@ documentation.json
 /tests/axe-report.json
 /tests/deprecated-imports-report.txt
 /tests/VerifyExamples
+
+/_build
+/log

.travis.yml

@@ -1,32 +1,13 @@
-sudo: false
-
-language: node_js
-node_js:
-  - "10"
+language: nix
 
 cache:
   directories:
+    - /nix
     - node_modules
-    - sysconfcpus
     - ~/.elm
 
-before_install:
-  - echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
-  - | # epic build time improvement - see https://github.com/elm-lang/elm-compiler/issues/1473#issuecomment-245704142
-    if [ ! -d sysconfcpus/bin ];
-    then
-      git clone https://github.com/obmarg/libsysconfcpus.git;
-      cd libsysconfcpus;
-      ./configure --prefix=$TRAVIS_BUILD_DIR/sysconfcpus;
-      make && make install;
-      cd ..;
-    fi
-
 install:
-  - travis_retry $TRAVIS_BUILD_DIR/sysconfcpus/bin/sysconfcpus -n 2 make -B setup
+  - nix-shell
 
 script:
-  - $TRAVIS_BUILD_DIR/sysconfcpus/bin/sysconfcpus -n 2 make ci
-
-notifications:
-  email: false
+  - nix-shell --pure --run 'shake --verbose ci'

Makefile

@@ -1,104 +0,0 @@
-SHELL:=env PATH=${PATH} /bin/sh
-export DEPRECATED_MODULES=Html,Accessibility,Accessibility.Aria,Accessibility.Key,Accessibility.Landmark,Accessibility.Live,Accessibility.Role,Accessibility.Style,Accessibility.Widget
-
-.PHONY: test
-test: node_modules tests/elm-verify-examples.json
-	npx elm-verify-examples
-	npx elm-test
-	make axe-report
-	make percy-tests
-	make deprecated-imports-report
-
-tests/elm-verify-examples.json: $(shell find src -name '*.elm') elm.json
-	jq --indent 4 '{ root: "../src", tests: .["exposed-modules"] }' elm.json > $@
-
-tests/axe-report.json: public script/run-axe.sh script/axe-puppeteer.js
-	script/run-axe.sh > $@
-
-.PHONY: axe-report
-axe-report: tests/axe-report.json script/format-axe-report.sh script/axe-report.jq
-	script/format-axe-report.sh $<
-
-.PHONY: percy-tests
-percy-tests:
-	script/percy-tests.sh
-
-tests/deprecated-imports-report.txt: $(shell find src tests -type f) script/deprecated-imports.py
-	script/deprecated-imports.py report > $@
-
-script/deprecated-imports.csv: $(shell find src tests -type f) script/deprecated-imports.py
-	script/deprecated-imports.py --imports-file $@ update
-
-.PHONY: deprecated-imports-report
-deprecated-imports-report: tests/deprecated-imports-report.txt script/deprecated-imports.py
-	@cat tests/deprecated-imports-report.txt
-	@script/deprecated-imports.py --check-message-fix-command='make script/deprecated-imports.csv' check
-
-.PHONY: checks
-checks:
-	script/check-exposed.py
-
-.PHONY: diff
-diff: node_modules
-	true
-# 	if (npx elm diff | tee /dev/stderr | grep -q MAJOR); then echo "MAJOR changes are not allowed!"; exit 1; fi
-
-.PHONY: format
-format: node_modules
-	npx elm-format --validate src && npx elm-format --validate --elm-version=0.19 tests styleguide-app
-
-.PHONY: clean
-clean:
-	rm -rf node_modules styleguide-app/elm.js styleguide-app/bundle.js $(shell find . -type d -name 'elm-stuff') public
-
-.PHONY: styleguide-app
-styleguide-app:
-	./script/develop.sh
-
-documentation.json: node_modules
-	npx elm make --docs $@
-
-styleguide-app/bundle.js: lib/index.js styleguide-app/manifest.js node_modules
-	npx browserify --entry styleguide-app/manifest.js --outfile styleguide-app/bundle.js
-
-styleguide-app/elm.js: styleguide-app/bundle.js $(shell find src styleguide-app -type f -name '*.elm')
-	cd styleguide-app && npx elm make Main.elm --output=$(@F)
-
-# for publishing styleguide
-
-# We don't want to have to generate new rules for every single asset, so we find
-# all the ones that exist (`STYLEGUIDE_ASSETS`) then replace the roots
-# (`PUBLIC_ASSETS`). The `%` wildcard works like it does in `public/%` below.
-STYLEGUIDE_ASSETS=$(shell find styleguide-app/assets -type f)
-PUBLIC_ASSETS=$(STYLEGUIDE_ASSETS:styleguide-app/assets/%=public/assets/%)
-
-public: public/index.html public/elm.js public/bundle.js $(PUBLIC_ASSETS)
-	touch -m $@
-
-# wildcard rule: % on the left-hand side will be matched and replaced on the
-# right-hand side. So `public/index.html` depends on `styleguide-app/index.html`
-#
-# - automatic variables: `$@` is the target (left-hand side of the rule.) `$<`
-#   is the first dependency.
-# - about the leading `@` in `mkdir`: leading `@` turns off echoing the
-#   command. We're just reducing log spam here.
-# - about `$(@D)`: $@ gets the target (left-hand side of the rule). Any
-#   automatic variable plus `D` gets the directory of that file, so `$(@D)` is
-#   the target's directory.
-public/%: styleguide-app/%
-	@mkdir -p $(@D)
-	cp $< $@
-
-# plumbing
-
-node_modules: package.json
-	npm install
-	touch -m $@
-
-# special targets for travis, but anyone can use them, really.
-
-.PHONY: setup
-setup: node_modules
-
-.PHONY: ci
-ci: checks test format documentation.json diff styleguide-app/elm.js

README.md

@@ -64,12 +64,12 @@ If you'd like to test your widget in the monolith before publishing, run `script
 Run tests with
 
 ```
-make test
+shake test
 ```
 
 ### CI (Travis)
 
-Travis will run `make ci` to verify everything looks good.
+Travis will run `shake ci` to verify everything looks good.
 You can run this locally to catch errors before you push!
 
 ## Deploying

Shakefile.hs

@@ -0,0 +1,153 @@
+import Development.Shake
+import Development.Shake.Command
+import Development.Shake.FilePath
+import Development.Shake.Util
+import qualified GHC.IO.Encoding
+import qualified System.IO
+
+main :: IO ()
+main = do
+  -- Work around `hGetContents: invalid argument (invalid byte sequence)` bug on
+  -- Nix: https://github.com/dhall-lang/dhall-haskell/issues/865
+  GHC.IO.Encoding.setLocaleEncoding System.IO.utf8
+  shakeArgs
+    shakeOptions
+      { shakeFiles = "_build",
+        shakeThreads = 0,
+        shakeChange = ChangeModtimeAndDigest,
+        -- we ignore a lot of generated/downloaded dependency files so
+        -- that the output of `shake --lint-fsatrace` is usable. There
+        -- are probably a few untracked dependencies due to these ignores
+        -- (in particular relying on scripts in `node_modules`) but the
+        -- additional benefits are marginal compared to the effort required
+        -- to get everything 100% buttoned down. Long term, it'd be better to
+        -- move node dependencies into nix (either by using proper packages
+        -- where available or npm2nix where not.)
+        shakeLintIgnore =
+          [ "node_modules/**/*",
+            "elm-stuff/**/*",
+            "styleguide-app/elm-stuff/**/*"
+          ]
+      }
+    $ do
+      -- phonies. These provide a nice public API for using shake (`shake
+      -- clean`, `shake test`, etc.)
+      phony "clean" $ do
+        removeFilesAfter "elm-stuff" ["//*"]
+        removeFilesAfter "log" ["//*"]
+        removeFilesAfter "node_modules" ["//*"]
+        removeFilesAfter "public" ["//*"]
+        removeFilesAfter "styleguide-app" ["elm.js", "bundle.js", "elm-stuff"]
+
+      phony "public" $ need ["log/public.txt"]
+
+      phony "test" $ do
+        need
+          [ "log/npm-install.txt",
+            "tests/elm-verify-examples.json",
+            "log/elm-verify-examples.txt",
+            "log/elm-test.txt",
+            "log/axe-report.txt",
+            "log/percy-tests.txt",
+            "log/deprecated-imports-report.txt",
+            "log/check-exposed.txt",
+            "log/format.txt",
+            "log/documentation.json"
+          ]
+
+      phony "ci" $ do need ["test", "public"]
+
+      -- things that should be kept under version control
+      "tests/elm-verify-examples.json" %> \out -> do
+        need ["elm.json"]
+        cmd (WithStdout True) (FileStdout out) "jq" "--indent" "4" ["{ root: \"../src\", tests: .[\"exposed-modules\"] }"] "elm.json"
+
+      "script/deprecated-imports.csv" %> \out -> do
+        getEnv "DEPRECATED_MODULES"
+        elmFiles <- getDirectoryFiles "." ["src/**/*.elm", "tests/**/*.elm"]
+        need (["elm.json", "script/deprecated-imports.py"] ++ elmFiles)
+        cmd_ "script/deprecated-imports.py" "--imports-file" out "update"
+
+      -- temporary files, used to produce CI reports
+      "log/elm-test.txt" %> \out -> do
+        elmFiles <- getDirectoryFiles "." ["tests/**/*.elm"]
+        -- I'm not sure why elm-test needs package.json, but fsatracing it
+        -- reveals the dep, so in it goes!
+        need (["package.json", "elm.json"] ++ elmFiles)
+        cmd (WithStdout True) (FileStdout out) "elm-test"
+
+      "log/elm-verify-examples.txt" %> \out -> do
+        elmFiles <- getDirectoryFiles "." ["src/**/*.elm"]
+        need (["tests/elm-verify-examples.json"] ++ elmFiles)
+        cmd (WithStdout True) (FileStdout out) "elm-verify-examples"
+
+      "log/format.txt" %> \out -> do
+        let placesToLook = ["src", "tests", "styleguide-app"]
+        elmFiles <- getDirectoryFiles "." (map (\place -> place </> "**" </> "*.elm") placesToLook)
+        need elmFiles
+        cmd (WithStdout True) (FileStdout out) "elm-format" "--validate" placesToLook
+
+      "log/percy-tests.txt" %> \out -> do
+        percyToken <- getEnv "PERCY_TOKEN"
+        case percyToken of
+          Nothing -> do
+            writeFileChanged out "Skipped running Percy tests, PERCY_TOKEN not set."
+          Just _ -> do
+            need ["log/npm-install.txt"]
+            cmd (WithStdout True) (FileStdout out) "script/percy-tests.js"
+
+      "log/axe-report.json" %> \out -> do
+        need ["log/npm-install.txt", "script/run-axe.sh", "script/axe-puppeteer.js", "log/public.txt"]
+        cmd (WithStdout True) (FileStdout out) "script/run-axe.sh"
+
+      "log/axe-report.txt" %> \out -> do
+        need ["log/axe-report.json", "script/format-axe-report.sh", "script/axe-report.jq"]
+        cmd (WithStdout True) (FileStdout out) "script/format-axe-report.sh" "log/axe-report.json"
+
+      "log/deprecated-imports-report.txt" %> \out -> do
+        getEnv "DEPRECATED_MODULES"
+        elmFiles <- getDirectoryFiles "." ["src/**/*.elm", "tests/**/*.elm"]
+        need (["elm.json", "script/deprecated-imports.py"] ++ elmFiles)
+        cmd (WithStdout True) (FileStdout out) "script/deprecated-imports.py" "check"
+
+      "log/check-exposed.txt" %> \out -> do
+        elmFiles <- getDirectoryFiles "." ["src/**/*.elm"]
+        need (["elm.json", "script/check-exposed.py"] ++ elmFiles)
+        cmd (WithStdout True) (FileStdout out) "script/check-exposed.py"
+
+      "log/documentation.json" %> \out -> do
+        elmFiles <- getDirectoryFiles "." ["src/**/*.elm"]
+        need elmFiles
+        cmd_ "elm" "make" "--docs" out
+
+      "public/bundle.js" %> \out -> do
+        libJsFiles <- getDirectoryFiles "." ["lib/**/*.js"]
+        need (["package.json", "lib/index.js", "styleguide-app/manifest.js", "log/npm-install.txt"] ++ libJsFiles)
+        cmd_ "./node_modules/.bin/browserify" "--entry" "styleguide-app/manifest.js" "--outfile" out
+
+      "public/elm.js" %> \out -> do
+        elmSources <- getDirectoryFiles "." ["styleguide-app/**/*.elm", "src/**/*.elm"]
+        need elmSources
+        cmd_ (Cwd "styleguide-app") "elm" "make" "Main.elm" "--output" (".." </> out)
+
+      "public/**/*" %> \out ->
+        copyFileChanged (replaceDirectory1 out "styleguide-app") out
+
+      "log/public.txt" %> \out -> do
+        styleguideAssets <- getDirectoryFiles ("styleguide-app" </> "assets") ["**/*"]
+        need
+          ( ["public/index.html", "public/elm.js", "public/bundle.js"]
+              ++ map (("public" </> "assets") </>) styleguideAssets
+          )
+        writeFileChanged out "built styleguide app successfully"
+
+      -- dev deps we get dynamically instead of from Nix (frowny face)
+      "log/npm-install.txt" %> \out -> do
+        -- npm looks in some unrelated files for whatever reason. We mark
+        -- them as used here to avoid getting linter errors.
+        gitHeads <- getDirectoryFiles "." [".git/refs/heads/*"]
+        trackRead (["README.md", ".git/HEAD"] ++ gitHeads)
+
+        -- now that we've satisfied the linter, let's build.
+        need ["package.json", "package-lock.json"]
+        cmd (WithStdout True) (FileStdout out) (FileStderr out) "npm" "install"

netlify.toml

@@ -1,3 +1,3 @@
 [build]
-command = "make public"
+command = "script/netlify.sh"
 publish = "public"

package.json

@@ -28,10 +28,6 @@
   "devDependencies": {
     "@percy/script": "^1.0.2",
     "browserify": "16.2.3",
-    "elm": "latest-0.19.1",
-    "elm-format": "0.8.3",
-    "elm-test": "0.19.1",
-    "elm-verify-examples": "^4.0.0",
     "request": "^2.88.0"
   },
   "dependencies": {

script/develop.sh

@@ -5,7 +5,9 @@ if test -d public; then
     rm -rf public
 fi
 
-make -j public
+SHAKE_TARGET="${1:-public}"
+
+shake --compact "$SHAKE_TARGET"
 
 cat <<EOF
 == 👋 Hello! ==================================================================
@@ -15,9 +17,7 @@ changes, I'll try to be smart about what should change (things end up in the
 "public" directory if you want to check my work.) If you remove a file and it's
 still showing up, delete the "public" directory and restart me.
 
-To rebuild manually, hit SPC.
-
-To quit, hit "q" or ctrl-c.
+To quit, hit ctrl-c.
 
 == thaaat's it from me! =======================================================
 
@@ -32,4 +32,4 @@ cleanup() {
 trap cleanup EXIT INT
 
 # start a watcher. This loops forever, so we don't need to loop ourselves.
-find src styleguide-app -type f -not -ipath '*elm-stuff*' | entr -c -p make public
+watchexec --clear --postpone -- shake --compact "$SHAKE_TARGET"

script/format-axe-report.sh

@@ -10,10 +10,8 @@ fi
 jq -r -f script/axe-report.jq "$JSON_FILE"
 
 NUM_ERRORS="$(jq '.violations | map(.nodes | length) | add' "$JSON_FILE")"
-if [ -z "$NUM_ERRORS" ];
+if test "$NUM_ERRORS" -lt 4;
 then
     echo "$NUM_ERRORS accessibility errors"
-    echo
-    echo "To see these errors, run 'make axe-report > errors.new' and open 'errors.new'"
     exit 1
 fi

script/netlify.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -xeuo pipefail
+
+# Netlify doesn't support building stuff via Haskell or
+# Nix. Those things are vaguely on the horizon (check the issues at
+# https://github.com/netlify/build-image) but for now it's way simpler to just
+# accept that Netlify will need separate instructions.
+
+# get our dependencies (--ignore-scripts=false is needed for puppeteer)
+npm install --ignore-scripts=false
+npm install elm
+
+# make sure we're building into a clean folder
+if test -d public; then rm -rf public; fi
+mkdir public
+
+# build the interactive parts
+(cd styleguide-app && npx elm make Main.elm --output ../public/elm.js)
+npx browserify --entry styleguide-app/manifest.js --outfile public/bundle.js
+
+# copy assets
+cp -r styleguide-app/assets public/assets

script/nixos-shell.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VOLUME_NAME="noredink-ui-nixos-shell-nix"
+
+if ! docker volume ls | grep -q "$VOLUME_NAME"; then
+  docker volume create "$VOLUME_NAME"
+fi
+
+docker run \
+  --interactive \
+  --tty \
+  --mount "type=bind,source=$(pwd),target=/app" \
+  --mount "type=volume,source=$VOLUME_NAME,target=/nix" \
+  --workdir /app \
+  lnl7/nix:latest \
+  nix-shell --command 'mkdir -p /etc/ssl/certs && ln -s $NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt && return'

script/run-axe.sh

@@ -9,4 +9,13 @@ cleanup() {
 }
 trap cleanup EXIT INT
 
+# Wait for the python server to launch since we're launching it async
+>&2 echo "Waiting for web server to launch on port 8000..."
+while ! nc -z localhost 8000 &>/dev/null; do
+  >&2 echo "Web server not up yet... waiting some more..."
+  sleep 1
+done
+>&2 echo "Web server launched"
+
+
 node script/axe-puppeteer.js http://localhost:8000

shell.nix

@@ -3,22 +3,38 @@ let
   nixpkgs = import sources.nixpkgs { };
   niv = import sources.niv { };
 in with nixpkgs;
-
 stdenv.mkDerivation {
   name = "noredink-ui";
   buildInputs = [
     # base dependencies
     git
-    gnumake
     niv.niv
     jq
 
+    # building with Shake
+    haskellPackages.ormolu
+    (haskellPackages.ghcWithPackages (ps: [ ps.shake ]))
+
     # node dependencies
     nodejs
     nodePackages.npm
 
+    # elm dependencies
+    elmPackages.elm
+    elmPackages.elm-format
+    elmPackages.elm-test
+    elmPackages.elm-verify-examples
+
     # preview dependencies
-    entr
     python3
-  ];
+    watchexec
+
+    # stuff we need for running builds in a `nix-shell --pure` environment.
+    which
+    netcat-gnu
+  ] ++ lib.optionals stdenv.isLinux [ pkgs.fsatrace pkgs.strace pkgs.cacert ];
+
+  # environment variables
+  DEPRECATED_MODULES =
+    "Html,Accessibility,Accessibility.Aria,Accessibility.Key,Accessibility.Landmark,Accessibility.Live,Accessibility.Role,Accessibility.Style,Accessibility.Widget";
 }

src/Nri/Ui/Page/V3.elm

@@ -43,7 +43,7 @@ type RecoveryText
 notFound : DefaultPage msg -> Html msg
 notFound defaultPage =
     view
-        { emoji = "\u{1F914}"
+        { emoji = "🤔"
         , title = "We couldn’t find that!"
         , subtitle = "Feel free to browse around, or check out our help center."
         , defaultPage = Just defaultPage
@@ -82,7 +82,7 @@ blocked details =
 noPermission : DefaultPage msg -> Html msg
 noPermission defaultPage =
     view
-        { emoji = "\u{1F910}"
+        { emoji = "🤐"
         , title = "You do not have access to this page!"
         , subtitle = "Talk to a site administrator if you believe you should have access to this page."
         , defaultPage = Just defaultPage