Merge pull request #1372 from NoRedInk/wat-589-build-elm-docs-and-the-component-catalog

build Elm docs and the component catalog Elm app in Buck

.buckconfig

@@ -1,6 +1,7 @@
 [repositories]
 root = .
 prelude = prelude
+prelude-nri = prelude-nri
 toolchains = toolchains
 none = none
 

.github/workflows/buck2-ci.yml

@@ -18,7 +18,7 @@ jobs:
           key: ${{ runner.os }}-buck-out
 
       - name: buck2 build
-        run: script/buck2 build //...
+        run: script/buck2 build //... --show-output
 
       - name: buck2 test
         run: script/buck2 test //...

BUCK

@@ -1,7 +1,14 @@
 # A list of available rules and their signatures can be found here: https://buck2.build/docs/api/rules/
+load("@prelude-nri//:elm.bzl", "elm_docs")
 
-genrule(
-    name = "hello_world",
-    out = "out.txt",
-    cmd = "echo BUILT BY BUCK2> $OUT",
+elm_docs(
+    name = "docs.json",
+    elm_json = "elm.json",
+    src = "src",
+)
+
+filegroup(
+    name = "src",
+    srcs = glob(["src/**/*.elm"]),
+    visibility = ["//component-catalog:app"]
 )

component-catalog/BUCK

@@ -0,0 +1,12 @@
+load("@prelude-nri//:elm.bzl", "elm_app")
+
+elm_app(
+    name = "app",
+    elm_json = "elm.json",
+    main = "src/Main.elm",
+    out = "elm.js",
+    source_directories = {
+        "../src": "//:src",
+        "src": "src",
+    }
+)

prelude-nri/elm.bzl

@@ -0,0 +1,106 @@
+load("//elm:toolchain.bzl", "ElmToolchainInfo")
+load("@prelude//python:toolchain.bzl", "PythonToolchainInfo")
+
+def _elm_docs_impl(ctx: "context") -> [DefaultInfo.type]:
+    build = ctx.actions.declare_output("build", dir = True)
+    docs = ctx.actions.declare_output(ctx.attrs.out)
+
+    elm_toolchain = ctx.attrs._elm_toolchain[ElmToolchainInfo]
+
+    cmd = cmd_args(
+        ctx.attrs._python_toolchain[PythonToolchainInfo].interpreter,
+        elm_toolchain.isolated_compile[DefaultInfo].default_outputs,
+        ctx.attrs.elm_json,
+        "--build-dir", build.as_output(),
+        "--elm-compiler", elm_toolchain.elm,
+        "--verbose",
+        "docs",
+        "--out", docs.as_output(),
+        "--src", ctx.attrs.src,
+    )
+
+    ctx.actions.run(
+        cmd,
+        category = "elm",
+        no_outputs_cleanup = True,
+    )
+
+    return [DefaultInfo(default_output = docs)]
+
+elm_docs = rule(
+    impl = _elm_docs_impl,
+    attrs = {
+        "out": attrs.string(default="docs.json"),
+        "elm_json": attrs.source(),
+        "src": attrs.source(allow_directory = True),
+        "_elm_toolchain": attrs.toolchain_dep(
+            default="toolchains//:elm",
+            providers=[ElmToolchainInfo]
+        ),
+        "_python_toolchain": attrs.toolchain_dep(
+            default="toolchains//:python",
+            providers=[PythonToolchainInfo]
+        ),
+    }
+)
+
+def _elm_app_impl(ctx: "context") -> [DefaultInfo.type]:
+    build = ctx.actions.declare_output("build", dir = True)
+    out = ctx.actions.declare_output(ctx.attrs.out)
+
+    elm_toolchain = ctx.attrs._elm_toolchain[ElmToolchainInfo]
+
+    cmd = cmd_args(
+        ctx.attrs._python_toolchain[PythonToolchainInfo].interpreter,
+        elm_toolchain.isolated_compile[DefaultInfo].default_outputs,
+        ctx.attrs.elm_json,
+        "--build-dir", build.as_output(),
+        "--elm-compiler", elm_toolchain.elm,
+        "--verbose",
+        "make",
+        ctx.attrs.main,
+        "--output", out.as_output(),
+    )
+
+    for (name, value) in ctx.attrs.source_directories.items():
+        cmd.add(cmd_args(value, format="--source-directory=" + name + "={}"))
+
+    if ctx.attrs.debug and ctx.attrs.optimize:
+        fail("Only one of `optimize` and `debug` may be true!")
+
+    if ctx.attrs.debug:
+        cmd.add("--debug")
+
+    if ctx.attrs.optimize:
+        cmd.add("--optimize")
+
+    ctx.actions.run(
+        cmd,
+        category = "elm",
+        no_outputs_cleanup = True,
+    )
+
+    return [DefaultInfo(default_output = out)]
+
+elm_app = rule(
+    impl = _elm_app_impl,
+    attrs = {
+        "out": attrs.string(default = "app.js"),
+        "elm_json": attrs.source(),
+        "main": attrs.source(),
+        "source_directories": attrs.dict(
+            attrs.string(),
+            attrs.source(allow_directory=True),
+        ),
+        "debug": attrs.bool(default = False),
+        "optimize": attrs.bool(default = False),
+        "_elm_toolchain": attrs.toolchain_dep(
+            default="toolchains//:elm",
+            providers=[ElmToolchainInfo]
+        ),
+        "_python_toolchain": attrs.toolchain_dep(
+            default="toolchains//:python",
+            providers=[PythonToolchainInfo]
+        ),
+    }
+)

prelude-nri/elm/BUCK

@@ -0,0 +1,4 @@
+export_file(
+    name = "isolated_compile.py",
+    visibility = ["PUBLIC"],
+)

prelude-nri/elm/isolated_compile.py

@@ -0,0 +1,287 @@
+#!/usr/bin/env python3
+"""
+Build an Elm app, but isolate the working directory so that multiple Elm
+compiler invocations can run in parallel without corrupting `.elmi` files.
+"""
+from argparse import ArgumentParser, ArgumentTypeError
+import json
+import logging
+import os
+import os.path
+import shutil
+from subprocess import Popen, PIPE
+import sys
+import tempfile
+
+
+def symlink_if_necessary(source, target):
+    """
+    Do the same thing as `os.symlink`, but with these edge cases checked:
+
+     - If the target already exists and points to the source, do nothing.
+     - If the target already exists and points somewhere else, remove it
+       and relink.
+    """
+    if os.path.exists(target):
+        if os.path.isfile(target):
+            logging.debug(f"`{target}` is a regular file")
+
+        elif os.readlink(target) == source:
+            logging.debug(f"`{target}` already points to `{source}`")
+            return
+
+        os.unlink(target)
+
+    logging.info(f"linking `{target}` to `{source}`")
+    os.symlink(source, target)
+
+
+def write_if_necessary(target, content):
+    """
+    Read the target file and check the content. If it's the same, don't write.
+    Otherwise, replace it.
+    """
+    if os.path.exists(target):
+        with open(target, "r") as fh:
+            if fh.read() == content:
+                logging.debug(f"`{target}` already had the requested content.")
+                return
+
+    logging.info(f"writing `{target}`")
+    with open(target, "w") as fh:
+        fh.write(content)
+
+
+def run_docs(args):
+    """
+    Compile JSON docs for an Elm library.
+    """
+    logging.info(f"copying {args.elm_json} to {args.build_dir}")
+    symlink_if_necessary(args.elm_json, os.path.join(args.build_dir, "elm.json"))
+
+    # for libraries, the Elm compiler always assumes the source lives in a
+    # directory named "src". We have one of those from the arguments, so let's
+    # set it up under the build root. For the sake of being able to reuse this
+    # build directory's `elm-stuff`, we take care to repoint the symlink if
+    # necessary.
+    symlink_if_necessary(os.path.abspath(args.src), os.path.join(args.build_dir, "src"))
+
+    command = [
+        args.elm_compiler,
+        "make",
+        "--docs",
+        # since we're changing cwd, we need to output the absolute path
+        # instead of a (potentially) relative one.
+        os.path.abspath(args.output),
+    ]
+    logging.debug(f"running {command} in `{args.build_dir}`")
+    process = Popen(command, cwd=args.build_dir)
+    process.communicate()
+
+    return process.returncode
+
+
+def run_make(args):
+    """
+    Compile an Elm app to HTML or JavaScript.
+
+    Our basic approach here is to symlink all the source directories and the
+    Main.elm file into place and modify elm.json to look in those places. This
+    means that we can use the build directory as a working directory so we get
+    an isolated elm-stuff directory.
+    """
+    logging.debug(f"reading `{args.elm_json}`")
+    with open(args.elm_json, "r") as fh:
+        elm_json = json.load(fh)
+
+    ##########################################################################
+    # STEP 1: symlink entries in `source-directories` to the right locations #
+    ##########################################################################
+
+    try:
+        original_source_directories = elm_json["source-directories"]
+    except KeyError:
+        logging.error(
+            f'`{args.elm_json}` did not have a "source-directories" entry. Is it a package?'
+        )
+        return 1
+
+    source_directory_replacements = dict(
+        (sd.src, sd.target) for sd in args.source_directory or []
+    )
+
+    # we're creating a mapping here so we can change the Main file to be relative
+    # to the symlinks later.
+    new_source_directories = {}
+    for i, directory in enumerate(original_source_directories):
+        try:
+            replacement = source_directory_replacements[directory]
+        except KeyError:
+            logging.error(
+                f"I don't have a replacement path for `{directory}`. Please specify one with --source-directory {directory}=real-path-to-directory"
+            )
+            return 1
+
+        dest = f"src-{i}"
+
+        # special case: if the replacement is inside `buck-out` and contains
+        # only a single directory, we're being passed a Buck2 artifact directory
+        # and should take the single directory inside.
+        if replacement.startswith("buck-out"):
+            contents = os.listdir(replacement)
+            if len(contents) == 1 and os.path.isdir(contents[0]):
+                logging.info(
+                    f"I think `{replacement}` is a Buck2 artifact, so I'm symlinking `{contents[0]}` inside it instead."
+                )
+                replacement = os.path.join(replacement, contents[0])
+
+        symlink_if_necessary(
+            os.path.abspath(replacement), os.path.join(args.build_dir, dest)
+        )
+
+        new_source_directories[replacement] = dest
+
+    logging.debug(f"new source directories: {new_source_directories}")
+
+    #############################################################
+    # STEP 2: Modify and write `elm.json` to the right location #
+    #############################################################
+
+    elm_json["source-directories"] = list(new_source_directories.values())
+    new_elm_json_path = os.path.join(args.build_dir, "elm.json")
+    logging.debug(f"writing `{new_elm_json_path}`")
+
+    # the compiler will do a full rebuild anytime the `elm.json` file changes.
+    # Writing only if necessary about halves the runtime for this script for
+    # noredink-ui's component catalog, with more expected savings the bigger the
+    # app gets.
+    write_if_necessary(
+        new_elm_json_path,
+        json.dumps(elm_json, indent=4),
+    )
+
+    ##########################################################
+    # STEP 3: Make sure we're poining at the right main file #
+    ##########################################################
+
+    main = args.main
+    replaced = False
+    logging.debug(f"original main: {main}")
+    for original, replacement in new_source_directories.items():
+        if main.startswith(original):
+            main = os.path.join(replacement, main[len(original) + 1 :])
+            logging.debug(f"using `{main}` instead of `{args.main}`")
+            replaced = True
+            break
+
+    if not replaced:
+        # it's fine to build a main file outside a source directory, but let's
+        # take the absolute path so we can make sure to get it.
+        main = os.path.abspath(main)
+
+    #####################################################
+    # STEP 4: Prepare and run the `elm make` invocation #
+    #####################################################
+
+    command = [
+        args.elm_compiler,
+        "make",
+        main,
+        "--output",
+        os.path.abspath(args.output),
+    ]
+
+    if args.debug and args.optimize:
+        print("Only one of --debug or --optimize can be set.")
+        return 1
+
+    if args.debug:
+        command.append("--debug")
+
+    if args.optimize:
+        command.append("--optimize")
+
+    logging.debug(f"running {command} in `{args.build_dir}`")
+    process = Popen(command, cwd=args.build_dir)
+    process.communicate()
+
+    return process.returncode
+
+
+class SourceDirectory:
+    def __init__(self, s):
+        try:
+            src, target = s.split("=")
+            self.src = src
+            self.target = target
+        except:
+            raise ArgumentTypeError("A pair must be `a=b`")
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description=__doc__)
+    parser.add_argument("elm_json", help="Location of the elm.json")
+    parser.add_argument(
+        "--build-dir",
+        help="Where to perform the build. Should be an empty directory, or one you've already run a build in.",
+    )
+    parser.add_argument(
+        "--elm-compiler", help="path to the Elm compiler", default="elm"
+    )
+    parser.add_argument(
+        "--verbose", help="Turn on verbose logging", action="store_true"
+    )
+
+    subparsers = parser.add_subparsers(required=True)
+
+    docs = subparsers.add_parser("docs", help=run_docs.__doc__)
+    docs.set_defaults(func=run_docs)
+    docs.add_argument(
+        "--output", help="Path for the resulting docs JSON file", default="docs.json"
+    )
+    docs.add_argument("--src", help="Path to library source", default="src")
+
+    make = subparsers.add_parser("make", help=run_make.__doc__)
+    make.set_defaults(func=run_make)
+    make.add_argument("main", help="Main.elm file to build")
+    make.add_argument(
+        "--output",
+        help="Path for the resulting JavaScript or HTML file",
+        default="app.js",
+    )
+    make.add_argument(
+        "--source-directory",
+        type=SourceDirectory,
+        metavar="SRC=TARGET",
+        action="append",
+    )
+    make.add_argument("--debug", action="store_true", help="Build in debug mode")
+    make.add_argument("--optimize", action="store_true", help="Build in optimize mode")
+
+    args = parser.parse_args()
+
+    # logging
+    logger = logging.getLogger()
+    handler = logging.StreamHandler(sys.stderr)
+    formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    logger.setLevel(logging.DEBUG if args.verbose else logging.WARNING)
+
+    # prep
+    if os.path.exists(args.elm_compiler):
+        args.elm_compiler = os.path.abspath(args.elm_compiler)
+
+    # run!
+    if args.build_dir is None:
+        with tempfile.TemporaryDirectory() as temp:
+            logging.warning(
+                f"building in temporary directory `{temp}`, which will be cleaned up after the build. If you want a persistent place to build, pass it in with `--build-dir`!"
+            )
+            args.build_dir = temp
+            sys.exit(args.func(args))
+    else:
+        if not os.path.exists(args.build_dir):
+            logging.info(f"creating build dir at `{args.build_dir}`")
+            os.mkdir(args.build_dir)
+        sys.exit(args.func(args))

prelude-nri/elm/toolchain.bzl

@@ -0,0 +1,48 @@
+ElmToolchainInfo = provider(fields=[
+    "elm",
+    "isolated_compile",
+])
+
+def _system_elm_toolchain_impl(ctx) -> [[DefaultInfo.type, ElmToolchainInfo.type]]:
+    """
+    An Elm toolchain that assumes the current environment has all the binaries
+    it needs in PATH.
+    """
+    return [
+        DefaultInfo(),
+        ElmToolchainInfo(
+            elm = RunInfo(args = ["elm"]),
+            isolated_compile = ctx.attrs._isolated_compile,
+        ),
+    ]
+
+
+system_elm_toolchain = rule(
+    impl = _system_elm_toolchain_impl,
+    attrs = {
+        "_isolated_compile": attrs.dep(default="prelude-nri//elm:isolated_compile.py"),
+    },
+    is_toolchain_rule = True,
+)
+
+def _elm_toolchain_impl(ctx: "context") -> [[DefaultInfo.type, ElmToolchainInfo.type]]:
+    """
+    An Elm toolchain which you can source binaries from wherever it makes sense
+    to you.
+    """
+    return [
+        DefaultInfo(),
+        ElmToolchainInfo(
+            elm = ctx.attrs.elm[RunInfo],
+            isolated_compile = ctx.attrs._isolated_compile,
+        ),
+    ]
+
+elm_toolchain = rule(
+    impl = _elm_toolchain_impl,
+    attrs = {
+        "elm": attrs.dep(providers = [RunInfo]),
+        "_isolated_compile": attrs.dep(default="prelude-nri//elm:isolated_compile.py"),
+    },
+    is_toolchain_rule = True,
+)

script/buck2

@@ -9,8 +9,20 @@ fi
 if ! test -f buck-out/buck2; then
   case "$(uname -s)" in
     Darwin)
-      # TODO: adjust for processor type
-      curl -L https://github.com/facebook/buck2/releases/download/latest/buck2-aarch64-apple-darwin.zst | zstd -d > buck-out/buck2
+      case "$(arch)" in
+        arm64)
+          curl -L https://github.com/facebook/buck2/releases/download/latest/buck2-aarch64-apple-darwin.zst | zstd -d > buck-out/buck2
+          ;;
+
+        i386)
+          curl -L https://github.com/facebook/buck2/releases/download/latest/buck2-x86_64-apple-darwin.zst | zstd -d > buck-out/buck2
+          ;;
+
+        *)
+          echo "I don't know how to get a download for $(arch) on macOS"
+          exit 1
+          ;;
+      esac
       ;;
 
     Linux)

toolchains/BUCK

@@ -1,6 +1,47 @@
 load("@prelude//toolchains:genrule.bzl", "system_genrule_toolchain")
+load("@prelude//toolchains:python.bzl", "system_python_bootstrap_toolchain", "system_python_toolchain")
+load("@prelude-nri//elm:toolchain.bzl", "elm_toolchain")
 
 system_genrule_toolchain(
     name = "genrule",
     visibility = ["PUBLIC"],
 )
+
+system_python_toolchain(
+    name = "python",
+    visibility = ["PUBLIC"],
+)
+
+system_python_bootstrap_toolchain(
+    name = "python_bootstrap",
+    visibility = ["PUBLIC"],
+)
+
+elm_toolchain(
+    name = "elm",
+    visibility = ["PUBLIC"],
+    elm = ":elm_compiler_binary",
+)
+
+ELM_COMPILER_URL = select({
+    "config//os:linux": "https://github.com/elm/compiler/releases/download/0.19.1/binary-for-linux-64-bit.gz",
+    "config//os:macos": "https://github.com/elm/compiler/releases/download/0.19.1/binary-for-mac-64-bit.gz",
+})
+
+ELM_COMPILER_SHA256 = select({
+    "config//os:linux": "e44af52bb27f725a973478e589d990a6428e115fe1bb14f03833134d6c0f155c",
+    "config//os:macos": "05289f0e3d4f30033487c05e689964c3bb17c0c48012510dbef1df43868545d1",
+})
+
+http_file(
+    name = "elm_compiler_archive",
+    urls = [ELM_COMPILER_URL],
+    sha256 = ELM_COMPILER_SHA256,
+)
+
+genrule(
+    name = "elm_compiler_binary",
+    cmd = "gzip --decompress --stdout --keep $(location :elm_compiler_archive) > $OUT && chmod +x $OUT",
+    out = "elm",
+    executable = True,
+)