diff --git a/.buckconfig b/.buckconfig index 2ad80471..cdb5ec1a 100644 --- a/.buckconfig +++ b/.buckconfig @@ -1,6 +1,7 @@ [repositories] root = . prelude = prelude +prelude-nri = prelude-nri toolchains = toolchains none = none diff --git a/.github/workflows/buck2-ci.yml b/.github/workflows/buck2-ci.yml index 71043008..47d37a61 100644 --- a/.github/workflows/buck2-ci.yml +++ b/.github/workflows/buck2-ci.yml @@ -18,7 +18,7 @@ jobs: key: ${{ runner.os }}-buck-out - name: buck2 build - run: script/buck2 build //... + run: script/buck2 build //... --show-output - name: buck2 test run: script/buck2 test //... diff --git a/BUCK b/BUCK index 1cb6b382..38f3c6f7 100644 --- a/BUCK +++ b/BUCK @@ -1,7 +1,14 @@ # A list of available rules and their signatures can be found here: https://buck2.build/docs/api/rules/ +load("@prelude-nri//:elm.bzl", "elm_docs") -genrule( - name = "hello_world", - out = "out.txt", - cmd = "echo BUILT BY BUCK2> $OUT", +elm_docs( + name = "docs.json", + elm_json = "elm.json", + src = "src", +) + +filegroup( + name = "src", + srcs = glob(["src/**/*.elm"]), + visibility = ["//component-catalog:app"] ) diff --git a/component-catalog/BUCK b/component-catalog/BUCK new file mode 100644 index 00000000..c0875d9b --- /dev/null +++ b/component-catalog/BUCK @@ -0,0 +1,12 @@ +load("@prelude-nri//:elm.bzl", "elm_app") + +elm_app( + name = "app", + elm_json = "elm.json", + main = "src/Main.elm", + out = "elm.js", + source_directories = { + "../src": "//:src", + "src": "src", + } +) diff --git a/prelude-nri/elm.bzl b/prelude-nri/elm.bzl new file mode 100644 index 00000000..dde549c4 --- /dev/null +++ b/prelude-nri/elm.bzl @@ -0,0 +1,106 @@ +load("//elm:toolchain.bzl", "ElmToolchainInfo") +load("@prelude//python:toolchain.bzl", "PythonToolchainInfo") + +def _elm_docs_impl(ctx: "context") -> [DefaultInfo.type]: + build = ctx.actions.declare_output("build", dir = True) + docs = ctx.actions.declare_output(ctx.attrs.out) + + elm_toolchain = ctx.attrs._elm_toolchain[ElmToolchainInfo] + + cmd = cmd_args( + ctx.attrs._python_toolchain[PythonToolchainInfo].interpreter, + elm_toolchain.isolated_compile[DefaultInfo].default_outputs, + ctx.attrs.elm_json, + "--build-dir", build.as_output(), + "--elm-compiler", elm_toolchain.elm, + "--verbose", + "docs", + "--out", docs.as_output(), + "--src", ctx.attrs.src, + ) + + ctx.actions.run( + cmd, + category = "elm", + no_outputs_cleanup = True, + ) + + return [DefaultInfo(default_output = docs)] + +elm_docs = rule( + impl = _elm_docs_impl, + attrs = { + "out": attrs.string(default="docs.json"), + "elm_json": attrs.source(), + "src": attrs.source(allow_directory = True), + "_elm_toolchain": attrs.toolchain_dep( + default="toolchains//:elm", + providers=[ElmToolchainInfo] + ), + "_python_toolchain": attrs.toolchain_dep( + default="toolchains//:python", + providers=[PythonToolchainInfo] + ), + } +) + +def _elm_app_impl(ctx: "context") -> [DefaultInfo.type]: + build = ctx.actions.declare_output("build", dir = True) + out = ctx.actions.declare_output(ctx.attrs.out) + + elm_toolchain = ctx.attrs._elm_toolchain[ElmToolchainInfo] + + cmd = cmd_args( + ctx.attrs._python_toolchain[PythonToolchainInfo].interpreter, + elm_toolchain.isolated_compile[DefaultInfo].default_outputs, + ctx.attrs.elm_json, + "--build-dir", build.as_output(), + "--elm-compiler", elm_toolchain.elm, + "--verbose", + "make", + ctx.attrs.main, + "--output", out.as_output(), + ) + + for (name, value) in ctx.attrs.source_directories.items(): + cmd.add(cmd_args(value, format="--source-directory=" + name + "={}")) + + if ctx.attrs.debug and ctx.attrs.optimize: + fail("Only one of `optimize` and `debug` may be true!") + + if ctx.attrs.debug: + cmd.add("--debug") + + if ctx.attrs.optimize: + cmd.add("--optimize") + + ctx.actions.run( + cmd, + category = "elm", + no_outputs_cleanup = True, + ) + + return [DefaultInfo(default_output = out)] + +elm_app = rule( + impl = _elm_app_impl, + attrs = { + "out": attrs.string(default = "app.js"), + "elm_json": attrs.source(), + "main": attrs.source(), + "source_directories": attrs.dict( + attrs.string(), + attrs.source(allow_directory=True), + ), + "debug": attrs.bool(default = False), + "optimize": attrs.bool(default = False), + "_elm_toolchain": attrs.toolchain_dep( + default="toolchains//:elm", + providers=[ElmToolchainInfo] + ), + "_python_toolchain": attrs.toolchain_dep( + default="toolchains//:python", + providers=[PythonToolchainInfo] + ), + } +) diff --git a/prelude-nri/elm/BUCK b/prelude-nri/elm/BUCK new file mode 100644 index 00000000..44364d40 --- /dev/null +++ b/prelude-nri/elm/BUCK @@ -0,0 +1,4 @@ +export_file( + name = "isolated_compile.py", + visibility = ["PUBLIC"], +) diff --git a/prelude-nri/elm/isolated_compile.py b/prelude-nri/elm/isolated_compile.py new file mode 100644 index 00000000..bf8a9fed --- /dev/null +++ b/prelude-nri/elm/isolated_compile.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +""" +Build an Elm app, but isolate the working directory so that multiple Elm +compiler invocations can run in parallel without corrupting `.elmi` files. +""" +from argparse import ArgumentParser, ArgumentTypeError +import json +import logging +import os +import os.path +import shutil +from subprocess import Popen, PIPE +import sys +import tempfile + + +def symlink_if_necessary(source, target): + """ + Do the same thing as `os.symlink`, but with these edge cases checked: + + - If the target already exists and points to the source, do nothing. + - If the target already exists and points somewhere else, remove it + and relink. + """ + if os.path.exists(target): + if os.path.isfile(target): + logging.debug(f"`{target}` is a regular file") + + elif os.readlink(target) == source: + logging.debug(f"`{target}` already points to `{source}`") + return + + os.unlink(target) + + logging.info(f"linking `{target}` to `{source}`") + os.symlink(source, target) + + +def write_if_necessary(target, content): + """ + Read the target file and check the content. If it's the same, don't write. + Otherwise, replace it. + """ + if os.path.exists(target): + with open(target, "r") as fh: + if fh.read() == content: + logging.debug(f"`{target}` already had the requested content.") + return + + logging.info(f"writing `{target}`") + with open(target, "w") as fh: + fh.write(content) + + +def run_docs(args): + """ + Compile JSON docs for an Elm library. + """ + logging.info(f"copying {args.elm_json} to {args.build_dir}") + symlink_if_necessary(args.elm_json, os.path.join(args.build_dir, "elm.json")) + + # for libraries, the Elm compiler always assumes the source lives in a + # directory named "src". We have one of those from the arguments, so let's + # set it up under the build root. For the sake of being able to reuse this + # build directory's `elm-stuff`, we take care to repoint the symlink if + # necessary. + symlink_if_necessary(os.path.abspath(args.src), os.path.join(args.build_dir, "src")) + + command = [ + args.elm_compiler, + "make", + "--docs", + # since we're changing cwd, we need to output the absolute path + # instead of a (potentially) relative one. + os.path.abspath(args.output), + ] + logging.debug(f"running {command} in `{args.build_dir}`") + process = Popen(command, cwd=args.build_dir) + process.communicate() + + return process.returncode + + +def run_make(args): + """ + Compile an Elm app to HTML or JavaScript. + + Our basic approach here is to symlink all the source directories and the + Main.elm file into place and modify elm.json to look in those places. This + means that we can use the build directory as a working directory so we get + an isolated elm-stuff directory. + """ + logging.debug(f"reading `{args.elm_json}`") + with open(args.elm_json, "r") as fh: + elm_json = json.load(fh) + + ########################################################################## + # STEP 1: symlink entries in `source-directories` to the right locations # + ########################################################################## + + try: + original_source_directories = elm_json["source-directories"] + except KeyError: + logging.error( + f'`{args.elm_json}` did not have a "source-directories" entry. Is it a package?' + ) + return 1 + + source_directory_replacements = dict( + (sd.src, sd.target) for sd in args.source_directory or [] + ) + + # we're creating a mapping here so we can change the Main file to be relative + # to the symlinks later. + new_source_directories = {} + for i, directory in enumerate(original_source_directories): + try: + replacement = source_directory_replacements[directory] + except KeyError: + logging.error( + f"I don't have a replacement path for `{directory}`. Please specify one with --source-directory {directory}=real-path-to-directory" + ) + return 1 + + dest = f"src-{i}" + + # special case: if the replacement is inside `buck-out` and contains + # only a single directory, we're being passed a Buck2 artifact directory + # and should take the single directory inside. + if replacement.startswith("buck-out"): + contents = os.listdir(replacement) + if len(contents) == 1 and os.path.isdir(contents[0]): + logging.info( + f"I think `{replacement}` is a Buck2 artifact, so I'm symlinking `{contents[0]}` inside it instead." + ) + replacement = os.path.join(replacement, contents[0]) + + symlink_if_necessary( + os.path.abspath(replacement), os.path.join(args.build_dir, dest) + ) + + new_source_directories[replacement] = dest + + logging.debug(f"new source directories: {new_source_directories}") + + ############################################################# + # STEP 2: Modify and write `elm.json` to the right location # + ############################################################# + + elm_json["source-directories"] = list(new_source_directories.values()) + new_elm_json_path = os.path.join(args.build_dir, "elm.json") + logging.debug(f"writing `{new_elm_json_path}`") + + # the compiler will do a full rebuild anytime the `elm.json` file changes. + # Writing only if necessary about halves the runtime for this script for + # noredink-ui's component catalog, with more expected savings the bigger the + # app gets. + write_if_necessary( + new_elm_json_path, + json.dumps(elm_json, indent=4), + ) + + ########################################################## + # STEP 3: Make sure we're poining at the right main file # + ########################################################## + + main = args.main + replaced = False + logging.debug(f"original main: {main}") + for original, replacement in new_source_directories.items(): + if main.startswith(original): + main = os.path.join(replacement, main[len(original) + 1 :]) + logging.debug(f"using `{main}` instead of `{args.main}`") + replaced = True + break + + if not replaced: + # it's fine to build a main file outside a source directory, but let's + # take the absolute path so we can make sure to get it. + main = os.path.abspath(main) + + ##################################################### + # STEP 4: Prepare and run the `elm make` invocation # + ##################################################### + + command = [ + args.elm_compiler, + "make", + main, + "--output", + os.path.abspath(args.output), + ] + + if args.debug and args.optimize: + print("Only one of --debug or --optimize can be set.") + return 1 + + if args.debug: + command.append("--debug") + + if args.optimize: + command.append("--optimize") + + logging.debug(f"running {command} in `{args.build_dir}`") + process = Popen(command, cwd=args.build_dir) + process.communicate() + + return process.returncode + + +class SourceDirectory: + def __init__(self, s): + try: + src, target = s.split("=") + self.src = src + self.target = target + except: + raise ArgumentTypeError("A pair must be `a=b`") + + +if __name__ == "__main__": + parser = ArgumentParser(description=__doc__) + parser.add_argument("elm_json", help="Location of the elm.json") + parser.add_argument( + "--build-dir", + help="Where to perform the build. Should be an empty directory, or one you've already run a build in.", + ) + parser.add_argument( + "--elm-compiler", help="path to the Elm compiler", default="elm" + ) + parser.add_argument( + "--verbose", help="Turn on verbose logging", action="store_true" + ) + + subparsers = parser.add_subparsers(required=True) + + docs = subparsers.add_parser("docs", help=run_docs.__doc__) + docs.set_defaults(func=run_docs) + docs.add_argument( + "--output", help="Path for the resulting docs JSON file", default="docs.json" + ) + docs.add_argument("--src", help="Path to library source", default="src") + + make = subparsers.add_parser("make", help=run_make.__doc__) + make.set_defaults(func=run_make) + make.add_argument("main", help="Main.elm file to build") + make.add_argument( + "--output", + help="Path for the resulting JavaScript or HTML file", + default="app.js", + ) + make.add_argument( + "--source-directory", + type=SourceDirectory, + metavar="SRC=TARGET", + action="append", + ) + make.add_argument("--debug", action="store_true", help="Build in debug mode") + make.add_argument("--optimize", action="store_true", help="Build in optimize mode") + + args = parser.parse_args() + + # logging + logger = logging.getLogger() + handler = logging.StreamHandler(sys.stderr) + formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s") + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(logging.DEBUG if args.verbose else logging.WARNING) + + # prep + if os.path.exists(args.elm_compiler): + args.elm_compiler = os.path.abspath(args.elm_compiler) + + # run! + if args.build_dir is None: + with tempfile.TemporaryDirectory() as temp: + logging.warning( + f"building in temporary directory `{temp}`, which will be cleaned up after the build. If you want a persistent place to build, pass it in with `--build-dir`!" + ) + args.build_dir = temp + sys.exit(args.func(args)) + else: + if not os.path.exists(args.build_dir): + logging.info(f"creating build dir at `{args.build_dir}`") + os.mkdir(args.build_dir) + sys.exit(args.func(args)) diff --git a/prelude-nri/elm/toolchain.bzl b/prelude-nri/elm/toolchain.bzl new file mode 100644 index 00000000..44887309 --- /dev/null +++ b/prelude-nri/elm/toolchain.bzl @@ -0,0 +1,48 @@ +ElmToolchainInfo = provider(fields=[ + "elm", + "isolated_compile", +]) + +def _system_elm_toolchain_impl(ctx) -> [[DefaultInfo.type, ElmToolchainInfo.type]]: + """ + An Elm toolchain that assumes the current environment has all the binaries + it needs in PATH. + """ + return [ + DefaultInfo(), + ElmToolchainInfo( + elm = RunInfo(args = ["elm"]), + isolated_compile = ctx.attrs._isolated_compile, + ), + ] + + +system_elm_toolchain = rule( + impl = _system_elm_toolchain_impl, + attrs = { + "_isolated_compile": attrs.dep(default="prelude-nri//elm:isolated_compile.py"), + }, + is_toolchain_rule = True, +) + +def _elm_toolchain_impl(ctx: "context") -> [[DefaultInfo.type, ElmToolchainInfo.type]]: + """ + An Elm toolchain which you can source binaries from wherever it makes sense + to you. + """ + return [ + DefaultInfo(), + ElmToolchainInfo( + elm = ctx.attrs.elm[RunInfo], + isolated_compile = ctx.attrs._isolated_compile, + ), + ] + +elm_toolchain = rule( + impl = _elm_toolchain_impl, + attrs = { + "elm": attrs.dep(providers = [RunInfo]), + "_isolated_compile": attrs.dep(default="prelude-nri//elm:isolated_compile.py"), + }, + is_toolchain_rule = True, +) diff --git a/script/buck2 b/script/buck2 index 35f9b51a..99017a00 100755 --- a/script/buck2 +++ b/script/buck2 @@ -9,8 +9,20 @@ fi if ! test -f buck-out/buck2; then case "$(uname -s)" in Darwin) - # TODO: adjust for processor type - curl -L https://github.com/facebook/buck2/releases/download/latest/buck2-aarch64-apple-darwin.zst | zstd -d > buck-out/buck2 + case "$(arch)" in + arm64) + curl -L https://github.com/facebook/buck2/releases/download/latest/buck2-aarch64-apple-darwin.zst | zstd -d > buck-out/buck2 + ;; + + i386) + curl -L https://github.com/facebook/buck2/releases/download/latest/buck2-x86_64-apple-darwin.zst | zstd -d > buck-out/buck2 + ;; + + *) + echo "I don't know how to get a download for $(arch) on macOS" + exit 1 + ;; + esac ;; Linux) diff --git a/toolchains/BUCK b/toolchains/BUCK index 0cac578c..937ece96 100644 --- a/toolchains/BUCK +++ b/toolchains/BUCK @@ -1,6 +1,47 @@ load("@prelude//toolchains:genrule.bzl", "system_genrule_toolchain") +load("@prelude//toolchains:python.bzl", "system_python_bootstrap_toolchain", "system_python_toolchain") +load("@prelude-nri//elm:toolchain.bzl", "elm_toolchain") system_genrule_toolchain( name = "genrule", visibility = ["PUBLIC"], ) + +system_python_toolchain( + name = "python", + visibility = ["PUBLIC"], +) + +system_python_bootstrap_toolchain( + name = "python_bootstrap", + visibility = ["PUBLIC"], +) + +elm_toolchain( + name = "elm", + visibility = ["PUBLIC"], + elm = ":elm_compiler_binary", +) + +ELM_COMPILER_URL = select({ + "config//os:linux": "https://github.com/elm/compiler/releases/download/0.19.1/binary-for-linux-64-bit.gz", + "config//os:macos": "https://github.com/elm/compiler/releases/download/0.19.1/binary-for-mac-64-bit.gz", +}) + +ELM_COMPILER_SHA256 = select({ + "config//os:linux": "e44af52bb27f725a973478e589d990a6428e115fe1bb14f03833134d6c0f155c", + "config//os:macos": "05289f0e3d4f30033487c05e689964c3bb17c0c48012510dbef1df43868545d1", +}) + +http_file( + name = "elm_compiler_archive", + urls = [ELM_COMPILER_URL], + sha256 = ELM_COMPILER_SHA256, +) + +genrule( + name = "elm_compiler_binary", + cmd = "gzip --decompress --stdout --keep $(location :elm_compiler_archive) > $OUT && chmod +x $OUT", + out = "elm", + executable = True, +)