build-tar: script to build ISL to a tarball

Summary:
This script takes the ISL source, uses `yarn` to build, then write the built
result to a tarball. It maintains a "source_hash" and skips building if the
destination "source_hash" does not change.

The yarn build commands and directories to include are from `isl/release.js`.

Reviewed By: evangrayk

Differential Revision: D49558562

fbshipit-source-id: 060fec91a1f65cda51efa525ec291cd8483cf42b

addons/.gitignore

@@ -2,3 +2,4 @@ node_modules
 isl-server/coverage
 shared/coverage/
 vscode-build/
+/*.tar.xz

addons/build-tar.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+import atexit
+import functools
+import glob
+import hashlib
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+from typing import List
+
+rm_rf = functools.partial(shutil.rmtree, ignore_errors=True)
+print_err = functools.partial(print, file=sys.stderr)
+glob_r = functools.partial(glob.glob, recursive=True)
+
+# used to detect if files are changed.
+def hash_path_contents(paths: List[str]):
+    h = hashlib.sha1()
+    sorted_paths = sorted(paths)
+    for path in sorted_paths:
+        try:
+            with open(path, "rb") as f:
+                h.update(f.read())
+                h.update(b"\0")
+        except IsADirectoryError:
+            pass
+    return h.hexdigest()
+
+
+WALK_EXCLUDE_DIRS = ["node_modules", "build", "dist", "vscode-build", "coverage"]
+WALK_EXCLUDE_EXTS = ".xz"
+
+# find source code files (to hash_path_contents), excluding build results and node_modules
+def walk_src_files(top: str):
+    for root, dirs, files in os.walk(top):
+        for exclude in WALK_EXCLUDE_DIRS:
+            if exclude in dirs:
+                dirs.remove(exclude)
+        for name in files:
+            if any(name.endswith(ext) for ext in WALK_EXCLUDE_EXTS):
+                continue
+            yield os.path.join(root, name)
+
+
+def run(command: List[str], cwd=None, env=None):
+    print_err(f"{cwd if cwd else ' '} $ {shlex.join(command)}")
+
+    if env is not None:
+        env = {**os.environ, **env}
+
+    # shell=True with a List `command` seems buggy on *nix.
+    # It might run ['sh', '-c', 'a', 'b'] instead of ['sh', '-c', 'a b'].
+    subprocess.run(command, shell=(os.name == "nt"), check=True, cwd=cwd, env=env)
+
+
+def realpath_args(args: List[str]) -> List[str]:
+    return [os.path.realpath(arg) if os.path.exists(arg) else arg for arg in args]
+
+
+def copy_writable(src, dst, *, follow_symlinks=True):
+    """shutil.copy, but ensure that yarn.lock is writable
+    - RE might make src/ read-only with its "restrictive mode".
+    - When copying the RE "restrictive" src/, yarn.lock is read-only.
+    - yarn wants yarn.lock to be writable, even with --frozen-lockfile.
+    """
+    shutil.copy(src, dst, follow_symlinks=follow_symlinks)
+    if dst.endswith("yarn.lock") and os.name != "nt":
+        os.chmod(dst, 0o666)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Creates a tarball of built ISL source."
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        nargs="?",
+        default="isl-dist.tar.xz",
+        help="Path to the output '.tar.xz' file.",
+    )
+    parser.add_argument(
+        "--yarn",
+        default="",
+        help="Path to yarn executable.",
+    )
+    parser.add_argument(
+        "--yarn-offline-mirror",
+        default=None,
+        help="Path to the yarn offline mirror.",
+    )
+    parser.add_argument(
+        "--src",
+        default=None,
+        help="Directory that contains the source code.",
+    )
+    parser.add_argument(
+        "--tmp",
+        default=None,
+        help="Temporary directory to run build. Do not modify src in-place.",
+    )
+
+    args = parser.parse_args()
+
+    # posix=False prevents shlex.split from treating \\ as escape character, breaking Windows.
+    yarn = realpath_args(
+        shlex.split(args.yarn or os.getenv("YARN") or "yarn", posix=False)
+    )
+
+    src = args.src or "."
+    out = args.output
+
+    if args.tmp:
+        # copy source to a temporary directory
+        # used by buck genrule, which does not guarantee src is writable
+        tmp_src_path = tempfile.mkdtemp(prefix="isl-src", dir=args.tmp)
+        atexit.register(lambda: rm_rf(tmp_src_path))
+        print_err(f"copying source {src} to {tmp_src_path}")
+        shutil.copytree(
+            src, tmp_src_path, dirs_exist_ok=True, copy_function=copy_writable
+        )
+        src = tmp_src_path
+
+    src_join = functools.partial(os.path.join, src)
+
+    source_hash = hash_path_contents(walk_src_files(src))
+    try:
+        with tarfile.open(out, "r") as tar:
+            old_source_hash = tar.pax_headers.get("source_hash")
+        if old_source_hash == source_hash:
+            print_err(f"source not changed, skip rebuilding {out}")
+            return
+        else:
+            print_err(
+                f"source changed {old_source_hash[:8]} -> {source_hash[:8]}, rebuilding {out}"
+            )
+            os.unlink(out)
+    except FileNotFoundError:
+        pass
+    except tarfile.ReadError:
+        os.unlink(out)
+
+    if args.yarn_offline_mirror:
+        env = {"YARN_YARN_OFFLINE_MIRROR": os.path.realpath(args.yarn_offline_mirror)}
+        run(
+            yarn
+            + [
+                "--cwd",
+                src_join(),
+                "install",
+                "--offline",
+                "--frozen-lockfile",
+                "--ignore-scripts",
+                "--check-files",
+            ],
+            env=env,
+        )
+    else:
+        run(yarn + ["--cwd", src_join(), "install", "--prefer-offline"])
+
+    rm_rf(src_join("server/dist"))
+    run(yarn + ["--cwd", src_join("isl-server"), "run", "build"])
+
+    rm_rf(src_join("isl/build"))
+    run(yarn + ["--cwd", src_join("isl"), "run", "build"])
+
+    print_err(f"writing {out}")
+
+    headers = {
+        "source_hash": source_hash,
+        "entry_point": "isl-server/dist/run-proxy.js",
+    }
+
+    with tarfile.open(
+        out, "w:xz", pax_headers=headers, format=tarfile.PAX_FORMAT
+    ) as tar:
+
+        def add(path):
+            tar.add(src_join(path), path)
+
+        add("isl-server/dist")
+        add("isl-server/node_modules/ws")
+        add("isl/build")
+
+
+if __name__ == "__main__":
+    main()