mirror of
https://github.com/facebook/sapling.git
synced 2024-09-11 18:47:10 +03:00
build-tar: script to build ISL to a tarball
Summary: This script takes the ISL source, uses `yarn` to build, then write the built result to a tarball. It maintains a "source_hash" and skips building if the destination "source_hash" does not change. The yarn build commands and directories to include are from `isl/release.js`. Reviewed By: evangrayk Differential Revision: D49558562 fbshipit-source-id: 060fec91a1f65cda51efa525ec291cd8483cf42b
This commit is contained in:
parent
b6d53bc1f0
commit
85cc5614ee
1
addons/.gitignore
vendored
1
addons/.gitignore
vendored
@ -2,3 +2,4 @@ node_modules
|
||||
isl-server/coverage
|
||||
shared/coverage/
|
||||
vscode-build/
|
||||
/*.tar.xz
|
||||
|
196
addons/build-tar.py
Executable file
196
addons/build-tar.py
Executable file
@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import argparse
|
||||
import atexit
|
||||
import functools
|
||||
import glob
|
||||
import hashlib
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
from typing import List
|
||||
|
||||
rm_rf = functools.partial(shutil.rmtree, ignore_errors=True)
|
||||
print_err = functools.partial(print, file=sys.stderr)
|
||||
glob_r = functools.partial(glob.glob, recursive=True)
|
||||
|
||||
# used to detect if files are changed.
|
||||
def hash_path_contents(paths: List[str]):
|
||||
h = hashlib.sha1()
|
||||
sorted_paths = sorted(paths)
|
||||
for path in sorted_paths:
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
h.update(f.read())
|
||||
h.update(b"\0")
|
||||
except IsADirectoryError:
|
||||
pass
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
WALK_EXCLUDE_DIRS = ["node_modules", "build", "dist", "vscode-build", "coverage"]
|
||||
WALK_EXCLUDE_EXTS = ".xz"
|
||||
|
||||
# find source code files (to hash_path_contents), excluding build results and node_modules
|
||||
def walk_src_files(top: str):
|
||||
for root, dirs, files in os.walk(top):
|
||||
for exclude in WALK_EXCLUDE_DIRS:
|
||||
if exclude in dirs:
|
||||
dirs.remove(exclude)
|
||||
for name in files:
|
||||
if any(name.endswith(ext) for ext in WALK_EXCLUDE_EXTS):
|
||||
continue
|
||||
yield os.path.join(root, name)
|
||||
|
||||
|
||||
def run(command: List[str], cwd=None, env=None):
|
||||
print_err(f"{cwd if cwd else ' '} $ {shlex.join(command)}")
|
||||
|
||||
if env is not None:
|
||||
env = {**os.environ, **env}
|
||||
|
||||
# shell=True with a List `command` seems buggy on *nix.
|
||||
# It might run ['sh', '-c', 'a', 'b'] instead of ['sh', '-c', 'a b'].
|
||||
subprocess.run(command, shell=(os.name == "nt"), check=True, cwd=cwd, env=env)
|
||||
|
||||
|
||||
def realpath_args(args: List[str]) -> List[str]:
|
||||
return [os.path.realpath(arg) if os.path.exists(arg) else arg for arg in args]
|
||||
|
||||
|
||||
def copy_writable(src, dst, *, follow_symlinks=True):
|
||||
"""shutil.copy, but ensure that yarn.lock is writable
|
||||
- RE might make src/ read-only with its "restrictive mode".
|
||||
- When copying the RE "restrictive" src/, yarn.lock is read-only.
|
||||
- yarn wants yarn.lock to be writable, even with --frozen-lockfile.
|
||||
"""
|
||||
shutil.copy(src, dst, follow_symlinks=follow_symlinks)
|
||||
if dst.endswith("yarn.lock") and os.name != "nt":
|
||||
os.chmod(dst, 0o666)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Creates a tarball of built ISL source."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
nargs="?",
|
||||
default="isl-dist.tar.xz",
|
||||
help="Path to the output '.tar.xz' file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--yarn",
|
||||
default="",
|
||||
help="Path to yarn executable.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--yarn-offline-mirror",
|
||||
default=None,
|
||||
help="Path to the yarn offline mirror.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--src",
|
||||
default=None,
|
||||
help="Directory that contains the source code.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tmp",
|
||||
default=None,
|
||||
help="Temporary directory to run build. Do not modify src in-place.",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# posix=False prevents shlex.split from treating \\ as escape character, breaking Windows.
|
||||
yarn = realpath_args(
|
||||
shlex.split(args.yarn or os.getenv("YARN") or "yarn", posix=False)
|
||||
)
|
||||
|
||||
src = args.src or "."
|
||||
out = args.output
|
||||
|
||||
if args.tmp:
|
||||
# copy source to a temporary directory
|
||||
# used by buck genrule, which does not guarantee src is writable
|
||||
tmp_src_path = tempfile.mkdtemp(prefix="isl-src", dir=args.tmp)
|
||||
atexit.register(lambda: rm_rf(tmp_src_path))
|
||||
print_err(f"copying source {src} to {tmp_src_path}")
|
||||
shutil.copytree(
|
||||
src, tmp_src_path, dirs_exist_ok=True, copy_function=copy_writable
|
||||
)
|
||||
src = tmp_src_path
|
||||
|
||||
src_join = functools.partial(os.path.join, src)
|
||||
|
||||
source_hash = hash_path_contents(walk_src_files(src))
|
||||
try:
|
||||
with tarfile.open(out, "r") as tar:
|
||||
old_source_hash = tar.pax_headers.get("source_hash")
|
||||
if old_source_hash == source_hash:
|
||||
print_err(f"source not changed, skip rebuilding {out}")
|
||||
return
|
||||
else:
|
||||
print_err(
|
||||
f"source changed {old_source_hash[:8]} -> {source_hash[:8]}, rebuilding {out}"
|
||||
)
|
||||
os.unlink(out)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except tarfile.ReadError:
|
||||
os.unlink(out)
|
||||
|
||||
if args.yarn_offline_mirror:
|
||||
env = {"YARN_YARN_OFFLINE_MIRROR": os.path.realpath(args.yarn_offline_mirror)}
|
||||
run(
|
||||
yarn
|
||||
+ [
|
||||
"--cwd",
|
||||
src_join(),
|
||||
"install",
|
||||
"--offline",
|
||||
"--frozen-lockfile",
|
||||
"--ignore-scripts",
|
||||
"--check-files",
|
||||
],
|
||||
env=env,
|
||||
)
|
||||
else:
|
||||
run(yarn + ["--cwd", src_join(), "install", "--prefer-offline"])
|
||||
|
||||
rm_rf(src_join("server/dist"))
|
||||
run(yarn + ["--cwd", src_join("isl-server"), "run", "build"])
|
||||
|
||||
rm_rf(src_join("isl/build"))
|
||||
run(yarn + ["--cwd", src_join("isl"), "run", "build"])
|
||||
|
||||
print_err(f"writing {out}")
|
||||
|
||||
headers = {
|
||||
"source_hash": source_hash,
|
||||
"entry_point": "isl-server/dist/run-proxy.js",
|
||||
}
|
||||
|
||||
with tarfile.open(
|
||||
out, "w:xz", pax_headers=headers, format=tarfile.PAX_FORMAT
|
||||
) as tar:
|
||||
|
||||
def add(path):
|
||||
tar.add(src_join(path), path)
|
||||
|
||||
add("isl-server/dist")
|
||||
add("isl-server/node_modules/ws")
|
||||
add("isl/build")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
Loading…
Reference in New Issue
Block a user