mirror of
https://github.com/facebook/sapling.git
synced 2024-10-11 01:07:15 +03:00
2eb88ab683
Summary: Automatically detect the `--facebook-internal` flag based on the current repository project name. Reviewed By: wez Differential Revision: D18621358 fbshipit-source-id: f2b3018169b151811eec455863a8bfc17667d4d8
787 lines
27 KiB
Python
Executable File
787 lines
27 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright (c) Facebook, Inc. and its affiliates.
|
|
#
|
|
# This source code is licensed under the MIT license found in the
|
|
# LICENSE file in the root directory of this source tree.
|
|
|
|
from __future__ import absolute_import, division, print_function, unicode_literals
|
|
|
|
import argparse
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
|
|
# We don't import cache.create_cache directly as the facebook
|
|
# specific import below may monkey patch it, and we want to
|
|
# observe the patched version of this function!
|
|
import getdeps.cache as cache_module
|
|
from getdeps.buildopts import setup_build_options
|
|
from getdeps.dyndeps import create_dyn_dep_munger
|
|
from getdeps.errors import TransientFailure
|
|
from getdeps.load import ManifestLoader
|
|
from getdeps.manifest import ManifestParser
|
|
from getdeps.platform import HostType
|
|
from getdeps.subcmd import SubCmd, add_subcommands, cmd
|
|
|
|
|
|
try:
|
|
import getdeps.facebook # noqa: F401
|
|
except ImportError:
|
|
# we don't ship the facebook specific subdir,
|
|
# so allow that to fail silently
|
|
pass
|
|
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "getdeps"))
|
|
|
|
|
|
class UsageError(Exception):
|
|
pass
|
|
|
|
|
|
@cmd("validate-manifest", "parse a manifest and validate that it is correct")
|
|
class ValidateManifest(SubCmd):
|
|
def run(self, args):
|
|
try:
|
|
ManifestParser(file_name=args.file_name)
|
|
print("OK", file=sys.stderr)
|
|
return 0
|
|
except Exception as exc:
|
|
print("ERROR: %s" % str(exc), file=sys.stderr)
|
|
return 1
|
|
|
|
def setup_parser(self, parser):
|
|
parser.add_argument("file_name", help="path to the manifest file")
|
|
|
|
|
|
@cmd("show-host-type", "outputs the host type tuple for the host machine")
|
|
class ShowHostType(SubCmd):
|
|
def run(self, args):
|
|
host = HostType()
|
|
print("%s" % host.as_tuple_string())
|
|
return 0
|
|
|
|
|
|
class ProjectCmdBase(SubCmd):
|
|
def run(self, args):
|
|
opts = setup_build_options(args)
|
|
|
|
if args.current_project is not None:
|
|
opts.repo_project = args.current_project
|
|
if args.project is None:
|
|
if opts.repo_project is None:
|
|
raise UsageError(
|
|
"no project name specified, and no .projectid file found"
|
|
)
|
|
if opts.repo_project == "fbsource":
|
|
# The fbsource repository is a little special. There is no project
|
|
# manifest file for it. A specific project must always be explicitly
|
|
# specified when building from fbsource.
|
|
raise UsageError(
|
|
"no project name specified (required when building in fbsource)"
|
|
)
|
|
args.project = opts.repo_project
|
|
|
|
ctx_gen = opts.get_context_generator(facebook_internal=args.facebook_internal)
|
|
if args.test_dependencies:
|
|
ctx_gen.set_value_for_all_projects("test", "on")
|
|
if args.enable_tests:
|
|
ctx_gen.set_value_for_project(args.project, "test", "on")
|
|
else:
|
|
ctx_gen.set_value_for_project(args.project, "test", "off")
|
|
|
|
loader = ManifestLoader(opts, ctx_gen)
|
|
self.process_project_dir_arguments(args, loader)
|
|
|
|
manifest = loader.load_manifest(args.project)
|
|
|
|
self.run_project_cmd(args, loader, manifest)
|
|
|
|
def process_project_dir_arguments(self, args, loader):
|
|
def parse_project_arg(arg, arg_type):
|
|
parts = arg.split(":")
|
|
if len(parts) == 2:
|
|
project, path = parts
|
|
elif len(parts) == 1:
|
|
project = args.project
|
|
path = parts[0]
|
|
else:
|
|
raise UsageError(
|
|
"invalid %s argument; too many ':' characters: %s" % (arg_type, arg)
|
|
)
|
|
|
|
return project, os.path.abspath(path)
|
|
|
|
# If we are currently running from a project repository,
|
|
# use the current repository for the project sources.
|
|
build_opts = loader.build_opts
|
|
if build_opts.repo_project is not None and build_opts.repo_root is not None:
|
|
loader.set_project_src_dir(build_opts.repo_project, build_opts.repo_root)
|
|
|
|
for arg in args.src_dir:
|
|
project, path = parse_project_arg(arg, "--src-dir")
|
|
loader.set_project_src_dir(project, path)
|
|
|
|
for arg in args.build_dir:
|
|
project, path = parse_project_arg(arg, "--build-dir")
|
|
loader.set_project_build_dir(project, path)
|
|
|
|
for arg in args.install_dir:
|
|
project, path = parse_project_arg(arg, "--install-dir")
|
|
loader.set_project_install_dir(project, path)
|
|
|
|
def setup_parser(self, parser):
|
|
parser.add_argument(
|
|
"project",
|
|
nargs="?",
|
|
help=(
|
|
"name of the project or path to a manifest "
|
|
"file describing the project"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--no-tests",
|
|
action="store_false",
|
|
dest="enable_tests",
|
|
default=True,
|
|
help="Disable building tests for this project.",
|
|
)
|
|
parser.add_argument(
|
|
"--test-dependencies",
|
|
action="store_true",
|
|
help="Enable building tests for dependencies as well.",
|
|
)
|
|
parser.add_argument(
|
|
"--current-project",
|
|
help="Specify the name of the fbcode_builder manifest file for the "
|
|
"current repository. If not specified, the code will attempt to find "
|
|
"this in a .projectid file in the repository root.",
|
|
)
|
|
parser.add_argument(
|
|
"--src-dir",
|
|
default=[],
|
|
action="append",
|
|
help="Specify a local directory to use for the project source, "
|
|
"rather than fetching it.",
|
|
)
|
|
parser.add_argument(
|
|
"--build-dir",
|
|
default=[],
|
|
action="append",
|
|
help="Explicitly specify the build directory to use for the "
|
|
"project, instead of the default location in the scratch path. "
|
|
"This only affects the project specified, and not its dependencies.",
|
|
)
|
|
parser.add_argument(
|
|
"--install-dir",
|
|
default=[],
|
|
action="append",
|
|
help="Explicitly specify the install directory to use for the "
|
|
"project, instead of the default location in the scratch path. "
|
|
"This only affects the project specified, and not its dependencies.",
|
|
)
|
|
|
|
self.setup_project_cmd_parser(parser)
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
pass
|
|
|
|
|
|
class CachedProject(object):
|
|
""" A helper that allows calling the cache logic for a project
|
|
from both the build and the fetch code """
|
|
|
|
def __init__(self, cache, loader, m):
|
|
self.m = m
|
|
self.inst_dir = loader.get_project_install_dir(m)
|
|
self.project_hash = loader.get_project_hash(m)
|
|
self.ctx = loader.ctx_gen.get_context(m.name)
|
|
self.loader = loader
|
|
self.cache = cache
|
|
|
|
self.cache_file_name = "-".join(
|
|
(
|
|
m.name,
|
|
self.ctx.get("os"),
|
|
self.ctx.get("distro") or "none",
|
|
self.ctx.get("distro_vers") or "none",
|
|
self.project_hash,
|
|
"buildcache.tgz",
|
|
)
|
|
)
|
|
|
|
def is_cacheable(self):
|
|
""" We only cache third party projects """
|
|
return self.cache and self.m.shipit_project is None
|
|
|
|
def download(self):
|
|
if self.is_cacheable() and not os.path.exists(self.inst_dir):
|
|
print("check cache for %s" % self.cache_file_name)
|
|
dl_dir = os.path.join(self.loader.build_opts.scratch_dir, "downloads")
|
|
if not os.path.exists(dl_dir):
|
|
os.makedirs(dl_dir)
|
|
try:
|
|
target_file_name = os.path.join(dl_dir, self.cache_file_name)
|
|
if self.cache.download_to_file(self.cache_file_name, target_file_name):
|
|
tf = tarfile.open(target_file_name, "r")
|
|
print(
|
|
"Extracting %s -> %s..." % (self.cache_file_name, self.inst_dir)
|
|
)
|
|
tf.extractall(self.inst_dir)
|
|
return True
|
|
except Exception as exc:
|
|
print("%s" % str(exc))
|
|
|
|
return False
|
|
|
|
def upload(self):
|
|
if self.is_cacheable():
|
|
# We can prepare an archive and stick it in LFS
|
|
tempdir = tempfile.mkdtemp()
|
|
tarfilename = os.path.join(tempdir, self.cache_file_name)
|
|
print("Archiving for cache: %s..." % tarfilename)
|
|
tf = tarfile.open(tarfilename, "w:gz")
|
|
tf.add(self.inst_dir, arcname=".")
|
|
tf.close()
|
|
try:
|
|
self.cache.upload_from_file(self.cache_file_name, tarfilename)
|
|
except Exception as exc:
|
|
print(
|
|
"Failed to upload to cache (%s), continue anyway" % str(exc),
|
|
file=sys.stderr,
|
|
)
|
|
shutil.rmtree(tempdir)
|
|
|
|
|
|
@cmd("fetch", "fetch the code for a given project")
|
|
class FetchCmd(ProjectCmdBase):
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--recursive",
|
|
help="fetch the transitive deps also",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
parser.add_argument(
|
|
"--host-type",
|
|
help=(
|
|
"When recursively fetching, fetch deps for "
|
|
"this host type rather than the current system"
|
|
),
|
|
)
|
|
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
if args.recursive:
|
|
projects = loader.manifests_in_dependency_order()
|
|
else:
|
|
projects = [manifest]
|
|
|
|
cache = cache_module.create_cache()
|
|
for m in projects:
|
|
cached_project = CachedProject(cache, loader, m)
|
|
if cached_project.download():
|
|
continue
|
|
|
|
inst_dir = loader.get_project_install_dir(m)
|
|
built_marker = os.path.join(inst_dir, ".built-by-getdeps")
|
|
if os.path.exists(built_marker):
|
|
with open(built_marker, "r") as f:
|
|
built_hash = f.read().strip()
|
|
|
|
project_hash = loader.get_project_hash(m)
|
|
if built_hash == project_hash:
|
|
continue
|
|
|
|
# We need to fetch the sources
|
|
fetcher = loader.create_fetcher(m)
|
|
fetcher.update()
|
|
|
|
|
|
@cmd("list-deps", "lists the transitive deps for a given project")
|
|
class ListDepsCmd(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
for m in loader.manifests_in_dependency_order():
|
|
print(m.name)
|
|
return 0
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--host-type",
|
|
help=(
|
|
"Produce the list for the specified host type, "
|
|
"rather than that of the current system"
|
|
),
|
|
)
|
|
|
|
|
|
def clean_dirs(opts):
|
|
for d in ["build", "installed", "extracted", "shipit"]:
|
|
d = os.path.join(opts.scratch_dir, d)
|
|
print("Cleaning %s..." % d)
|
|
if os.path.exists(d):
|
|
shutil.rmtree(d)
|
|
|
|
|
|
@cmd("clean", "clean up the scratch dir")
|
|
class CleanCmd(SubCmd):
|
|
def run(self, args):
|
|
opts = setup_build_options(args)
|
|
clean_dirs(opts)
|
|
|
|
|
|
@cmd("show-inst-dir", "print the installation dir for a given project")
|
|
class ShowInstDirCmd(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
if args.recursive:
|
|
manifests = loader.manifests_in_dependency_order()
|
|
else:
|
|
manifests = [manifest]
|
|
|
|
for m in manifests:
|
|
inst_dir = loader.get_project_install_dir(m)
|
|
print(inst_dir)
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--recursive",
|
|
help="print the transitive deps also",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
|
|
|
|
@cmd("show-source-dir", "print the source dir for a given project")
|
|
class ShowSourceDirCmd(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
if args.recursive:
|
|
manifests = loader.manifests_in_dependency_order()
|
|
else:
|
|
manifests = [manifest]
|
|
|
|
for m in manifests:
|
|
fetcher = loader.create_fetcher(m)
|
|
print(fetcher.get_src_dir())
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--recursive",
|
|
help="print the transitive deps also",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
|
|
|
|
@cmd("build", "build a given project")
|
|
class BuildCmd(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
if args.clean:
|
|
clean_dirs(loader.build_opts)
|
|
|
|
print("Building on %s" % loader.ctx_gen.get_context(args.project))
|
|
projects = loader.manifests_in_dependency_order()
|
|
|
|
cache = cache_module.create_cache() if args.use_build_cache else None
|
|
|
|
# Accumulate the install directories so that the build steps
|
|
# can find their dep installation
|
|
install_dirs = []
|
|
|
|
for m in projects:
|
|
fetcher = loader.create_fetcher(m)
|
|
|
|
if args.clean:
|
|
fetcher.clean()
|
|
|
|
build_dir = loader.get_project_build_dir(m)
|
|
inst_dir = loader.get_project_install_dir(m)
|
|
|
|
if m == manifest or not args.no_deps:
|
|
print("Assessing %s..." % m.name)
|
|
project_hash = loader.get_project_hash(m)
|
|
ctx = loader.ctx_gen.get_context(m.name)
|
|
built_marker = os.path.join(inst_dir, ".built-by-getdeps")
|
|
|
|
cached_project = CachedProject(cache, loader, m)
|
|
|
|
reconfigure, sources_changed = self.compute_source_change_status(
|
|
cached_project, fetcher, m, built_marker, project_hash
|
|
)
|
|
|
|
if sources_changed or reconfigure or not os.path.exists(built_marker):
|
|
if os.path.exists(built_marker):
|
|
os.unlink(built_marker)
|
|
src_dir = fetcher.get_src_dir()
|
|
builder = m.create_builder(
|
|
loader.build_opts, src_dir, build_dir, inst_dir, ctx
|
|
)
|
|
builder.build(install_dirs, reconfigure=reconfigure)
|
|
|
|
with open(built_marker, "w") as f:
|
|
f.write(project_hash)
|
|
|
|
# Only populate the cache from continuous build runs
|
|
if args.schedule_type == "continuous":
|
|
cached_project.upload()
|
|
|
|
install_dirs.append(inst_dir)
|
|
|
|
def compute_source_change_status(
|
|
self, cached_project, fetcher, m, built_marker, project_hash
|
|
):
|
|
reconfigure = False
|
|
sources_changed = False
|
|
if not cached_project.download():
|
|
check_fetcher = True
|
|
if os.path.exists(built_marker):
|
|
check_fetcher = False
|
|
with open(built_marker, "r") as f:
|
|
built_hash = f.read().strip()
|
|
if built_hash == project_hash:
|
|
if cached_project.is_cacheable():
|
|
# We can blindly trust the build status
|
|
reconfigure = False
|
|
sources_changed = False
|
|
else:
|
|
# Otherwise, we may have changed the source, so let's
|
|
# check in with the fetcher layer
|
|
check_fetcher = True
|
|
else:
|
|
# Some kind of inconsistency with a prior build,
|
|
# let's run it again to be sure
|
|
os.unlink(built_marker)
|
|
reconfigure = True
|
|
sources_changed = True
|
|
|
|
if check_fetcher:
|
|
change_status = fetcher.update()
|
|
reconfigure = change_status.build_changed()
|
|
sources_changed = change_status.sources_changed()
|
|
|
|
return reconfigure, sources_changed
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--clean",
|
|
action="store_true",
|
|
default=False,
|
|
help=(
|
|
"Clean up the build and installation area prior to building, "
|
|
"causing the projects to be built from scratch"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--no-deps",
|
|
action="store_true",
|
|
default=False,
|
|
help=(
|
|
"Only build the named project, not its deps. "
|
|
"This is most useful after you've built all of the deps, "
|
|
"and helps to avoid waiting for relatively "
|
|
"slow up-to-date-ness checks"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--no-build-cache",
|
|
action="store_false",
|
|
default=True,
|
|
dest="use_build_cache",
|
|
help="Do not attempt to use the build cache.",
|
|
)
|
|
parser.add_argument(
|
|
"--schedule-type", help="Indicates how the build was activated"
|
|
)
|
|
|
|
|
|
@cmd("fixup-dyn-deps", "Adjusts dynamic dependencies for packaging purposes")
|
|
class FixupDeps(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
projects = loader.manifests_in_dependency_order()
|
|
|
|
# Accumulate the install directories so that the build steps
|
|
# can find their dep installation
|
|
install_dirs = []
|
|
|
|
for m in projects:
|
|
inst_dir = loader.get_project_install_dir(m)
|
|
install_dirs.append(inst_dir)
|
|
|
|
if m == manifest:
|
|
dep_munger = create_dyn_dep_munger(loader.build_opts, install_dirs)
|
|
dep_munger.process_deps(args.destdir, args.final_install_prefix)
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument("destdir", help=("Where to copy the fixed up executables"))
|
|
parser.add_argument(
|
|
"--final-install-prefix", help=("specify the final installation prefix")
|
|
)
|
|
|
|
|
|
@cmd("test", "test a given project")
|
|
class TestCmd(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
projects = loader.manifests_in_dependency_order()
|
|
|
|
# Accumulate the install directories so that the test steps
|
|
# can find their dep installation
|
|
install_dirs = []
|
|
|
|
for m in projects:
|
|
inst_dir = loader.get_project_install_dir(m)
|
|
|
|
if m == manifest or args.test_dependencies:
|
|
built_marker = os.path.join(inst_dir, ".built-by-getdeps")
|
|
if not os.path.exists(built_marker):
|
|
print("project %s has not been built" % m.name)
|
|
# TODO: we could just go ahead and build it here, but I
|
|
# want to tackle that as part of adding build-for-test
|
|
# support.
|
|
return 1
|
|
fetcher = loader.create_fetcher(m)
|
|
src_dir = fetcher.get_src_dir()
|
|
ctx = loader.ctx_gen.get_context(m.name)
|
|
build_dir = loader.get_project_build_dir(m)
|
|
builder = m.create_builder(
|
|
loader.build_opts, src_dir, build_dir, inst_dir, ctx
|
|
)
|
|
builder.run_tests(
|
|
install_dirs,
|
|
schedule_type=args.schedule_type,
|
|
owner=args.test_owner,
|
|
)
|
|
|
|
install_dirs.append(inst_dir)
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument(
|
|
"--schedule-type", help="Indicates how the build was activated"
|
|
)
|
|
parser.add_argument("--test-owner", help="Owner for testpilot")
|
|
|
|
|
|
@cmd("generate-github-actions", "generate a GitHub actions configuration")
|
|
class GenerateGitHubActionsCmd(ProjectCmdBase):
|
|
def run_project_cmd(self, args, loader, manifest):
|
|
platforms = [
|
|
HostType("linux", "ubuntu", "18"),
|
|
HostType("darwin", None, None),
|
|
HostType("windows", None, None),
|
|
]
|
|
|
|
with open(args.output_file, "w") as out:
|
|
# Deliberate line break here because the @ and the generated
|
|
# symbols are meaningful to our internal tooling when they
|
|
# appear in a single token
|
|
out.write("# This file was @")
|
|
out.write("generated by getdeps.py\n")
|
|
out.write(
|
|
"""
|
|
name: CI
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- master
|
|
pull_request:
|
|
branches:
|
|
- master
|
|
|
|
jobs:
|
|
"""
|
|
)
|
|
for p in platforms:
|
|
build_opts = setup_build_options(args, p)
|
|
self.write_job_for_platform(out, args, build_opts)
|
|
|
|
def write_job_for_platform(self, out, args, build_opts):
|
|
ctx_gen = build_opts.get_context_generator()
|
|
loader = ManifestLoader(build_opts, ctx_gen)
|
|
manifest = loader.load_manifest(args.project)
|
|
manifest_ctx = loader.ctx_gen.get_context(manifest.name)
|
|
|
|
# Some projects don't do anything "useful" as a leaf project, only
|
|
# as a dep for a leaf project. Check for those here; we don't want
|
|
# to waste the effort scheduling them on CI.
|
|
# We do this by looking at the builder type in the manifest file
|
|
# rather than creating a builder and checking its type because we
|
|
# don't know enough to create the full builder instance here.
|
|
if manifest.get("build", "builder", ctx=manifest_ctx) == "nop":
|
|
return None
|
|
|
|
if build_opts.is_linux():
|
|
job_name = "linux"
|
|
runs_on = "ubuntu-18.04"
|
|
elif build_opts.is_windows():
|
|
# We're targeting the windows-2016 image because it has
|
|
# Visual Studio 2017 installed, and at the time of writing,
|
|
# the version of boost in the manifests (1.69) is not
|
|
# buildable with Visual Studio 2019
|
|
job_name = "windows"
|
|
runs_on = "windows-2016"
|
|
else:
|
|
job_name = "mac"
|
|
runs_on = "macOS-latest"
|
|
|
|
out.write(" %s:\n" % job_name)
|
|
out.write(" runs-on: %s\n" % runs_on)
|
|
out.write(" steps:\n")
|
|
out.write(" - uses: actions/checkout@v1\n")
|
|
|
|
projects = loader.manifests_in_dependency_order()
|
|
|
|
for m in projects:
|
|
if m != manifest:
|
|
out.write(" - name: Fetch %s\n" % m.name)
|
|
out.write(
|
|
" run: python build/fbcode_builder/getdeps.py fetch "
|
|
"--no-tests %s\n" % m.name
|
|
)
|
|
|
|
for m in projects:
|
|
if m != manifest:
|
|
out.write(" - name: Build %s\n" % m.name)
|
|
out.write(
|
|
" run: python build/fbcode_builder/getdeps.py build "
|
|
"--no-tests %s\n" % m.name
|
|
)
|
|
|
|
out.write(" - name: Build %s\n" % manifest.name)
|
|
out.write(
|
|
" run: python build/fbcode_builder/getdeps.py build --src-dir=. %s\n"
|
|
% manifest.name
|
|
)
|
|
|
|
out.write(" - name: Copy artifacts\n")
|
|
out.write(
|
|
" run: python build/fbcode_builder/getdeps.py fixup-dyn-deps "
|
|
"--src-dir=. %s _artifacts/%s\n" % (manifest.name, job_name)
|
|
)
|
|
out.write(" - uses: actions/upload-artifact@master\n")
|
|
out.write(" with:\n")
|
|
out.write(" name: %s\n" % manifest.name)
|
|
out.write(" path: _artifacts\n")
|
|
|
|
out.write(" - name: Test %s\n" % manifest.name)
|
|
out.write(
|
|
" run: python build/fbcode_builder/getdeps.py test --src-dir=. %s\n"
|
|
% manifest.name
|
|
)
|
|
|
|
def setup_project_cmd_parser(self, parser):
|
|
parser.add_argument("--output-file", help="The name of the yaml file")
|
|
|
|
|
|
def get_arg_var_name(args):
|
|
for arg in args:
|
|
if arg.startswith("--"):
|
|
return arg[2:].replace("-", "_")
|
|
|
|
raise Exception("unable to determine argument variable name from %r" % (args,))
|
|
|
|
|
|
def parse_args():
|
|
# We want to allow common arguments to be specified either before or after
|
|
# the subcommand name. In order to do this we add them to the main parser
|
|
# and to subcommand parsers. In order for this to work, we need to tell
|
|
# argparse that the default value is SUPPRESS, so that the default values
|
|
# from the subparser arguments won't override values set by the user from
|
|
# the main parser. We maintain our own list of desired defaults in the
|
|
# common_defaults dictionary, and manually set those if the argument wasn't
|
|
# present at all.
|
|
common_args = argparse.ArgumentParser(add_help=False)
|
|
common_defaults = {}
|
|
|
|
def add_common_arg(*args, **kwargs):
|
|
var_name = get_arg_var_name(args)
|
|
default_value = kwargs.pop("default", None)
|
|
common_defaults[var_name] = default_value
|
|
kwargs["default"] = argparse.SUPPRESS
|
|
common_args.add_argument(*args, **kwargs)
|
|
|
|
add_common_arg("--scratch-path", help="Where to maintain checkouts and build dirs")
|
|
add_common_arg(
|
|
"--vcvars-path", default=None, help="Path to the vcvarsall.bat on Windows."
|
|
)
|
|
add_common_arg(
|
|
"--install-prefix",
|
|
help=(
|
|
"Where the final build products will be installed "
|
|
"(default is [scratch-path]/installed)"
|
|
),
|
|
)
|
|
add_common_arg(
|
|
"--num-jobs",
|
|
type=int,
|
|
help=(
|
|
"Number of concurrent jobs to use while building. "
|
|
"(default=number of cpu cores)"
|
|
),
|
|
)
|
|
add_common_arg(
|
|
"--use-shipit",
|
|
help="use the real ShipIt instead of the simple shipit transformer",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
add_common_arg(
|
|
"--facebook-internal",
|
|
help="Setup the build context as an FB internal build",
|
|
action="store_true",
|
|
default=None,
|
|
)
|
|
add_common_arg(
|
|
"--no-facebook-internal",
|
|
help="Perform a non-FB internal build, even when in an fbsource repository",
|
|
action="store_false",
|
|
dest="facebook_internal",
|
|
)
|
|
|
|
ap = argparse.ArgumentParser(
|
|
description="Get and build dependencies and projects", parents=[common_args]
|
|
)
|
|
sub = ap.add_subparsers(
|
|
# metavar suppresses the long and ugly default list of subcommands on a
|
|
# single line. We still render the nicer list below where we would
|
|
# have shown the nasty one.
|
|
metavar="",
|
|
title="Available commands",
|
|
help="",
|
|
)
|
|
|
|
add_subcommands(sub, common_args)
|
|
|
|
args = ap.parse_args()
|
|
for var_name, default_value in common_defaults.items():
|
|
if not hasattr(args, var_name):
|
|
setattr(args, var_name, default_value)
|
|
|
|
return ap, args
|
|
|
|
|
|
def main():
|
|
ap, args = parse_args()
|
|
if getattr(args, "func", None) is None:
|
|
ap.print_help()
|
|
return 0
|
|
try:
|
|
return args.func(args)
|
|
except UsageError as exc:
|
|
ap.error(str(exc))
|
|
return 1
|
|
except TransientFailure as exc:
|
|
print("TransientFailure: %s" % str(exc))
|
|
# This return code is treated as a retryable transient infrastructure
|
|
# error by Facebook's internal CI, rather than eg: a build or code
|
|
# related error that needs to be fixed before progress can be made.
|
|
return 128
|
|
except subprocess.CalledProcessError as exc:
|
|
print("%s" % str(exc), file=sys.stderr)
|
|
print("!! Failed", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|