mirror of
https://github.com/facebook/sapling.git
synced 2024-10-11 09:17:30 +03:00
b04ce1a181
Summary: with the reshuffling around getdeps.py, and changes in folly/thrift/wangle, our internal package stopped building. This diff brings it back to life. It is a little bit interwined: * Introduce a `--facebook-internal` switch to getdeps that causes `fb` in the build context to be set to `on` rather than `off`. This allows specifying options in the manifest that are FB specific. * Introduce cmake options to control the WATCHMAN_STATE_DIR and whether cmake should attempt to create WATCHMAN_STATE_DIR when it is installed. * For FB specific builds on macOS, set the state dir to match our existing location and disable creation of WATCHMAN_STATE_DIR because that will ultimately be handling during package installation and we don't have privs for that on the machine generating the package. * Adjust Facebook CI scripts to use getdeps for building and testing watchman Reviewed By: simpkins Differential Revision: D15337970 fbshipit-source-id: 8ab3c1f810decf5f0b16337dc1cbe6dc73d26426
437 lines
14 KiB
Python
Executable File
437 lines
14 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright (c) 2019-present, Facebook, Inc.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the BSD-style license found in the
|
|
# LICENSE file in the root directory of this source tree. An additional grant
|
|
# of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
|
|
from __future__ import absolute_import, division, print_function, unicode_literals
|
|
|
|
import argparse
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
from getdeps.buildopts import setup_build_options
|
|
from getdeps.errors import TransientFailure
|
|
from getdeps.load import load_project, manifests_in_dependency_order
|
|
from getdeps.manifest import ManifestParser
|
|
from getdeps.platform import HostType, context_from_host_tuple
|
|
from getdeps.subcmd import SubCmd, add_subcommands, cmd
|
|
|
|
|
|
try:
|
|
import getdeps.facebook # noqa: F401
|
|
except ImportError:
|
|
# we don't ship the facebook specific subdir,
|
|
# so allow that to fail silently
|
|
pass
|
|
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "getdeps"))
|
|
|
|
|
|
@cmd("validate-manifest", "parse a manifest and validate that it is correct")
|
|
class ValidateManifest(SubCmd):
|
|
def run(self, args):
|
|
try:
|
|
ManifestParser(file_name=args.file_name)
|
|
print("OK", file=sys.stderr)
|
|
return 0
|
|
except Exception as exc:
|
|
print("ERROR: %s" % str(exc), file=sys.stderr)
|
|
return 1
|
|
|
|
def setup_parser(self, parser):
|
|
parser.add_argument("file_name", help="path to the manifest file")
|
|
|
|
|
|
@cmd("show-host-type", "outputs the host type tuple for the host machine")
|
|
class ShowHostType(SubCmd):
|
|
def run(self, args):
|
|
host = HostType()
|
|
print("%s" % host.as_tuple_string())
|
|
return 0
|
|
|
|
|
|
@cmd("fetch", "fetch the code for a given project")
|
|
class FetchCmd(SubCmd):
|
|
def setup_parser(self, parser):
|
|
parser.add_argument(
|
|
"project",
|
|
help=(
|
|
"name of the project or path to a manifest "
|
|
"file describing the project"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--recursive",
|
|
help="fetch the transitive deps also",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
parser.add_argument(
|
|
"--host-type",
|
|
help=(
|
|
"When recursively fetching, fetch deps for "
|
|
"this host type rather than the current system"
|
|
),
|
|
)
|
|
|
|
def run(self, args):
|
|
opts = setup_build_options(args)
|
|
manifest = load_project(opts, args.project)
|
|
ctx = context_from_host_tuple(args.host_type)
|
|
if args.recursive:
|
|
projects = manifests_in_dependency_order(opts, manifest, ctx)
|
|
else:
|
|
projects = [manifest]
|
|
for m in projects:
|
|
fetcher = m.create_fetcher(opts, ctx)
|
|
fetcher.update()
|
|
|
|
|
|
@cmd("list-deps", "lists the transitive deps for a given project")
|
|
class ListDepsCmd(SubCmd):
|
|
def run(self, args):
|
|
opts = setup_build_options(args)
|
|
manifest = load_project(opts, args.project)
|
|
ctx = context_from_host_tuple(args.host_type)
|
|
for m in manifests_in_dependency_order(opts, manifest, ctx):
|
|
print(m.name)
|
|
return 0
|
|
|
|
def setup_parser(self, parser):
|
|
parser.add_argument(
|
|
"--host-type",
|
|
help=(
|
|
"Produce the list for the specified host type, "
|
|
"rather than that of the current system"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"project",
|
|
help=(
|
|
"name of the project or path to a manifest "
|
|
"file describing the project"
|
|
),
|
|
)
|
|
|
|
|
|
def clean_dirs(opts):
|
|
for d in ["build", "installed", "extracted", "shipit"]:
|
|
d = os.path.join(opts.scratch_dir, d)
|
|
print("Cleaning %s..." % d)
|
|
if os.path.exists(d):
|
|
shutil.rmtree(d)
|
|
|
|
|
|
@cmd("clean", "clean up the scratch dir")
|
|
class CleanCmd(SubCmd):
|
|
def run(self, args):
|
|
opts = setup_build_options(args)
|
|
clean_dirs(opts)
|
|
|
|
|
|
@cmd("show-inst-dir", "print the installation dir for a given project")
|
|
class ShowInstDirCmd(SubCmd):
|
|
def run(self, args):
|
|
opts = setup_build_options(args)
|
|
manifest = load_project(opts, args.project)
|
|
ctx = context_from_host_tuple()
|
|
projects = manifests_in_dependency_order(opts, manifest, ctx)
|
|
manifests_by_name = {m.name: m for m in projects}
|
|
|
|
if args.recursive:
|
|
manifests = projects
|
|
else:
|
|
manifests = [manifest]
|
|
|
|
for m in manifests:
|
|
fetcher = m.create_fetcher(opts, ctx)
|
|
dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx)
|
|
inst_dir = dirs["inst_dir"]
|
|
print(inst_dir)
|
|
|
|
def setup_parser(self, parser):
|
|
parser.add_argument(
|
|
"project",
|
|
help=(
|
|
"name of the project or path to a manifest "
|
|
"file describing the project"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--recursive",
|
|
help="print the transitive deps also",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
|
|
|
|
@cmd("show-source-dir", "print the source dir for a given project")
|
|
class ShowSourceDirCmd(SubCmd):
|
|
def run(self, args):
|
|
opts = setup_build_options(args)
|
|
manifest = load_project(opts, args.project)
|
|
ctx = context_from_host_tuple()
|
|
|
|
if args.recursive:
|
|
manifests = manifests_in_dependency_order(opts, manifest, ctx)
|
|
else:
|
|
manifests = [manifest]
|
|
|
|
for m in manifests:
|
|
fetcher = m.create_fetcher(opts, ctx)
|
|
print(fetcher.get_src_dir())
|
|
|
|
def setup_parser(self, parser):
|
|
parser.add_argument(
|
|
"project",
|
|
help=(
|
|
"name of the project or path to a manifest "
|
|
"file describing the project"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--recursive",
|
|
help="print the transitive deps also",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
|
|
|
|
@cmd("build", "build a given project")
|
|
class BuildCmd(SubCmd):
|
|
def run(self, args):
|
|
opts = setup_build_options(args)
|
|
if args.clean:
|
|
clean_dirs(opts)
|
|
|
|
manifest = load_project(opts, args.project)
|
|
|
|
ctx = context_from_host_tuple(facebook_internal=args.facebook_internal)
|
|
print("Building on %s" % ctx)
|
|
projects = manifests_in_dependency_order(opts, manifest, ctx)
|
|
manifests_by_name = {m.name: m for m in projects}
|
|
|
|
# Accumulate the install directories so that the build steps
|
|
# can find their dep installation
|
|
install_dirs = []
|
|
|
|
for m in projects:
|
|
ctx = dict(ctx)
|
|
if args.enable_tests and m.name == manifest.name:
|
|
ctx["test"] = "on"
|
|
else:
|
|
ctx["test"] = "off"
|
|
fetcher = m.create_fetcher(opts, ctx)
|
|
|
|
if args.clean:
|
|
fetcher.clean()
|
|
|
|
dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx)
|
|
build_dir = dirs["build_dir"]
|
|
inst_dir = dirs["inst_dir"]
|
|
|
|
if m == manifest or not args.no_deps:
|
|
print("Assessing %s..." % m.name)
|
|
change_status = fetcher.update()
|
|
reconfigure = change_status.build_changed()
|
|
sources_changed = change_status.sources_changed()
|
|
|
|
built_marker = os.path.join(inst_dir, ".built-by-getdeps")
|
|
if os.path.exists(built_marker):
|
|
with open(built_marker, "r") as f:
|
|
built_hash = f.read().strip()
|
|
if built_hash != dirs["hash"]:
|
|
# Some kind of inconsistency with a prior build,
|
|
# let's run it again to be sure
|
|
os.unlink(built_marker)
|
|
reconfigure = True
|
|
|
|
if sources_changed or reconfigure or not os.path.exists(built_marker):
|
|
if os.path.exists(built_marker):
|
|
os.unlink(built_marker)
|
|
src_dir = fetcher.get_src_dir()
|
|
builder = m.create_builder(opts, src_dir, build_dir, inst_dir, ctx)
|
|
builder.build(install_dirs, reconfigure=reconfigure)
|
|
|
|
with open(built_marker, "w") as f:
|
|
f.write(dirs["hash"])
|
|
|
|
install_dirs.append(inst_dir)
|
|
|
|
def setup_parser(self, parser):
|
|
parser.add_argument(
|
|
"project",
|
|
help=(
|
|
"name of the project or path to a manifest "
|
|
"file describing the project"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--clean",
|
|
action="store_true",
|
|
default=False,
|
|
help=(
|
|
"Clean up the build and installation area prior to building, "
|
|
"causing the projects to be built from scratch"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--no-deps",
|
|
action="store_true",
|
|
default=False,
|
|
help=(
|
|
"Only build the named project, not its deps. "
|
|
"This is most useful after you've built all of the deps, "
|
|
"and helps to avoid waiting for relatively "
|
|
"slow up-to-date-ness checks"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--enable-tests",
|
|
action="store_true",
|
|
default=False,
|
|
help=(
|
|
"For the named project, build tests so that the test command "
|
|
"is able to execute tests"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--schedule-type", help="Indicates how the build was activated"
|
|
)
|
|
|
|
|
|
@cmd("test", "test a given project")
|
|
class TestCmd(SubCmd):
|
|
def run(self, args):
|
|
opts = setup_build_options(args)
|
|
manifest = load_project(opts, args.project)
|
|
|
|
ctx = context_from_host_tuple(facebook_internal=args.facebook_internal)
|
|
ctx["test"] = "on"
|
|
projects = manifests_in_dependency_order(opts, manifest, ctx)
|
|
manifests_by_name = {m.name: m for m in projects}
|
|
|
|
# Accumulate the install directories so that the test steps
|
|
# can find their dep installation
|
|
install_dirs = []
|
|
|
|
for m in projects:
|
|
fetcher = m.create_fetcher(opts, ctx)
|
|
|
|
dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx)
|
|
build_dir = dirs["build_dir"]
|
|
inst_dir = dirs["inst_dir"]
|
|
|
|
if m == manifest or args.test_all:
|
|
built_marker = os.path.join(inst_dir, ".built-by-getdeps")
|
|
if not os.path.exists(built_marker):
|
|
print("project %s has not been built" % m.name)
|
|
# TODO: we could just go ahead and build it here, but I
|
|
# want to tackle that as part of adding build-for-test
|
|
# support.
|
|
return 1
|
|
src_dir = fetcher.get_src_dir()
|
|
builder = m.create_builder(opts, src_dir, build_dir, inst_dir, ctx)
|
|
builder.run_tests(install_dirs, schedule_type=args.schedule_type)
|
|
|
|
install_dirs.append(inst_dir)
|
|
|
|
def setup_parser(self, parser):
|
|
parser.add_argument(
|
|
"project",
|
|
help=(
|
|
"name of the project or path to a manifest "
|
|
"file describing the project"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--test-all",
|
|
action="store_true",
|
|
default=False,
|
|
help="Enable running tests for the named project and all of its deps",
|
|
)
|
|
parser.add_argument(
|
|
"--schedule-type", help="Indicates how the build was activated"
|
|
)
|
|
|
|
|
|
def build_argparser():
|
|
common_args = argparse.ArgumentParser(add_help=False)
|
|
common_args.add_argument(
|
|
"--scratch-path", help="Where to maintain checkouts and build dirs"
|
|
)
|
|
common_args.add_argument(
|
|
"--install-prefix",
|
|
help=(
|
|
"Where the final build products will be installed "
|
|
"(default is [scratch-path]/installed)"
|
|
),
|
|
)
|
|
common_args.add_argument(
|
|
"--num-jobs",
|
|
type=int,
|
|
help=(
|
|
"Number of concurrent jobs to use while building. "
|
|
"(default=number of cpu cores)"
|
|
),
|
|
)
|
|
common_args.add_argument(
|
|
"--use-shipit",
|
|
help="use the real ShipIt instead of the simple shipit transformer",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
common_args.add_argument(
|
|
"--facebook-internal",
|
|
help="Setup the build context as an FB internal build",
|
|
action="store_true",
|
|
default=False,
|
|
)
|
|
|
|
ap = argparse.ArgumentParser(
|
|
description="Get and build dependencies and projects", parents=[common_args]
|
|
)
|
|
sub = ap.add_subparsers(
|
|
# metavar suppresses the long and ugly default list of subcommands on a
|
|
# single line. We still render the nicer list below where we would
|
|
# have shown the nasty one.
|
|
metavar="",
|
|
title="Available commands",
|
|
help="",
|
|
)
|
|
|
|
add_subcommands(sub, common_args)
|
|
|
|
return ap
|
|
|
|
|
|
def main():
|
|
ap = build_argparser()
|
|
args = ap.parse_args()
|
|
if getattr(args, "func", None) is None:
|
|
ap.print_help()
|
|
return 0
|
|
try:
|
|
return args.func(args)
|
|
except TransientFailure as exc:
|
|
print("TransientFailure: %s" % str(exc))
|
|
# This return code is treated as a retryable transient infrastructure
|
|
# error by Facebook's internal CI, rather than eg: a build or code
|
|
# related error that needs to be fixed before progress can be made.
|
|
return 128
|
|
except subprocess.CalledProcessError as exc:
|
|
print("%s" % str(exc), file=sys.stderr)
|
|
print("!! Failed", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|