mirror of
https://github.com/facebook/sapling.git
synced 2024-08-16 14:00:24 +03:00
bdd335b648
Summary: as above; this makes the build faster and makes us slightly less sensitive to problems with building its tests. Reviewed By: pkaush Differential Revision: D14137089 fbshipit-source-id: 7edf4ac4a24bd90db55410363821f30635010627
487 lines
14 KiB
Python
Executable File
487 lines
14 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# Copyright (c) 2004-present, Facebook, Inc.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the BSD-style license found in the
|
|
# LICENSE file in the root directory of this source tree. An additional grant
|
|
# of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
from __future__ import absolute_import, division, print_function, unicode_literals
|
|
|
|
import argparse
|
|
import os
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
try:
|
|
from shlex import quote as shellquote
|
|
except ImportError:
|
|
from pipes import quote as shellquote
|
|
|
|
|
|
class BuildOptions(object):
|
|
def __init__(self, num_jobs, external_dir, install_dir):
|
|
self.num_jobs = num_jobs
|
|
if not self.num_jobs:
|
|
import multiprocessing
|
|
|
|
self.num_jobs = multiprocessing.cpu_count()
|
|
|
|
self.external_dir = external_dir
|
|
self.install_dir = install_dir
|
|
|
|
def project_dir(self, name, *paths):
|
|
return os.path.join(self.external_dir, name, *paths)
|
|
|
|
|
|
class Project(object):
|
|
def __init__(self, name, opts, updater, builder):
|
|
self.name = name
|
|
self.opts = opts
|
|
self.updater = updater
|
|
self.builder = builder
|
|
self.path = self.opts.project_dir(self.name)
|
|
|
|
def update(self):
|
|
self.updater.update(self)
|
|
|
|
def ensure_checkedout(self):
|
|
self.updater.ensure_checkedout(self)
|
|
|
|
def build(self):
|
|
self.builder.build(self)
|
|
|
|
def clean(self):
|
|
self.updater.clean(self)
|
|
|
|
|
|
class GitUpdater(object):
|
|
def __init__(self, repo, branch="master"):
|
|
self.origin_repo = repo
|
|
self.branch = branch
|
|
|
|
def ensure_checkedout(self, project):
|
|
if not os.path.exists(project.path):
|
|
self._checkout(project)
|
|
|
|
def update(self, project):
|
|
if os.path.exists(project.path):
|
|
print("Updating %s..." % project.name)
|
|
run_cmd(["git", "-C", project.path, "fetch", "origin"])
|
|
run_cmd(
|
|
[
|
|
"git",
|
|
"-C",
|
|
project.path,
|
|
"merge",
|
|
"--ff-only",
|
|
"origin/%s" % self.branch,
|
|
]
|
|
)
|
|
else:
|
|
self._checkout(project)
|
|
|
|
def _checkout(self, project):
|
|
print("Cloning %s..." % project.name)
|
|
run_cmd(
|
|
["git", "clone", self.origin_repo, project.path, "--branch", self.branch]
|
|
)
|
|
run_cmd(["git", "submodule", "update", "--init"], cwd=project.path)
|
|
|
|
def clean(self, project):
|
|
run_cmd(["git", "-C", project.path, "clean", "-fxd"])
|
|
|
|
|
|
def fixup_env_for_darwin(env):
|
|
def add_flag(name, item, separator, append=True):
|
|
val = env.get(name, "").split(separator)
|
|
if append:
|
|
val.append(item)
|
|
else:
|
|
val.insert(0, item)
|
|
env[name] = separator.join(val)
|
|
|
|
# The brew/openssl installation situation is a bit too weird for vanilla
|
|
# cmake logic to find, and most packages don't deal with this correctly,
|
|
# so inject these into the environment to give them a hand
|
|
add_flag("PKG_CONFIG_PATH", "/usr/local/opt/openssl/lib/pkgconfig", ":")
|
|
add_flag("LDFLAGS", "-L/usr/local/opt/openssl/lib", " ")
|
|
add_flag("CPPFLAGS", "-I/usr/local/opt/openssl/include", " ")
|
|
|
|
# system bison is ancient, so ensure that the brew installed one takes
|
|
# precedence. Brew refuses to to install or link bison into /usr/local/bin,
|
|
# so we have to insert this opt path instead. Likewise for flex.
|
|
add_flag("PATH", "/usr/local/opt/bison/bin", ":", append=False)
|
|
add_flag("PATH", "/usr/local/opt/flex/bin", ":", append=False)
|
|
|
|
# flex generates code that sprinkles the `register` keyword liberally
|
|
# and the thrift compilation flags hate that in C++17 code. Disable
|
|
# the warning that promotes this to an error.
|
|
add_flag("CXXFLAGS", "-Wno-register", " ")
|
|
|
|
|
|
class BuilderBase(object):
|
|
def __init__(self, subdir=None, env=None, build_dir=None):
|
|
self.env = os.environ.copy()
|
|
if sys.platform == "darwin":
|
|
fixup_env_for_darwin(self.env)
|
|
if env:
|
|
self.env.update(env)
|
|
|
|
self.subdir = subdir
|
|
self.build_dir = build_dir
|
|
self._build_path = None
|
|
|
|
def _run_cmd(self, cmd):
|
|
run_cmd(cmd=cmd, env=self.env, cwd=self._build_path)
|
|
|
|
def build(self, project):
|
|
print("Building %s..." % project.name)
|
|
if self.subdir:
|
|
build_path = os.path.join(project.path, self.subdir)
|
|
else:
|
|
build_path = project.path
|
|
|
|
if self.build_dir is not None:
|
|
build_path = os.path.join(build_path, self.build_dir)
|
|
if not os.path.isdir(build_path):
|
|
os.mkdir(build_path)
|
|
|
|
self._build_path = build_path
|
|
try:
|
|
self._build(project)
|
|
finally:
|
|
self._build_path = None
|
|
|
|
|
|
class NullBuilder(BuilderBase):
|
|
def __init__(self):
|
|
super(NullBuilder, self).__init__()
|
|
|
|
def _build(self, project):
|
|
pass
|
|
|
|
|
|
class MakeBuilder(BuilderBase):
|
|
def __init__(self, subdir=None, env=None, args=None):
|
|
super(MakeBuilder, self).__init__(subdir=subdir, env=env)
|
|
self.args = args or []
|
|
|
|
def _build(self, project):
|
|
cmd = ["make", "-j%s" % project.opts.num_jobs] + self.args
|
|
self._run_cmd(cmd)
|
|
|
|
install_cmd = ["make", "install", "PREFIX=" + project.opts.install_dir]
|
|
self._run_cmd(install_cmd)
|
|
|
|
|
|
class AutoconfBuilder(BuilderBase):
|
|
def __init__(self, subdir=None, env=None, args=None):
|
|
super(AutoconfBuilder, self).__init__(subdir=subdir, env=env)
|
|
self.args = args or []
|
|
|
|
def _build(self, project):
|
|
configure_path = os.path.join(self._build_path, "configure")
|
|
if not os.path.exists(configure_path):
|
|
self._run_cmd(["autoreconf", "--install"])
|
|
configure_cmd = [
|
|
configure_path,
|
|
"--prefix=" + project.opts.install_dir,
|
|
] + self.args
|
|
self._run_cmd(configure_cmd)
|
|
self._run_cmd(["make", "-j%s" % project.opts.num_jobs])
|
|
self._run_cmd(["make", "install"])
|
|
|
|
|
|
class CMakeBuilder(BuilderBase):
|
|
def __init__(self, subdir=None, env=None, defines=None):
|
|
super(CMakeBuilder, self).__init__(subdir=subdir, env=env, build_dir="_build")
|
|
self.defines = defines or {}
|
|
|
|
def _build(self, project):
|
|
defines = {
|
|
"CMAKE_INSTALL_PREFIX": project.opts.install_dir,
|
|
"BUILD_SHARED_LIBS": "OFF",
|
|
}
|
|
defines.update(self.defines)
|
|
define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()]
|
|
|
|
self._run_cmd(["cmake", "configure", ".."] + define_args)
|
|
self._run_cmd(["make", "-j%s" % project.opts.num_jobs])
|
|
self._run_cmd(["make", "install"])
|
|
|
|
|
|
def run_cmd(cmd, env=None, cwd=None):
|
|
cmd_str = " ".join(shellquote(arg) for arg in cmd)
|
|
print("+ " + cmd_str)
|
|
subprocess.check_call(cmd, env=env, cwd=cwd)
|
|
|
|
|
|
def install_apt(pkgs):
|
|
cmd = ["sudo", "apt-get", "install", "-yq"] + pkgs
|
|
run_cmd(cmd)
|
|
|
|
|
|
def get_projects(opts):
|
|
projects = [
|
|
Project(
|
|
"mstch",
|
|
opts,
|
|
GitUpdater("https://github.com/no1msd/mstch.git"),
|
|
CMakeBuilder(),
|
|
),
|
|
Project(
|
|
"cpptoml",
|
|
opts,
|
|
GitUpdater("https://github.com/skystrife/cpptoml.git"),
|
|
CMakeBuilder(),
|
|
),
|
|
Project(
|
|
"rocksdb",
|
|
opts,
|
|
GitUpdater("https://github.com/facebook/rocksdb.git"),
|
|
CMakeBuilder(defines={"WITH_SNAPPY": "ON", "WITH_TESTS": "OFF"}),
|
|
),
|
|
Project(
|
|
"googletest",
|
|
opts,
|
|
GitUpdater("https://github.com/google/googletest.git"),
|
|
CMakeBuilder(),
|
|
),
|
|
Project(
|
|
"folly",
|
|
opts,
|
|
GitUpdater("https://github.com/facebook/folly.git"),
|
|
CMakeBuilder(defines={"BUILD_TESTS": "OFF"}),
|
|
),
|
|
Project(
|
|
"libsodium",
|
|
opts,
|
|
GitUpdater("https://github.com/jedisct1/libsodium.git"),
|
|
AutoconfBuilder(),
|
|
),
|
|
Project(
|
|
"fizz",
|
|
opts,
|
|
GitUpdater("https://github.com/facebookincubator/fizz.git"),
|
|
CMakeBuilder(
|
|
subdir="fizz", defines={"BUILD_EXAMPLES": "OFF", "BUILD_TESTS": "OFF"}
|
|
),
|
|
),
|
|
Project(
|
|
"wangle",
|
|
opts,
|
|
GitUpdater("https://github.com/facebook/wangle.git"),
|
|
CMakeBuilder(subdir="wangle", defines={"BUILD_TESTS": "OFF"}),
|
|
),
|
|
Project(
|
|
"rsocket-cpp",
|
|
opts,
|
|
GitUpdater("https://github.com/rsocket/rsocket-cpp.git"),
|
|
CMakeBuilder(
|
|
defines={
|
|
"BUILD_EXAMPLES": "OFF",
|
|
"BUILD_BENCHMARKS": "OFF",
|
|
"BUILD_TESTS": "OFF",
|
|
}
|
|
),
|
|
),
|
|
Project(
|
|
"fbthrift",
|
|
opts,
|
|
GitUpdater("https://github.com/facebook/fbthrift.git"),
|
|
CMakeBuilder(),
|
|
),
|
|
]
|
|
if sys.platform == "darwin":
|
|
projects.append(
|
|
Project(
|
|
"osxfuse",
|
|
opts,
|
|
GitUpdater(
|
|
"https://github.com/osxfuse/osxfuse.git", branch="support/osxfuse-3"
|
|
),
|
|
NullBuilder(),
|
|
)
|
|
)
|
|
|
|
return projects
|
|
|
|
|
|
def get_linux_type():
|
|
try:
|
|
with open("/etc/os-release") as f:
|
|
data = f.read()
|
|
except EnvironmentError:
|
|
return (None, None)
|
|
|
|
os_vars = {}
|
|
for line in data.splitlines():
|
|
parts = line.split("=", 1)
|
|
if len(parts) != 2:
|
|
continue
|
|
key = parts[0].strip()
|
|
value_parts = shlex.split(parts[1].strip())
|
|
if not value_parts:
|
|
value = ""
|
|
else:
|
|
value = value_parts[0]
|
|
os_vars[key] = value
|
|
|
|
return os_vars.get("NAME"), os_vars.get("VERSION_ID")
|
|
|
|
|
|
def get_os_type():
|
|
if sys.platform.startswith("linux"):
|
|
return get_linux_type()
|
|
elif sys.platform.startswith("darwin"):
|
|
return ("darwin", None)
|
|
elif sys.platform == "windows":
|
|
return ("windows", sys.getwindowsversion().major)
|
|
else:
|
|
return (None, None)
|
|
|
|
|
|
def install_platform_deps():
|
|
os_name, os_version = get_os_type()
|
|
if os_name is None:
|
|
raise Exception("unable to detect OS type")
|
|
elif os_name == "Ubuntu":
|
|
# These dependencies have been tested on Ubuntu 16.04
|
|
print("Installing necessary Ubuntu packages...")
|
|
ubuntu_pkgs = (
|
|
"autoconf automake libdouble-conversion-dev "
|
|
"libssl-dev make zip git libtool g++ libboost-all-dev "
|
|
"libevent-dev flex bison libgoogle-glog-dev libkrb5-dev "
|
|
"libsnappy-dev libsasl2-dev libnuma-dev libcurl4-gnutls-dev "
|
|
"libpcap-dev libdb5.3-dev cmake libfuse-dev libgit2-dev mercurial "
|
|
"libsqlite3-dev libzstd-dev"
|
|
).split()
|
|
install_apt(ubuntu_pkgs)
|
|
elif os_name == "darwin":
|
|
print("Installing necessary packages via Homebrew...")
|
|
run_cmd(
|
|
[
|
|
"brew",
|
|
"install",
|
|
"autoconf",
|
|
"automake",
|
|
"bison",
|
|
"boost",
|
|
"boost-python",
|
|
"cmake",
|
|
"curl",
|
|
"double-conversion",
|
|
"flex",
|
|
"gflags",
|
|
"glog",
|
|
"libevent",
|
|
"libgit2",
|
|
"libtool",
|
|
"lz4",
|
|
"openssl",
|
|
"snappy",
|
|
"xz",
|
|
"zstd",
|
|
]
|
|
)
|
|
else:
|
|
# TODO: Handle distributions other than Ubuntu.
|
|
raise Exception(
|
|
"installing OS dependencies on %s is not supported yet" % (os_name,)
|
|
)
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument(
|
|
"-o",
|
|
"--external-dir",
|
|
help="The directory where external projects should be "
|
|
'created (default="external")',
|
|
)
|
|
ap.add_argument(
|
|
"-u",
|
|
"--update",
|
|
action="store_true",
|
|
default=False,
|
|
help="Updates the external projects repositories before building them",
|
|
)
|
|
ap.add_argument(
|
|
"-C",
|
|
"--clean",
|
|
action="store_true",
|
|
default=None,
|
|
help="Cleans the external project repositories before "
|
|
"building them (defaults to on when updating projects)",
|
|
)
|
|
ap.add_argument(
|
|
"--no-clean",
|
|
action="store_false",
|
|
default=None,
|
|
dest="clean",
|
|
help="Do not clean the external project repositories "
|
|
"even after updating them.",
|
|
)
|
|
ap.add_argument(
|
|
"-j",
|
|
"--jobs",
|
|
dest="num_jobs",
|
|
type=int,
|
|
default=None,
|
|
help="The number of jobs to run in parallel when building",
|
|
)
|
|
ap.add_argument(
|
|
"--install-dir",
|
|
help="Directory where external projects should be "
|
|
"installed (default=<external-dir>/install)",
|
|
)
|
|
ap.add_argument(
|
|
"--system-deps",
|
|
action="store_true",
|
|
default=False,
|
|
help="Also install necessary packages available from OS's package "
|
|
"distribution system",
|
|
)
|
|
|
|
args = ap.parse_args()
|
|
|
|
if args.external_dir is None:
|
|
script_dir = os.path.abspath(os.path.dirname(__file__))
|
|
args.external_dir = os.path.join(script_dir, "external")
|
|
if args.install_dir is None:
|
|
args.install_dir = os.path.join(args.external_dir, "install")
|
|
if args.clean is None:
|
|
args.clean = args.update
|
|
|
|
opts = BuildOptions(args.num_jobs, args.external_dir, args.install_dir)
|
|
|
|
if args.system_deps:
|
|
install_platform_deps()
|
|
|
|
if not os.path.isdir(opts.external_dir):
|
|
os.makedirs(opts.external_dir)
|
|
|
|
projects = get_projects(opts)
|
|
for project in projects:
|
|
if args.update:
|
|
project.update()
|
|
else:
|
|
project.ensure_checkedout()
|
|
|
|
if args.clean:
|
|
shutil.rmtree(args.install_dir)
|
|
for project in projects:
|
|
project.clean()
|
|
|
|
for project in projects:
|
|
project.build()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|