mirror of
https://github.com/facebook/sapling.git
synced 2024-10-10 16:57:49 +03:00
d2520b62e2
Summary: This should fix the Travis CI builds. It adds rust toolchain support inside docker and sets the required THRIFT env variable. Pull Request resolved: https://github.com/facebookexperimental/rust-shed/pull/3 Reviewed By: krallin Differential Revision: D18905608 Pulled By: lukaspiatkowski fbshipit-source-id: 5db1eff6f215a6617d8acaa0c99a62d45225956b
470 lines
17 KiB
Python
470 lines
17 KiB
Python
#!/usr/bin/env python
|
|
# Copyright (c) Facebook, Inc. and its affiliates.
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
from __future__ import unicode_literals
|
|
'''
|
|
|
|
This is a small DSL to describe builds of Facebook's open-source projects
|
|
that are published to Github from a single internal repo, including projects
|
|
that depend on folly, wangle, proxygen, fbthrift, etc.
|
|
|
|
This file defines the interface of the DSL, and common utilieis, but you
|
|
will have to instantiate a specific builder, with specific options, in
|
|
order to get work done -- see e.g. make_docker_context.py.
|
|
|
|
== Design notes ==
|
|
|
|
Goals:
|
|
|
|
- A simple declarative language for what needs to be checked out & built,
|
|
how, in what order.
|
|
|
|
- The same specification should work for external continuous integration
|
|
builds (e.g. Travis + Docker) and for internal VM-based continuous
|
|
integration builds.
|
|
|
|
- One should be able to build without root, and to install to a prefix.
|
|
|
|
Non-goals:
|
|
|
|
- General usefulness. The only point of this is to make it easier to build
|
|
and test Facebook's open-source services.
|
|
|
|
Ideas for the future -- these may not be very good :)
|
|
|
|
- Especially on Ubuntu 14.04 the current initial setup is inefficient:
|
|
we add PPAs after having installed a bunch of packages -- this prompts
|
|
reinstalls of large amounts of code. We also `apt-get update` a few
|
|
times.
|
|
|
|
- A "shell script" builder. Like DockerFBCodeBuilder, but outputs a
|
|
shell script that runs outside of a container. Or maybe even
|
|
synchronously executes the shell commands, `make`-style.
|
|
|
|
- A "Makefile" generator. That might make iterating on builds even quicker
|
|
than what you can currently get with Docker build caching.
|
|
|
|
- Generate a rebuild script that can be run e.g. inside the built Docker
|
|
container by tagging certain steps with list-inheriting Python objects:
|
|
* do change directories
|
|
* do NOT `git clone` -- if we want to update code this should be a
|
|
separate script that e.g. runs rebase on top of specific targets
|
|
across all the repos.
|
|
* do NOT install software (most / all setup can be skipped)
|
|
* do NOT `autoreconf` or `configure`
|
|
* do `make` and `cmake`
|
|
|
|
- If we get non-Debian OSes, part of ccache setup should be factored out.
|
|
'''
|
|
|
|
import os
|
|
import re
|
|
|
|
from shell_quoting import path_join, shell_join, ShellQuoted
|
|
|
|
|
|
def _read_project_github_hashes():
|
|
base_dir = 'deps/github_hashes/' # trailing slash used in regex below
|
|
for dirname, _, files in os.walk(base_dir):
|
|
for filename in files:
|
|
path = os.path.join(dirname, filename)
|
|
with open(path) as f:
|
|
m_proj = re.match('^' + base_dir + '(.*)-rev\.txt$', path)
|
|
if m_proj is None:
|
|
raise RuntimeError('Not a hash file? {0}'.format(path))
|
|
m_hash = re.match('^Subproject commit ([0-9a-f]+)\n$', f.read())
|
|
if m_hash is None:
|
|
raise RuntimeError('No hash in {0}'.format(path))
|
|
yield m_proj.group(1), m_hash.group(1)
|
|
|
|
|
|
class FBCodeBuilder(object):
|
|
|
|
def __init__(self, **kwargs):
|
|
self._options_do_not_access = kwargs # Use .option() instead.
|
|
# This raises upon detecting options that are specified but unused,
|
|
# because otherwise it is very easy to make a typo in option names.
|
|
self.options_used = set()
|
|
# Mark 'projects_dir' used even if the build installs no github
|
|
# projects. This is needed because driver programs like
|
|
# `shell_builder.py` unconditionally set this for all builds.
|
|
self._github_dir = self.option('projects_dir')
|
|
self._github_hashes = dict(_read_project_github_hashes())
|
|
|
|
def __repr__(self):
|
|
return '{0}({1})'.format(
|
|
self.__class__.__name__,
|
|
', '.join(
|
|
'{0}={1}'.format(k, repr(v))
|
|
for k, v in self._options_do_not_access.items()
|
|
)
|
|
)
|
|
|
|
def option(self, name, default=None):
|
|
value = self._options_do_not_access.get(name, default)
|
|
if value is None:
|
|
raise RuntimeError('Option {0} is required'.format(name))
|
|
self.options_used.add(name)
|
|
return value
|
|
|
|
def has_option(self, name):
|
|
return name in self._options_do_not_access
|
|
|
|
def add_option(self, name, value):
|
|
if name in self._options_do_not_access:
|
|
raise RuntimeError('Option {0} already set'.format(name))
|
|
self._options_do_not_access[name] = value
|
|
|
|
#
|
|
# Abstract parts common to every installation flow
|
|
#
|
|
|
|
def render(self, steps):
|
|
'''
|
|
|
|
Converts nested actions to your builder's expected output format.
|
|
Typically takes the output of build().
|
|
|
|
'''
|
|
res = self._render_impl(steps) # Implementation-dependent
|
|
# Now that the output is rendered, we expect all options to have
|
|
# been used.
|
|
unused_options = set(self._options_do_not_access)
|
|
unused_options -= self.options_used
|
|
if unused_options:
|
|
raise RuntimeError(
|
|
'Unused options: {0} -- please check if you made a typo '
|
|
'in any of them. Those that are truly not useful should '
|
|
'be not be set so that this typo detection can be useful.'
|
|
.format(unused_options)
|
|
)
|
|
return res
|
|
|
|
def build(self, steps):
|
|
if not steps:
|
|
raise RuntimeError('Please ensure that the config you are passing '
|
|
'contains steps')
|
|
return [self.setup(), self.diagnostics()] + steps
|
|
|
|
def setup(self):
|
|
'Your builder may want to install packages here.'
|
|
raise NotImplementedError
|
|
|
|
def diagnostics(self):
|
|
'Log some system diagnostics before/after setup for ease of debugging'
|
|
# The builder's repr is not used in a command to avoid pointlessly
|
|
# invalidating Docker's build cache.
|
|
return self.step('Diagnostics', [
|
|
self.comment('Builder {0}'.format(repr(self))),
|
|
self.run(ShellQuoted('hostname')),
|
|
self.run(ShellQuoted('cat /etc/issue || echo no /etc/issue')),
|
|
self.run(ShellQuoted('g++ --version || echo g++ not installed')),
|
|
self.run(ShellQuoted('cmake --version || echo cmake not installed')),
|
|
])
|
|
|
|
def step(self, name, actions):
|
|
'A labeled collection of actions or other steps'
|
|
raise NotImplementedError
|
|
|
|
def run(self, shell_cmd):
|
|
'Run this bash command'
|
|
raise NotImplementedError
|
|
|
|
def set_env(self, key, value):
|
|
'Set the environment "key" to value "value"'
|
|
raise NotImplementedError
|
|
|
|
def workdir(self, dir):
|
|
'Create this directory if it does not exist, and change into it'
|
|
raise NotImplementedError
|
|
|
|
def copy_local_repo(self, dir, dest_name):
|
|
'''
|
|
Copy the local repo at `dir` into this step's `workdir()`, analog of:
|
|
cp -r /path/to/folly folly
|
|
'''
|
|
raise NotImplementedError
|
|
|
|
def python_deps(self):
|
|
return [
|
|
'wheel',
|
|
'cython==0.28.6',
|
|
]
|
|
|
|
def debian_deps(self):
|
|
return [
|
|
'autoconf-archive',
|
|
'bison',
|
|
'build-essential',
|
|
'cmake',
|
|
'curl',
|
|
'flex',
|
|
'git',
|
|
'gperf',
|
|
'joe',
|
|
'libboost-all-dev',
|
|
'libcap-dev',
|
|
'libdouble-conversion-dev',
|
|
'libevent-dev',
|
|
'libgflags-dev',
|
|
'libgoogle-glog-dev',
|
|
'libkrb5-dev',
|
|
'libpcre3-dev',
|
|
'libpthread-stubs0-dev',
|
|
'libnuma-dev',
|
|
'libsasl2-dev',
|
|
'libsnappy-dev',
|
|
'libsqlite3-dev',
|
|
'libssl-dev',
|
|
'libtool',
|
|
'netcat-openbsd',
|
|
'pkg-config',
|
|
'sudo',
|
|
'unzip',
|
|
'wget',
|
|
'python3-venv',
|
|
]
|
|
|
|
#
|
|
# Specific build helpers
|
|
#
|
|
|
|
def install_debian_deps(self):
|
|
actions = [
|
|
self.run(
|
|
ShellQuoted('apt-get update && apt-get install -yq {deps}').format(
|
|
deps=shell_join(' ', (
|
|
ShellQuoted(dep) for dep in self.debian_deps())))
|
|
),
|
|
]
|
|
gcc_version = self.option('gcc_version')
|
|
|
|
# Make the selected GCC the default before building anything
|
|
actions.extend([
|
|
self.run(ShellQuoted('apt-get install -yq {c} {cpp}').format(
|
|
c=ShellQuoted('gcc-{v}').format(v=gcc_version),
|
|
cpp=ShellQuoted('g++-{v}').format(v=gcc_version),
|
|
)),
|
|
self.run(ShellQuoted(
|
|
'update-alternatives --install /usr/bin/gcc gcc {c} 40 '
|
|
'--slave /usr/bin/g++ g++ {cpp}'
|
|
).format(
|
|
c=ShellQuoted('/usr/bin/gcc-{v}').format(v=gcc_version),
|
|
cpp=ShellQuoted('/usr/bin/g++-{v}').format(v=gcc_version),
|
|
)),
|
|
self.run(ShellQuoted('update-alternatives --config gcc')),
|
|
])
|
|
|
|
actions.extend(self.debian_ccache_setup_steps())
|
|
|
|
return self.step('Install packages for Debian-based OS', actions)
|
|
|
|
def create_python_venv(self):
|
|
actions = []
|
|
if self.option("PYTHON_VENV", "OFF") == "ON":
|
|
actions.append(self.run(ShellQuoted("python3 -m venv {p}").format(
|
|
p=path_join(self.option('prefix'), "venv"))))
|
|
return(actions)
|
|
|
|
def python_venv(self):
|
|
actions = []
|
|
if self.option("PYTHON_VENV", "OFF") == "ON":
|
|
actions.append(ShellQuoted("source {p}").format(
|
|
p=path_join(self.option('prefix'), "venv", "bin", "activate")))
|
|
|
|
actions.append(self.run(
|
|
ShellQuoted("python3 -m pip install {deps}").format(
|
|
deps=shell_join(' ', (ShellQuoted(dep) for dep in
|
|
self.python_deps())))))
|
|
return(actions)
|
|
|
|
def enable_rust_toolchain(self, toolchain="stable", is_bootstrap=True):
|
|
choices = set(["stable", "beta", "nightly"])
|
|
|
|
assert toolchain in choices, (
|
|
"while enabling rust toolchain: {} is not in {}"
|
|
).format(toolchain, choices)
|
|
|
|
rust_toolchain_opt = (toolchain, is_bootstrap)
|
|
prev_opt = self.option("rust_toolchain", rust_toolchain_opt)
|
|
assert prev_opt == rust_toolchain_opt, (
|
|
"while enabling rust toolchain: previous toolchain already set to"
|
|
" {}, but trying to set it to {} now"
|
|
).format(prev_opt, rust_toolchain_opt)
|
|
|
|
self.add_option("rust_toolchain", rust_toolchain_opt)
|
|
|
|
def rust_toolchain(self):
|
|
actions = []
|
|
if self.option("rust_toolchain", False):
|
|
(toolchain, is_bootstrap) = self.option("rust_toolchain")
|
|
rust_dir = path_join(self.option("prefix"), "rust")
|
|
actions = [
|
|
self.set_env("CARGO_HOME", rust_dir),
|
|
self.set_env("RUSTUP_HOME", rust_dir),
|
|
self.set_env("RUSTC_BOOTSTRAP", "1" if is_bootstrap else "0"),
|
|
self.run(
|
|
ShellQuoted(
|
|
"curl -sSf https://build.travis-ci.com/files/rustup-init.sh"
|
|
" | sh -s --"
|
|
" --default-toolchain={r} "
|
|
" --profile=minimal"
|
|
" --no-modify-path"
|
|
" -y"
|
|
).format(p=rust_dir, r=toolchain)
|
|
),
|
|
self.set_env(
|
|
"PATH",
|
|
ShellQuoted("{p}:$PATH").format(p=path_join(rust_dir, "bin")),
|
|
),
|
|
self.run(ShellQuoted("rustup update")),
|
|
self.run(ShellQuoted("rustc --version")),
|
|
self.run(ShellQuoted("rustup --version")),
|
|
self.run(ShellQuoted("cargo --version")),
|
|
]
|
|
return actions
|
|
|
|
def debian_ccache_setup_steps(self):
|
|
return [] # It's ok to ship a renderer without ccache support.
|
|
|
|
def github_project_workdir(self, project, path):
|
|
# Only check out a non-default branch if requested. This especially
|
|
# makes sense when building from a local repo.
|
|
git_hash = self.option(
|
|
'{0}:git_hash'.format(project),
|
|
# Any repo that has a hash in deps/github_hashes defaults to
|
|
# that, with the goal of making builds maximally consistent.
|
|
self._github_hashes.get(project, '')
|
|
)
|
|
maybe_change_branch = [
|
|
self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
|
|
] if git_hash else []
|
|
|
|
local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
|
|
return self.step('Check out {0}, workdir {1}'.format(project, path), [
|
|
self.workdir(self._github_dir),
|
|
self.run(
|
|
ShellQuoted('git clone {opts} https://github.com/{p}').format(
|
|
p=project,
|
|
opts=ShellQuoted(self.option('{}:git_clone_opts'.format(project), '')))
|
|
) if not local_repo_dir else self.copy_local_repo(
|
|
local_repo_dir, os.path.basename(project)
|
|
),
|
|
self.workdir(
|
|
path_join(self._github_dir, os.path.basename(project), path),
|
|
),
|
|
] + maybe_change_branch)
|
|
|
|
def fb_github_project_workdir(self, project_and_path, github_org='facebook'):
|
|
'This helper lets Facebook-internal CI special-cases FB projects'
|
|
project, path = project_and_path.split('/', 1)
|
|
return self.github_project_workdir(github_org + '/' + project, path)
|
|
|
|
def _make_vars(self, make_vars):
|
|
return shell_join(' ', (
|
|
ShellQuoted('{k}={v}').format(k=k, v=v)
|
|
for k, v in ({} if make_vars is None else make_vars).items()
|
|
))
|
|
|
|
def parallel_make(self, make_vars=None):
|
|
return self.run(ShellQuoted('make -j {n} VERBOSE=1 {vars}').format(
|
|
n=self.option('make_parallelism'),
|
|
vars=self._make_vars(make_vars),
|
|
))
|
|
|
|
def make_and_install(self, make_vars=None):
|
|
return [
|
|
self.parallel_make(make_vars),
|
|
self.run(ShellQuoted('make install VERBOSE=1 {vars}').format(
|
|
vars=self._make_vars(make_vars),
|
|
)),
|
|
]
|
|
|
|
def configure(self, name=None):
|
|
autoconf_options = {}
|
|
if name is not None:
|
|
autoconf_options.update(
|
|
self.option('{0}:autoconf_options'.format(name), {})
|
|
)
|
|
return [
|
|
self.run(ShellQuoted(
|
|
'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" '
|
|
'CFLAGS="$CFLAGS -I"{p}"/include" '
|
|
'CPPFLAGS="$CPPFLAGS -I"{p}"/include" '
|
|
'PY_PREFIX={p} '
|
|
'./configure --prefix={p} {args}'
|
|
).format(
|
|
p=self.option('prefix'),
|
|
args=shell_join(' ', (
|
|
ShellQuoted('{k}={v}').format(k=k, v=v)
|
|
for k, v in autoconf_options.items()
|
|
)),
|
|
)),
|
|
]
|
|
|
|
def autoconf_install(self, name):
|
|
return self.step('Build and install {0}'.format(name), [
|
|
self.run(ShellQuoted('autoreconf -ivf')),
|
|
] + self.configure() + self.make_and_install())
|
|
|
|
def cmake_configure(self, name, cmake_path='..'):
|
|
cmake_defines = {
|
|
'BUILD_SHARED_LIBS': 'ON',
|
|
'CMAKE_INSTALL_PREFIX': self.option('prefix'),
|
|
}
|
|
cmake_defines.update(
|
|
self.option('{0}:cmake_defines'.format(name), {})
|
|
)
|
|
return [
|
|
self.run(ShellQuoted(
|
|
'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" '
|
|
'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" '
|
|
'cmake {args} {cmake_path}'
|
|
).format(
|
|
p=self.option('prefix'),
|
|
args=shell_join(' ', (
|
|
ShellQuoted('-D{k}={v}').format(k=k, v=v)
|
|
for k, v in cmake_defines.items()
|
|
)),
|
|
cmake_path=cmake_path,
|
|
)),
|
|
]
|
|
|
|
def cmake_install(self, name, cmake_path='..'):
|
|
return self.step(
|
|
'Build and install {0}'.format(name),
|
|
self.cmake_configure(name, cmake_path) + self.make_and_install()
|
|
)
|
|
|
|
def cargo_build(self, name):
|
|
return self.step(
|
|
"Build {0}".format(name),
|
|
[
|
|
self.run(
|
|
ShellQuoted("cargo build -j {n}").format(
|
|
n=self.option("make_parallelism")
|
|
)
|
|
)
|
|
],
|
|
)
|
|
|
|
def fb_github_autoconf_install(self, project_and_path, github_org='facebook'):
|
|
return [
|
|
self.fb_github_project_workdir(project_and_path, github_org),
|
|
self.autoconf_install(project_and_path),
|
|
]
|
|
|
|
def fb_github_cmake_install(self, project_and_path, cmake_path='..', github_org='facebook'):
|
|
return [
|
|
self.fb_github_project_workdir(project_and_path, github_org),
|
|
self.cmake_install(project_and_path, cmake_path),
|
|
]
|
|
|
|
def fb_github_cargo_build(self, project_and_path, github_org="facebook"):
|
|
return [
|
|
self.fb_github_project_workdir(project_and_path, github_org),
|
|
self.cargo_build(project_and_path),
|
|
]
|