facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-12-26 22:47:26 +03:00
Code Issues Projects Releases Wiki Activity

add fbcode_builder sources

Browse Source 
Summary:
Initial commit to include the fbcode_builder sources in the
eden github repository.

fbshipit-source-id: 49098cecda04a7e9dd9dcc0b569fffc96f0f719b
This commit is contained in:
Adam Simpkins 2019-04-26 11:29:35 -07:00
parent 1a562484b1
commit bcad7419bf
40 changed files with 2298 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
.travis.yml Normal file
View File

@ -0,0 +1,45 @@
# Facebook projects that use `fbcode_builder` for continuous integration
# share this Travis configuration to run builds via Docker.
sudo: required
# Docker disables IPv6 in containers by default. Enable it for unit tests that need [::1].
before_script:
- if [[ "$TRAVIS_OS_NAME" != "osx" ]];
then
sudo build/fbcode_builder/docker_enable_ipv6.sh;
fi
env:
global:
- travis_cache_dir=$HOME/travis_ccache
# Travis times out after 50 minutes. Very generously leave 10 minutes
# for setup (e.g. cache download, compression, and upload), so we never
# fail to cache the progress we made.
- docker_build_timeout=40m
cache:
# Our build caches can be 200-300MB, so increase the timeout to 7 minutes
# to make sure we never fail to cache the progress we made.
timeout: 420
directories:
- $HOME/travis_ccache # see docker_build_with_ccache.sh
# Ugh, `services:` must be in the matrix, or we get `docker: command not found`
# https://github.com/travis-ci/travis-ci/issues/5142
matrix:
include:
- env: ['os_image=ubuntu:16.04', gcc_version=5]
services: [docker]
addons:
apt:
packages: python2.7
script:
# We don't want to write the script inline because of Travis kludginess --
# it looks like it escapes " and \ in scripts when using `matrix:`.
- ./build/fbcode_builder/travis_docker_build.sh
notifications:
webhooks: https://code.facebook.com/travis/webhook/

1
build/deps/github_hashes/facebook/fbthrift-rev.txt Normal file
View File

@ -0,0 +1 @@
Subproject commit 9ec0a9819c2fcab1a9d61ade27d0ca93d30f0aa1

1
build/deps/github_hashes/facebook/folly-rev.txt Normal file
View File

@ -0,0 +1 @@
Subproject commit d52f0f52c7e442db6c66254953d353796bcf1011

1
build/deps/github_hashes/facebook/wangle-rev.txt Normal file
View File

@ -0,0 +1 @@
Subproject commit 540095b482bbe3b8d00a93a1055e33c3847bbf22

1
build/deps/github_hashes/facebookincubator/fizz-rev.txt Normal file
View File

@ -0,0 +1 @@
Subproject commit 7a70bc5af71d7ce1103ebdfaa4b3d9c20f69f183

5
build/fbcode_builder/.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
# Facebook-internal CI builds don't have write permission outside of the
# source tree, so we install all projects into this directory.
/facebook_ci
__pycache__/
*.pyc

80
build/fbcode_builder/CMake/FindGMock.cmake Normal file
View File

@ -0,0 +1,80 @@
# Copyright (c) Facebook, Inc. and its affiliates.
# Find libgmock
#
# LIBGMOCK_DEFINES - List of defines when using libgmock.
# LIBGMOCK_INCLUDE_DIR - where to find gmock/gmock.h, etc.
# LIBGMOCK_LIBRARIES - List of libraries when using libgmock.
# LIBGMOCK_FOUND - True if libgmock found.
IF (LIBGMOCK_INCLUDE_DIR)
# Already in cache, be silent
SET(LIBGMOCK_FIND_QUIETLY TRUE)
ENDIF ()
find_package(GTest CONFIG QUIET)
if (TARGET GTest::gmock)
get_target_property(LIBGMOCK_DEFINES GTest::gtest INTERFACE_COMPILE_DEFINITIONS)
if (NOT ${LIBGMOCK_DEFINES})
# Explicitly set to empty string if not found to avoid it being
# set to NOTFOUND and breaking compilation
set(LIBGMOCK_DEFINES "")
endif()
get_target_property(LIBGMOCK_INCLUDE_DIR GTest::gtest INTERFACE_INCLUDE_DIRECTORIES)
set(LIBGMOCK_LIBRARIES GTest::gmock_main GTest::gmock GTest::gtest)
set(LIBGMOCK_FOUND ON)
message(STATUS "Found gmock via config, defines=${LIBGMOCK_DEFINES}, include=${LIBGMOCK_INCLUDE_DIR}, libs=${LIBGMOCK_LIBRARIES}")
else()
FIND_PATH(LIBGMOCK_INCLUDE_DIR gmock/gmock.h)
FIND_LIBRARY(LIBGMOCK_MAIN_LIBRARY_DEBUG NAMES gmock_maind)
FIND_LIBRARY(LIBGMOCK_MAIN_LIBRARY_RELEASE NAMES gmock_main)
FIND_LIBRARY(LIBGMOCK_LIBRARY_DEBUG NAMES gmockd)
FIND_LIBRARY(LIBGMOCK_LIBRARY_RELEASE NAMES gmock)
FIND_LIBRARY(LIBGTEST_LIBRARY_DEBUG NAMES gtestd)
FIND_LIBRARY(LIBGTEST_LIBRARY_RELEASE NAMES gtest)
find_package(Threads REQUIRED)
INCLUDE(SelectLibraryConfigurations)
SELECT_LIBRARY_CONFIGURATIONS(LIBGMOCK_MAIN)
SELECT_LIBRARY_CONFIGURATIONS(LIBGMOCK)
SELECT_LIBRARY_CONFIGURATIONS(LIBGTEST)
set(LIBGMOCK_LIBRARIES
${LIBGMOCK_MAIN_LIBRARY}
${LIBGMOCK_LIBRARY}
${LIBGTEST_LIBRARY}
Threads::Threads
)
if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
# The GTEST_LINKED_AS_SHARED_LIBRARY macro must be set properly on Windows.
#
# There isn't currently an easy way to determine if a library was compiled as
# a shared library on Windows, so just assume we've been built against a
# shared build of gmock for now.
SET(LIBGMOCK_DEFINES "GTEST_LINKED_AS_SHARED_LIBRARY=1" CACHE STRING "")
endif()
# handle the QUIETLY and REQUIRED arguments and set LIBGMOCK_FOUND to TRUE if
# all listed variables are TRUE
INCLUDE(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(
GMock
DEFAULT_MSG
LIBGMOCK_MAIN_LIBRARY
LIBGMOCK_LIBRARY
LIBGTEST_LIBRARY
LIBGMOCK_LIBRARIES
LIBGMOCK_INCLUDE_DIR
)
MARK_AS_ADVANCED(
LIBGMOCK_DEFINES
LIBGMOCK_MAIN_LIBRARY
LIBGMOCK_LIBRARY
LIBGTEST_LIBRARY
LIBGMOCK_LIBRARIES
LIBGMOCK_INCLUDE_DIR
)
endif()

81
build/fbcode_builder/CMake/FindGflags.cmake Normal file
View File

@ -0,0 +1,81 @@
# Copyright (c) Facebook, Inc. and its affiliates.
# Find libgflags.
# There's a lot of compatibility cruft going on in here, both
# to deal with changes across the FB consumers of this and also
# to deal with variances in behavior of cmake itself.
#
# Since this file is named FindGflags.cmake the cmake convention
# is for the module to export both GFLAGS_FOUND and Gflags_FOUND.
# The convention expected by consumers is that we export the
# following variables, even though these do not match the cmake
# conventions:
#
# LIBGFLAGS_INCLUDE_DIR - where to find gflags/gflags.h, etc.
# LIBGFLAGS_LIBRARY - List of libraries when using libgflags.
# LIBGFLAGS_FOUND - True if libgflags found.
#
# We need to be able to locate gflags both from an installed
# cmake config file and just from the raw headers and libs, so
# test for the former and then the latter, and then stick
# the results together and export them into the variables
# listed above.
#
# For forwards compatibility, we export the following variables:
#
# gflags_INCLUDE_DIR - where to find gflags/gflags.h, etc.
# gflags_TARGET / GFLAGS_TARGET / gflags_LIBRARIES
# - List of libraries when using libgflags.
# gflags_FOUND - True if libgflags found.
#
IF (LIBGFLAGS_INCLUDE_DIR)
# Already in cache, be silent
SET(Gflags_FIND_QUIETLY TRUE)
ENDIF ()
find_package(gflags CONFIG QUIET)
if (gflags_FOUND)
if (NOT Gflags_FIND_QUIETLY)
message(STATUS "Found gflags from package config ${gflags_CONFIG}")
endif()
# Re-export the config-specified libs with our local names
set(LIBGFLAGS_LIBRARY ${gflags_LIBRARIES})
set(LIBGFLAGS_INCLUDE_DIR ${gflags_INCLUDE_DIR})
set(LIBGFLAGS_FOUND ${gflags_FOUND})
# cmake module compat
set(GFLAGS_FOUND ${gflags_FOUND})
set(Gflags_FOUND ${gflags_FOUND})
else()
FIND_PATH(LIBGFLAGS_INCLUDE_DIR gflags/gflags.h)
FIND_LIBRARY(LIBGFLAGS_LIBRARY_DEBUG NAMES gflagsd gflags_staticd)
FIND_LIBRARY(LIBGFLAGS_LIBRARY_RELEASE NAMES gflags gflags_static)
INCLUDE(SelectLibraryConfigurations)
SELECT_LIBRARY_CONFIGURATIONS(LIBGFLAGS)
# handle the QUIETLY and REQUIRED arguments and set LIBGFLAGS_FOUND to TRUE if
# all listed variables are TRUE
INCLUDE(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(gflags DEFAULT_MSG LIBGFLAGS_LIBRARY LIBGFLAGS_INCLUDE_DIR)
# cmake module compat
set(Gflags_FOUND ${GFLAGS_FOUND})
# compat with some existing FindGflags consumers
set(LIBGFLAGS_FOUND ${GFLAGS_FOUND})
# Compat with the gflags CONFIG based detection
set(gflags_FOUND ${GFLAGS_FOUND})
set(gflags_INCLUDE_DIR ${LIBGFLAGS_INCLUDE_DIR})
set(gflags_LIBRARIES ${LIBGFLAGS_LIBRARY})
set(GFLAGS_TARGET ${LIBGFLAGS_LIBRARY})
set(gflags_TARGET ${LIBGFLAGS_LIBRARY})
MARK_AS_ADVANCED(LIBGFLAGS_LIBRARY LIBGFLAGS_INCLUDE_DIR)
endif()
# Compat with the gflags CONFIG based detection
if (LIBGFLAGS_FOUND AND NOT TARGET gflags)
add_library(gflags UNKNOWN IMPORTED)
set_target_properties(gflags PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LIBGFLAGS_INCLUDE_DIR}")
set_target_properties(gflags PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" IMPORTED_LOCATION "${LIBGFLAGS_LIBRARY}")
endif()

32
build/fbcode_builder/CMake/FindGlog.cmake Normal file
View File

@ -0,0 +1,32 @@
# Copyright (c) Facebook, Inc. and its affiliates.
# - Try to find Glog
# Once done, this will define
#
# GLOG_FOUND - system has Glog
# GLOG_INCLUDE_DIRS - the Glog include directories
# GLOG_LIBRARIES - link these to use Glog
include(FindPackageHandleStandardArgs)
find_library(GLOG_LIBRARY glog
PATHS ${GLOG_LIBRARYDIR})
find_path(GLOG_INCLUDE_DIR glog/logging.h
PATHS ${GLOG_INCLUDEDIR})
find_package_handle_standard_args(glog DEFAULT_MSG
GLOG_LIBRARY
GLOG_INCLUDE_DIR)
mark_as_advanced(
GLOG_LIBRARY
GLOG_INCLUDE_DIR)
set(GLOG_LIBRARIES ${GLOG_LIBRARY})
set(GLOG_INCLUDE_DIRS ${GLOG_INCLUDE_DIR})
if (NOT TARGET glog::glog)
add_library(glog::glog UNKNOWN IMPORTED)
set_target_properties(glog::glog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${GLOG_INCLUDE_DIRS}")
set_target_properties(glog::glog PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" IMPORTED_LOCATION "${GLOG_LIBRARIES}")
endif()

77
build/fbcode_builder/CMake/FindLibEvent.cmake Normal file
View File

@ -0,0 +1,77 @@
# Copyright (c) Facebook, Inc. and its affiliates.
# - Find LibEvent (a cross event library)
# This module defines
# LIBEVENT_INCLUDE_DIR, where to find LibEvent headers
# LIBEVENT_LIB, LibEvent libraries
# LibEvent_FOUND, If false, do not try to use libevent
set(LibEvent_EXTRA_PREFIXES /usr/local /opt/local "$ENV{HOME}")
foreach(prefix ${LibEvent_EXTRA_PREFIXES})
list(APPEND LibEvent_INCLUDE_PATHS "${prefix}/include")
list(APPEND LibEvent_LIB_PATHS "${prefix}/lib")
endforeach()
find_package(Libevent CONFIG QUIET)
if (TARGET event)
# Re-export the config under our own names
# Somewhat gross, but some vcpkg installed libevents have a relative
# `include` path exported into LIBEVENT_INCLUDE_DIRS, which triggers
# a cmake error because it resolves to the `include` dir within the
# folly repo, which is not something cmake allows to be in the
# INTERFACE_INCLUDE_DIRECTORIES. Thankfully on such a system the
# actual include directory is already part of the global include
# directories, so we can just skip it.
if (NOT "${LIBEVENT_INCLUDE_DIRS}" STREQUAL "include")
set(LIBEVENT_INCLUDE_DIR ${LIBEVENT_INCLUDE_DIRS})
else()
set(LIBEVENT_INCLUDE_DIR)
endif()
# Unfortunately, with a bare target name `event`, downstream consumers
# of the package that depends on `Libevent` located via CONFIG end
# up exporting just a bare `event` in their libraries. This is problematic
# because this in interpreted as just `-levent` with no library path.
# When libevent is not installed in the default installation prefix
# this results in linker errors.
# To resolve this, we ask cmake to lookup the full path to the library
# and use that instead.
cmake_policy(PUSH)
if(POLICY CMP0026)
# Allow reading the LOCATION property
cmake_policy(SET CMP0026 OLD)
endif()
get_target_property(LIBEVENT_LIB event LOCATION)
cmake_policy(POP)
set(LibEvent_FOUND ${Libevent_FOUND})
if (NOT LibEvent_FIND_QUIETLY)
message(STATUS "Found libevent from package config include=${LIBEVENT_INCLUDE_DIRS} lib=${LIBEVENT_LIB}")
endif()
else()
find_path(LIBEVENT_INCLUDE_DIR event.h PATHS ${LibEvent_INCLUDE_PATHS})
find_library(LIBEVENT_LIB NAMES event PATHS ${LibEvent_LIB_PATHS})
if (LIBEVENT_LIB AND LIBEVENT_INCLUDE_DIR)
set(LibEvent_FOUND TRUE)
set(LIBEVENT_LIB ${LIBEVENT_LIB})
else ()
set(LibEvent_FOUND FALSE)
endif ()
if (LibEvent_FOUND)
if (NOT LibEvent_FIND_QUIETLY)
message(STATUS "Found libevent: ${LIBEVENT_LIB}")
endif ()
else ()
if (LibEvent_FIND_REQUIRED)
message(FATAL_ERROR "Could NOT find libevent.")
endif ()
message(STATUS "libevent NOT found.")
endif ()
mark_as_advanced(
LIBEVENT_LIB
LIBEVENT_INCLUDE_DIR
)
endif()

11
build/fbcode_builder/CMake/FindPCRE.cmake Normal file
View File

@ -0,0 +1,11 @@
# Copyright (c) Facebook, Inc. and its affiliates.
include(FindPackageHandleStandardArgs)
find_path(PCRE_INCLUDE_DIR NAMES pcre.h)
find_library(PCRE_LIBRARY NAMES pcre)
find_package_handle_standard_args(
PCRE
DEFAULT_MSG
PCRE_LIBRARY
PCRE_INCLUDE_DIR
)
mark_as_advanced(PCRE_INCLUDE_DIR PCRE_LIBRARY)

120
build/fbcode_builder/CMake/ThriftCppLibrary.cmake Normal file
View File

@ -0,0 +1,120 @@
# Copyright (c) Facebook, Inc. and its affiliates.
# NOTE: If you change this file, fbcode/fboss/github/ThriftCppLibrary.cmake also
# needs to be changed. TODO: this should be handled via shipit.
function(add_thrift_cpp2_library LIB_NAME THRIFT_FILE)
# Parse the arguments
set(SERVICES)
set(DEPENDS)
set(GEN_ARGS)
set(mode "UNSET")
foreach(arg IN LISTS ARGN)
if("${arg}" STREQUAL "SERVICES")
set(mode "SERVICES")
elseif("${arg}" STREQUAL "DEPENDS")
set(mode "DEPENDS")
elseif("${arg}" STREQUAL "OPTIONS")
set(mode "OPTIONS")
else()
if("${mode}" STREQUAL "SERVICES")
list(APPEND SERVICES "${arg}")
elseif("${mode}" STREQUAL "DEPENDS")
list(APPEND DEPENDS "${arg}")
elseif("${mode}" STREQUAL "OPTIONS")
list(APPEND GEN_ARGS "${arg}")
else()
message(
FATAL_ERROR
"expected SERVICES, DEPENDS, or OPTIONS argument, found ${arg}"
)
endif()
endif()
endforeach()
get_filename_component(base ${THRIFT_FILE} NAME_WE)
get_filename_component(
output_dir
${CMAKE_CURRENT_BINARY_DIR}/${THRIFT_FILE}
DIRECTORY
)
list(APPEND GEN_ARGS "include_prefix=${output_dir}")
# CMake 3.12 is finally getting a list(JOIN) function, but until then
# treating the list as a string and replacing the semicolons is good enough.
string(REPLACE ";" "," GEN_ARG_STR "${GEN_ARGS}")
# Compute the list of generated files
list(APPEND generated_headers
${output_dir}/gen-cpp2/${base}_constants.h
${output_dir}/gen-cpp2/${base}_constants.cpp
${output_dir}/gen-cpp2/${base}_types.h
${output_dir}/gen-cpp2/${base}_types.tcc
${output_dir}/gen-cpp2/${base}_types_custom_protocol.h
)
list(APPEND generated_sources
${output_dir}/gen-cpp2/${base}_data.h
${output_dir}/gen-cpp2/${base}_data.cpp
${output_dir}/gen-cpp2/${base}_types.cpp
)
foreach(service IN LISTS SERVICES)
list(APPEND generated_headers
${output_dir}/gen-cpp2/${service}.h
${output_dir}/gen-cpp2/${service}.tcc
${output_dir}/gen-cpp2/${service}AsyncClient.h
${output_dir}/gen-cpp2/${service}_custom_protocol.h
)
list(APPEND generated_sources
${output_dir}/gen-cpp2/${service}.cpp
${output_dir}/gen-cpp2/${service}AsyncClient.cpp
${output_dir}/gen-cpp2/${service}_processmap_binary.cpp
${output_dir}/gen-cpp2/${service}_processmap_compact.cpp
)
endforeach()
# Emit the rule to run the thrift compiler
add_custom_command(
OUTPUT
${generated_headers}
${generated_sources}
COMMAND
${CMAKE_COMMAND} -E make_directory ${output_dir}
COMMAND
${FBTHRIFT_COMPILER}
--strict
--templates ${FBTHRIFT_TEMPLATES_DIR}
--gen "mstch_cpp2:${GEN_ARG_STR}"
-I ${CMAKE_SOURCE_DIR}
-o ${output_dir}
${CMAKE_CURRENT_SOURCE_DIR}/${THRIFT_FILE}
WORKING_DIRECTORY
${CMAKE_BINARY_DIR}
MAIN_DEPENDENCY
${THRIFT_FILE}
DEPENDS
${DEPENDS}
)
# Now emit the library rule to compile the sources
add_library(${LIB_NAME} STATIC
${generated_sources}
)
set_property(
TARGET ${LIB_NAME}
PROPERTY PUBLIC_HEADER
${generated_headers}
)
target_include_directories(
${LIB_NAME}
PUBLIC
${CMAKE_SOURCE_DIR}
${CMAKE_BINARY_DIR}
${FOLLY_INCLUDE_DIR}
${FBTHRIFT_INCLUDE_DIR}
)
target_link_libraries(
${LIB_NAME}
PUBLIC
${DEPENDS}
FBThrift::thriftcpp2
Folly::folly
)
endfunction()

44
build/fbcode_builder/README.docker Normal file
View File

@ -0,0 +1,44 @@
## Debugging Docker builds
To debug a a build failure, start up a shell inside the just-failed image as
follows:
```
docker ps -a | head # Grab the container ID
docker commit CONTAINER_ID # Grab the SHA string
docker run -it SHA_STRING /bin/bash
# Debug as usual, e.g. `./run-cmake.sh Debug`, `make`, `apt-get install gdb`
```
## A note on Docker security
While the Dockerfile generated above is quite simple, you must be aware that
using Docker to run arbitrary code can present significant security risks:
- Code signature validation is off by default (as of 2016), exposing you to
man-in-the-middle malicious code injection.
- You implicitly trust the world -- a Dockerfile cannot annotate that
you trust the image `debian:8.6` because you trust a particular
certificate -- rather, you trust the name, and that it will never be
hijacked.
- Sandboxing in the Linux kernel is not perfect, and the builds run code as
root. Any compromised code can likely escalate to the host system.
Specifically, you must be very careful only to add trusted OS images to the
build flow.
Consider setting this variable before running any Docker container -- this
will validate a signature on the base image before running code from it:
```
export DOCKER_CONTENT_TRUST=1
```
Note that unless you go through the extra steps of notarizing the resulting
images, you will have to disable trust to enter intermediate images, e.g.
```
DOCKER_CONTENT_TRUST= docker run -it YOUR_IMAGE_ID /bin/bash
```

60
build/fbcode_builder/README.md Normal file
View File

@ -0,0 +1,60 @@
# Easy builds for Facebook projects
This is a Python 2.6+ library designed to simplify continuous-integration
(and other builds) of Facebook projects.
For external Travis builds, the entry point is `travis_docker_build.sh`.
## Using Docker to reproduce a CI build
If you are debugging or enhancing a CI build, you will want to do so from
host or virtual machine that can run a reasonably modern version of Docker:
``` sh
./make_docker_context.py --help # See available options for OS & compiler
# Tiny wrapper that starts a Travis-like build with compile caching:
os_image=ubuntu:16.04 \
gcc_version=5 \
make_parallelism=2 \
travis_cache_dir=~/travis_ccache \
./travis_docker_build.sh &> build_at_$(date +'%Y%m%d_%H%M%S').log
```
**IMPORTANT**: Read `fbcode_builder/README.docker` before diving in!
Setting `travis_cache_dir` turns on [ccache](https://ccache.samba.org/),
saving a fresh copy of `ccache.tgz` after every build. This will invalidate
Docker's layer cache, foring it to rebuild starting right after OS package
setup, but the builds will be fast because all the compiles will be cached.
To iterate without invalidating the Docker layer cache, just `cd
/tmp/docker-context-*` and interact with the `Dockerfile` normally. Note
that the `docker-context-*` dirs preserve a copy of `ccache.tgz` as they
first used it.
# What to read next
The *.py files are fairly well-documented. You might want to peruse them
in this order:
- shell_quoting.py
- fbcode_builder.py
- docker_builder.py
- make_docker_context.py
As far as runs on Travis go, the control flow is:
- .travis.yml calls
- travis_docker_build.sh calls
- docker_build_with_ccache.sh
This library also has an (unpublished) component targeting Facebook's
internal continuous-integration platform using the same build-step DSL.
# Contributing
Please follow the ambient style (or PEP-8), and keep the code Python 2.6
compatible -- since `fbcode_builder`'s only dependency is Docker, we want to
allow building projects on even fairly ancient base systems. We also wish
to be compatible with Python 3, and would appreciate it if you kept that
in mind while making changes also.

219
build/fbcode_builder/docker_build_with_ccache.sh Executable file
View File

@ -0,0 +1,219 @@
#!/bin/bash -uex
# Copyright (c) Facebook, Inc. and its affiliates.
set -o pipefail # Be sure to `|| :` commands that are allowed to fail.
#
# Future: port this to Python if you are making significant changes.
#
# Parse command-line arguments
build_timeout="" # Default to no time-out
print_usage() {
echo "Usage: $0 [--build-timeout TIMEOUT_VAL] SAVE-CCACHE-TO-DIR"
echo "SAVE-CCACHE-TO-DIR is required. An empty string discards the ccache."
}
while [[ $# -gt 0 ]]; do
case "$1" in
--build-timeout)
shift
build_timeout="$1"
if [[ "$build_timeout" != "" ]] ; then
timeout "$build_timeout" true # fail early on invalid timeouts
fi
;;
-h|--help)
print_usage
exit
;;
*)
break
;;
esac
shift
done
# There is one required argument, but an empty string is allowed.
if [[ "$#" != 1 ]] ; then
print_usage
exit 1
fi
save_ccache_to_dir="$1"
if [[ "$save_ccache_to_dir" != "" ]] ; then
mkdir -p "$save_ccache_to_dir" # fail early if there's nowhere to save
else
echo "WARNING: Will not save /ccache from inside the Docker container"
fi
rand_guid() {
echo "$(date +%s)_${RANDOM}_${RANDOM}_${RANDOM}_${RANDOM}"
}
id=fbcode_builder_image_id=$(rand_guid)
logfile=$(mktemp)
echo "
Running build with timeout '$build_timeout', label $id, and log in $logfile
"
if [[ "$build_timeout" != "" ]] ; then
# Kill the container after $build_timeout. Using `/bin/timeout` would cause
# Docker to destroy the most recent container and lose its cache.
(
sleep "$build_timeout"
echo "Build timed out after $build_timeout" 1>&2
while true; do
maybe_container=$(
grep -E '^( ---> Running in [0-9a-f]+|FBCODE_BUILDER_EXIT)$' "$logfile" |
tail -n 1 | awk '{print $NF}'
)
if [[ "$maybe_container" == "FBCODE_BUILDER_EXIT" ]] ; then
echo "Time-out successfully terminated build" 1>&2
break
fi
echo "Time-out: trying to kill $maybe_container" 1>&2
# This kill fail if we get unlucky, try again soon.
docker kill "$maybe_container" || sleep 5
done
) &
fi
build_exit_code=0
# `docker build` is allowed to fail, and `pipefail` means we must check the
# failure explicitly.
if ! docker build --label="$id" . 2>&1 | tee "$logfile" ; then
build_exit_code="${PIPESTATUS[0]}"
# NB: We are going to deliberately forge ahead even if `tee` failed.
# If it did, we have a problem with tempfile creation, and all is sad.
echo "Build failed with code $build_exit_code, trying to save ccache" 1>&2
fi
# Stop trying to kill the container.
echo $'\nFBCODE_BUILDER_EXIT' >> "$logfile"
if [[ "$save_ccache_to_dir" == "" ]] ; then
echo "Not inspecting Docker build, since saving the ccache wasn't requested."
exit "$build_exit_code"
fi
img=$(docker images --filter "label=$id" -a -q)
if [[ "$img" == "" ]] ; then
docker images -a
echo "In the above list, failed to find most recent image with $id" 1>&2
# Usually, the above `docker kill` will leave us with an up-to-the-second
# container, from which we can extract the cache. However, if that fails
# for any reason, this loop will instead grab the latest available image.
#
# It's possible for this log search to get confused due to the output of
# the build command itself, but since our builds aren't **trying** to
# break cache, we probably won't randomly hit an ID from another build.
img=$(
grep -E '^ ---> (Running in [0-9a-f]+|[0-9a-f]+)$' "$logfile" | tac |
sed 's/Running in /container_/;s/ ---> //;' | (
while read -r x ; do
# Both docker commands below print an image ID to stdout on
# success, so we just need to know when to stop.
if [[ "$x" =~ container_.* ]] ; then
if docker commit "${x#container_}" ; then
break
fi
elif docker inspect --type image -f '{{.Id}}' "$x" ; then
break
fi
done
)
)
if [[ "$img" == "" ]] ; then
echo "Failed to find valid container or image ID in log $logfile" 1>&2
exit 1
fi
elif [[ "$(echo "$img" | wc -l)" != 1 ]] ; then
# Shouldn't really happen, but be explicit if it does.
echo "Multiple images with label $id, taking the latest of:"
echo "$img"
img=$(echo "$img" | head -n 1)
fi
container_name="fbcode_builder_container_$(rand_guid)"
echo "Starting $container_name from latest image of the build with $id --"
echo "$img"
# ccache collection must be done outside of the Docker build steps because
# we need to be able to kill it on timeout.
#
# This step grows the max cache size to slightly exceed than the working set
# of a successful build. This simple design persists the max size in the
# cache directory itself (the env var CCACHE_MAXSIZE does not even work with
# older ccaches like the one on 14.04).
#
# Future: copy this script into the Docker image via Dockerfile.
(
# By default, fbcode_builder creates an unsigned image, so the `docker
# run` below would fail if DOCKER_CONTENT_TRUST were set. So we unset it
# just for this one run.
export DOCKER_CONTENT_TRUST=
# CAUTION: The inner bash runs without -uex, so code accordingly.
docker run --user root --name "$container_name" "$img" /bin/bash -c '
build_exit_code='"$build_exit_code"'
# Might be useful if debugging whether max cache size is too small?
grep " Cleaning up cache directory " /tmp/ccache.log
export CCACHE_DIR=/ccache
ccache -s
echo "Total bytes in /ccache:";
total_bytes=$(du -sb /ccache | awk "{print \$1}")
echo "$total_bytes"
echo "Used bytes in /ccache:";
used_bytes=$(
du -sb $(find /ccache -type f -newermt @$(
cat /FBCODE_BUILDER_CCACHE_START_TIME
)) | awk "{t += \$1} END {print t}"
)
echo "$used_bytes"
# Goal: set the max cache to 750MB over 125% of the usage of a
# successful build. If this is too small, it takes too long to get a
# cache fully warmed up. Plus, ccache cleans 100-200MB before reaching
# the max cache size, so a large margin is essential to prevent misses.
desired_mb=$(( 750 + used_bytes / 800000 )) # 125% in decimal MB: 1e6/1.25
if [[ "$build_exit_code" != "0" ]] ; then
# For a bad build, disallow shrinking the max cache size. Instead of
# the max cache size, we use on-disk size, which ccache keeps at least
# 150MB under the actual max size, hence the 400MB safety margin.
cur_max_mb=$(( 400 + total_bytes / 1000000 )) # ccache uses decimal MB
if [[ "$desired_mb" -le "$cur_max_mb" ]] ; then
desired_mb=""
fi
fi
if [[ "$desired_mb" != "" ]] ; then
echo "Updating cache size to $desired_mb MB"
ccache -M "${desired_mb}M"
ccache -s
fi
# Subshell because `time` the binary may not be installed.
if (time tar czf /ccache.tgz /ccache) ; then
ls -l /ccache.tgz
else
# This `else` ensures we never overwrite the current cache with
# partial data in case of error, even if somebody adds code below.
rm /ccache.tgz
exit 1
fi
'
)
echo "Updating $save_ccache_to_dir/ccache.tgz"
# This will not delete the existing cache if `docker run` didn't make one
docker cp "$container_name:/ccache.tgz" "$save_ccache_to_dir/"
# Future: it'd be nice if Travis allowed us to retry if the build timed out,
# since we'll make more progress thanks to the cache. As-is, we have to
# wait for the next commit to land.
echo "Build exited with code $build_exit_code"
exit "$build_exit_code"

182
build/fbcode_builder/docker_builder.py Normal file
View File

@ -0,0 +1,182 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'''
Extends FBCodeBuilder to produce Docker context directories.
In order to get the largest iteration-time savings from Docker's build
caching, you will want to:
- Use fine-grained steps as appropriate (e.g. separate make & make install),
- Start your action sequence with the lowest-risk steps, and with the steps
that change the least often, and
- Put the steps that you are debugging towards the very end.
'''
import logging
import os
import shutil
import tempfile
from fbcode_builder import FBCodeBuilder
from shell_quoting import (
raw_shell, shell_comment, shell_join, ShellQuoted, path_join
)
from utils import recursively_flatten_list, run_command
class DockerFBCodeBuilder(FBCodeBuilder):
def _user(self):
return self.option('user', 'root')
def _change_user(self):
return ShellQuoted('USER {u}').format(u=self._user())
def setup(self):
# Please add RPM-based OSes here as appropriate.
#
# To allow exercising non-root installs -- we change users after the
# system packages are installed. TODO: For users not defined in the
# image, we should probably `useradd`.
return self.step('Setup', [
# Docker's FROM does not understand shell quoting.
ShellQuoted('FROM {}'.format(self.option('os_image'))),
# /bin/sh syntax is a pain
ShellQuoted('SHELL ["/bin/bash", "-c"]'),
] + self.install_debian_deps() + [self._change_user()]
+ [self.workdir(self.option('prefix')),
self.create_python_venv(),
self.python_venv()])
def python_venv(self):
# To both avoid calling venv activate on each RUN command AND to ensure
# it is present when the resulting container is run add to PATH
actions = []
if self.option("PYTHON_VENV", "OFF") == "ON":
actions = ShellQuoted('ENV PATH={p}:$PATH').format(
p=path_join(self.option('prefix'), "venv", "bin"))
return(actions)
def step(self, name, actions):
assert '\n' not in name, 'Name {0} would span > 1 line'.format(name)
b = ShellQuoted('')
return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b]
def run(self, shell_cmd):
return ShellQuoted('RUN {cmd}').format(cmd=shell_cmd)
def workdir(self, dir):
return [
# As late as Docker 1.12.5, this results in `build` being owned
# by root:root -- the explicit `mkdir` works around the bug:
# USER nobody
# WORKDIR build
ShellQuoted('USER root'),
ShellQuoted('RUN mkdir -p {d} && chown {u} {d}').format(
d=dir, u=self._user()
),
self._change_user(),
ShellQuoted('WORKDIR {dir}').format(dir=dir),
]
def comment(self, comment):
# This should not be a command since we don't want comment changes
# to invalidate the Docker build cache.
return shell_comment(comment)
def copy_local_repo(self, repo_dir, dest_name):
fd, archive_path = tempfile.mkstemp(
prefix='local_repo_{0}_'.format(dest_name),
suffix='.tgz',
dir=os.path.abspath(self.option('docker_context_dir')),
)
os.close(fd)
run_command('tar', 'czf', archive_path, '.', cwd=repo_dir)
return [
ShellQuoted('ADD {archive} {dest_name}').format(
archive=os.path.basename(archive_path), dest_name=dest_name
),
# Docker permissions make very little sense... see also workdir()
ShellQuoted('USER root'),
ShellQuoted('RUN chown -R {u} {d}').format(
d=dest_name, u=self._user()
),
self._change_user(),
]
def _render_impl(self, steps):
return raw_shell(shell_join('\n', recursively_flatten_list(steps)))
def debian_ccache_setup_steps(self):
source_ccache_tgz = self.option('ccache_tgz', '')
if not source_ccache_tgz:
logging.info('Docker ccache not enabled')
return []
dest_ccache_tgz = os.path.join(
self.option('docker_context_dir'), 'ccache.tgz'
)
try:
try:
os.link(source_ccache_tgz, dest_ccache_tgz)
except OSError:
logging.exception(
'Hard-linking {s} to {d} failed, falling back to copy'
.format(s=source_ccache_tgz, d=dest_ccache_tgz)
)
shutil.copyfile(source_ccache_tgz, dest_ccache_tgz)
except Exception:
logging.exception(
'Failed to copy or link {s} to {d}, aborting'
.format(s=source_ccache_tgz, d=dest_ccache_tgz)
)
raise
return [
# Separate layer so that in development we avoid re-downloads.
self.run(ShellQuoted('apt-get install -yq ccache')),
ShellQuoted('ADD ccache.tgz /'),
ShellQuoted(
# Set CCACHE_DIR before the `ccache` invocations below.
'ENV CCACHE_DIR=/ccache '
# No clang support for now, so it's easiest to hardcode gcc.
'CC="ccache gcc" CXX="ccache g++" '
# Always log for ease of debugging. For real FB projects,
# this log is several megabytes, so dumping it to stdout
# would likely exceed the Travis log limit of 4MB.
#
# On a local machine, `docker cp` will get you the data. To
# get the data out from Travis, I would compress and dump
# uuencoded bytes to the log -- for Bistro this was about
# 600kb or 8000 lines:
#
# apt-get install sharutils
# bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2
'CCACHE_LOGFILE=/tmp/ccache.log'
),
self.run(ShellQuoted(
# Future: Skipping this part made this Docker step instant,
# saving ~1min of build time. It's unclear if it is the
# chown or the du, but probably the chown -- since a large
# part of the cost is incurred at image save time.
#
# ccache.tgz may be empty, or may have the wrong
# permissions.
'mkdir -p /ccache && time chown -R nobody /ccache && '
'time du -sh /ccache && '
# Reset stats so `docker_build_with_ccache.sh` can print
# useful values at the end of the run.
'echo === Prev run stats === && ccache -s && ccache -z && '
# Record the current time to let travis_build.sh figure out
# the number of bytes in the cache that are actually used --
# this is crucial for tuning the maximum cache size.
'date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && '
# The build running as `nobody` should be able to write here
'chown nobody /tmp/ccache.log'
)),
]

13
build/fbcode_builder/docker_enable_ipv6.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/sh
# Copyright (c) Facebook, Inc. and its affiliates.
# `daemon.json` is normally missing, but let's log it in case that changes.
touch /etc/docker/daemon.json
service docker stop
echo '{"ipv6": true, "fixed-cidr-v6": "2001:db8:1::/64"}' > /etc/docker/daemon.json
service docker start
# Fail early if docker failed on start -- add `- sudo dockerd` to debug.
docker info
# Paranoia log: what if our config got overwritten?
cat /etc/docker/daemon.json

384
build/fbcode_builder/fbcode_builder.py Normal file
View File

@ -0,0 +1,384 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'''
This is a small DSL to describe builds of Facebook's open-source projects
that are published to Github from a single internal repo, including projects
that depend on folly, wangle, proxygen, fbthrift, etc.
This file defines the interface of the DSL, and common utilieis, but you
will have to instantiate a specific builder, with specific options, in
order to get work done -- see e.g. make_docker_context.py.
== Design notes ==
Goals:
- A simple declarative language for what needs to be checked out & built,
how, in what order.
- The same specification should work for external continuous integration
builds (e.g. Travis + Docker) and for internal VM-based continuous
integration builds.
- One should be able to build without root, and to install to a prefix.
Non-goals:
- General usefulness. The only point of this is to make it easier to build
and test Facebook's open-source services.
Ideas for the future -- these may not be very good :)
- Especially on Ubuntu 14.04 the current initial setup is inefficient:
we add PPAs after having installed a bunch of packages -- this prompts
reinstalls of large amounts of code. We also `apt-get update` a few
times.
- A "shell script" builder. Like DockerFBCodeBuilder, but outputs a
shell script that runs outside of a container. Or maybe even
synchronously executes the shell commands, `make`-style.
- A "Makefile" generator. That might make iterating on builds even quicker
than what you can currently get with Docker build caching.
- Generate a rebuild script that can be run e.g. inside the built Docker
container by tagging certain steps with list-inheriting Python objects:
* do change directories
* do NOT `git clone` -- if we want to update code this should be a
separate script that e.g. runs rebase on top of specific targets
across all the repos.
* do NOT install software (most / all setup can be skipped)
* do NOT `autoreconf` or `configure`
* do `make` and `cmake`
- If we get non-Debian OSes, part of ccache setup should be factored out.
'''
import os
import re
from shell_quoting import path_join, shell_join, ShellQuoted
def _read_project_github_hashes():
base_dir = 'deps/github_hashes/' # trailing slash used in regex below
for dirname, _, files in os.walk(base_dir):
for filename in files:
path = os.path.join(dirname, filename)
with open(path) as f:
m_proj = re.match('^' + base_dir + '(.*)-rev\.txt$', path)
if m_proj is None:
raise RuntimeError('Not a hash file? {0}'.format(path))
m_hash = re.match('^Subproject commit ([0-9a-f]+)\n$', f.read())
if m_hash is None:
raise RuntimeError('No hash in {0}'.format(path))
yield m_proj.group(1), m_hash.group(1)
class FBCodeBuilder(object):
def __init__(self, **kwargs):
self._options_do_not_access = kwargs # Use .option() instead.
# This raises upon detecting options that are specified but unused,
# because otherwise it is very easy to make a typo in option names.
self.options_used = set()
self._github_hashes = dict(_read_project_github_hashes())
def __repr__(self):
return '{0}({1})'.format(
self.__class__.__name__,
', '.join(
'{0}={1}'.format(k, repr(v))
for k, v in self._options_do_not_access.items()
)
)
def option(self, name, default=None):
value = self._options_do_not_access.get(name, default)
if value is None:
raise RuntimeError('Option {0} is required'.format(name))
self.options_used.add(name)
return value
def has_option(self, name):
return name in self._options_do_not_access
def add_option(self, name, value):
if name in self._options_do_not_access:
raise RuntimeError('Option {0} already set'.format(name))
self._options_do_not_access[name] = value
#
# Abstract parts common to every installation flow
#
def render(self, steps):
'''
Converts nested actions to your builder's expected output format.
Typically takes the output of build().
'''
res = self._render_impl(steps) # Implementation-dependent
# Now that the output is rendered, we expect all options to have
# been used.
unused_options = set(self._options_do_not_access)
unused_options -= self.options_used
if unused_options:
raise RuntimeError(
'Unused options: {0} -- please check if you made a typo '
'in any of them. Those that are truly not useful should '
'be not be set so that this typo detection can be useful.'
.format(unused_options)
)
return res
def build(self, steps):
if not steps:
raise RuntimeError('Please ensure that the config you are passing '
'contains steps')
return [self.setup(), self.diagnostics()] + steps
def setup(self):
'Your builder may want to install packages here.'
raise NotImplementedError
def diagnostics(self):
'Log some system diagnostics before/after setup for ease of debugging'
# The builder's repr is not used in a command to avoid pointlessly
# invalidating Docker's build cache.
return self.step('Diagnostics', [
self.comment('Builder {0}'.format(repr(self))),
self.run(ShellQuoted('hostname')),
self.run(ShellQuoted('cat /etc/issue || echo no /etc/issue')),
self.run(ShellQuoted('g++ --version || echo g++ not installed')),
self.run(ShellQuoted('cmake --version || echo cmake not installed')),
])
def step(self, name, actions):
'A labeled collection of actions or other steps'
raise NotImplementedError
def run(self, shell_cmd):
'Run this bash command'
raise NotImplementedError
def workdir(self, dir):
'Create this directory if it does not exist, and change into it'
raise NotImplementedError
def copy_local_repo(self, dir, dest_name):
'''
Copy the local repo at `dir` into this step's `workdir()`, analog of:
cp -r /path/to/folly folly
'''
raise NotImplementedError
def debian_deps(self):
return [
'autoconf-archive',
'bison',
'build-essential',
'cmake',
'curl',
'flex',
'git',
'gperf',
'joe',
'libboost-all-dev',
'libcap-dev',
'libdouble-conversion-dev',
'libevent-dev',
'libgflags-dev',
'libgoogle-glog-dev',
'libkrb5-dev',
'libpcre3-dev',
'libpthread-stubs0-dev',
'libnuma-dev',
'libsasl2-dev',
'libsnappy-dev',
'libsqlite3-dev',
'libssl-dev',
'libtool',
'netcat-openbsd',
'pkg-config',
'sudo',
'unzip',
'wget',
'python3-venv',
]
#
# Specific build helpers
#
def install_debian_deps(self):
actions = [
self.run(
ShellQuoted('apt-get update && apt-get install -yq {deps}').format(
deps=shell_join(' ', (
ShellQuoted(dep) for dep in self.debian_deps())))
),
]
gcc_version = self.option('gcc_version')
# Make the selected GCC the default before building anything
actions.extend([
self.run(ShellQuoted('apt-get install -yq {c} {cpp}').format(
c=ShellQuoted('gcc-{v}').format(v=gcc_version),
cpp=ShellQuoted('g++-{v}').format(v=gcc_version),
)),
self.run(ShellQuoted(
'update-alternatives --install /usr/bin/gcc gcc {c} 40 '
'--slave /usr/bin/g++ g++ {cpp}'
).format(
c=ShellQuoted('/usr/bin/gcc-{v}').format(v=gcc_version),
cpp=ShellQuoted('/usr/bin/g++-{v}').format(v=gcc_version),
)),
self.run(ShellQuoted('update-alternatives --config gcc')),
])
actions.extend(self.debian_ccache_setup_steps())
return self.step('Install packages for Debian-based OS', actions)
def create_python_venv(self):
action = []
if self.option("PYTHON_VENV", "OFF") == "ON":
action = self.run(ShellQuoted("python3 -m venv {p}").format(
p=path_join(self.option('prefix'), "venv")))
return(action)
def python_venv(self):
action = []
if self.option("PYTHON_VENV", "OFF") == "ON":
action = ShellQuoted("source {p}").format(
p=path_join(self.option('prefix'), "venv", "bin", "activate"))
return(action)
def debian_ccache_setup_steps(self):
return [] # It's ok to ship a renderer without ccache support.
def github_project_workdir(self, project, path):
# Only check out a non-default branch if requested. This especially
# makes sense when building from a local repo.
git_hash = self.option(
'{0}:git_hash'.format(project),
# Any repo that has a hash in deps/github_hashes defaults to
# that, with the goal of making builds maximally consistent.
self._github_hashes.get(project, '')
)
maybe_change_branch = [
self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
] if git_hash else []
base_dir = self.option('projects_dir')
local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
return self.step('Check out {0}, workdir {1}'.format(project, path), [
self.workdir(base_dir),
self.run(
ShellQuoted('git clone https://github.com/{p}').format(p=project)
) if not local_repo_dir else self.copy_local_repo(
local_repo_dir, os.path.basename(project)
),
self.workdir(path_join(base_dir, os.path.basename(project), path)),
] + maybe_change_branch)
def fb_github_project_workdir(self, project_and_path, github_org='facebook'):
'This helper lets Facebook-internal CI special-cases FB projects'
project, path = project_and_path.split('/', 1)
return self.github_project_workdir(github_org + '/' + project, path)
def _make_vars(self, make_vars):
return shell_join(' ', (
ShellQuoted('{k}={v}').format(k=k, v=v)
for k, v in ({} if make_vars is None else make_vars).items()
))
def parallel_make(self, make_vars=None):
return self.run(ShellQuoted('make -j {n} VERBOSE=1 {vars}').format(
n=self.option('make_parallelism'),
vars=self._make_vars(make_vars),
))
def make_and_install(self, make_vars=None):
return [
self.parallel_make(make_vars),
self.run(ShellQuoted('make install VERBOSE=1 {vars}').format(
vars=self._make_vars(make_vars),
)),
]
def configure(self, name=None):
autoconf_options = {}
if name is not None:
autoconf_options.update(
self.option('{0}:autoconf_options'.format(name), {})
)
return [
self.run(ShellQuoted(
'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" '
'CFLAGS="$CFLAGS -I"{p}"/include" '
'CPPFLAGS="$CPPFLAGS -I"{p}"/include" '
'PY_PREFIX={p} '
'./configure --prefix={p} {args}'
).format(
p=self.option('prefix'),
args=shell_join(' ', (
ShellQuoted('{k}={v}').format(k=k, v=v)
for k, v in autoconf_options.items()
)),
)),
]
def autoconf_install(self, name):
return self.step('Build and install {0}'.format(name), [
self.run(ShellQuoted('autoreconf -ivf')),
] + self.configure() + self.make_and_install())
def cmake_configure(self, name, cmake_path='..'):
cmake_defines = {
'BUILD_SHARED_LIBS': 'ON',
'CMAKE_INSTALL_PREFIX': self.option('prefix'),
}
cmake_defines.update(
self.option('{0}:cmake_defines'.format(name), {})
)
return [
self.run(ShellQuoted(
'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" '
'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" '
'cmake {args} {cmake_path}'
).format(
p=self.option('prefix'),
args=shell_join(' ', (
ShellQuoted('-D{k}={v}').format(k=k, v=v)
for k, v in cmake_defines.items()
)),
cmake_path=cmake_path,
)),
]
def cmake_install(self, name, cmake_path='..'):
return self.step(
'Build and install {0}'.format(name),
self.cmake_configure(name, cmake_path) + self.make_and_install()
)
def fb_github_autoconf_install(self, project_and_path, github_org='facebook'):
return [
self.fb_github_project_workdir(project_and_path, github_org),
self.autoconf_install(project_and_path),
]
def fb_github_cmake_install(self, project_and_path, cmake_path='..', github_org='facebook'):
return [
self.fb_github_project_workdir(project_and_path, github_org),
self.cmake_install(project_and_path, cmake_path),
]

15
build/fbcode_builder/fbcode_builder_config.py Normal file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'Demo config, so that `make_docker_context.py --help` works in this directory.'
config = {
'fbcode_builder_spec': lambda _builder: {
'depends_on': [],
'steps': [],
},
'github_project': 'demo/project',
}

175
build/fbcode_builder/make_docker_context.py Executable file
View File

@ -0,0 +1,175 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'''
Reads `fbcode_builder_config.py` from the current directory, and prepares a
Docker context directory to build this project. Prints to stdout the path
to the context directory.
Try `.../make_docker_context.py --help` from a project's `build/` directory.
By default, the Docker context directory will be in /tmp. It will always
contain a Dockerfile, and might also contain copies of your local repos, and
other data needed for the build container.
'''
import os
import tempfile
import textwrap
from docker_builder import DockerFBCodeBuilder
from parse_args import parse_args_to_fbcode_builder_opts
def make_docker_context(
get_steps_fn, github_project, opts=None, default_context_dir=None
):
'''
Returns a path to the Docker context directory. See parse_args.py.
Helper for making a command-line utility that writes your project's
Dockerfile and associated data into a (temporary) directory. Your main
program might look something like this:
print(make_docker_context(
lambda builder: [builder.step(...), ...],
'facebook/your_project',
))
'''
if opts is None:
opts = {}
valid_versions = (
('ubuntu:16.04', '5'),
)
def add_args(parser):
parser.add_argument(
'--docker-context-dir', metavar='DIR',
default=default_context_dir,
help='Write the Dockerfile and its context into this directory. '
'If empty, make a temporary directory. Default: %(default)s.',
)
parser.add_argument(
'--user', metavar='NAME', default=opts.get('user', 'nobody'),
help='Build and install as this user. Default: %(default)s.',
)
parser.add_argument(
'--prefix', metavar='DIR',
default=opts.get('prefix', '/home/install'),
help='Install all libraries in this prefix. Default: %(default)s.',
)
parser.add_argument(
'--projects-dir', metavar='DIR',
default=opts.get('projects_dir', '/home'),
help='Place project code directories here. Default: %(default)s.',
)
parser.add_argument(
'--os-image', metavar='IMG', choices=zip(*valid_versions)[0],
default=opts.get('os_image', valid_versions[0][0]),
help='Docker OS image -- be sure to use only ones you trust (See '
'README.docker). Choices: %(choices)s. Default: %(default)s.',
)
parser.add_argument(
'--gcc-version', metavar='VER',
choices=set(zip(*valid_versions)[1]),
default=opts.get('gcc_version', valid_versions[0][1]),
help='Choices: %(choices)s. Default: %(default)s.',
)
parser.add_argument(
'--make-parallelism', metavar='NUM', type=int,
default=opts.get('make_parallelism', 1),
help='Use `make -j` on multi-CPU systems with lots of RAM. '
'Default: %(default)s.',
)
parser.add_argument(
'--local-repo-dir', metavar='DIR',
help='If set, build {0} from a local directory instead of Github.'
.format(github_project),
)
parser.add_argument(
'--ccache-tgz', metavar='PATH',
help='If set, enable ccache for the build. To initialize the '
'cache, first try to hardlink, then to copy --cache-tgz '
'as ccache.tgz into the --docker-context-dir.'
)
opts = parse_args_to_fbcode_builder_opts(
add_args,
# These have add_argument() calls, others are set via --option.
(
'docker_context_dir',
'user',
'prefix',
'projects_dir',
'os_image',
'gcc_version',
'make_parallelism',
'local_repo_dir',
'ccache_tgz',
),
opts,
help=textwrap.dedent('''
Reads `fbcode_builder_config.py` from the current directory, and
prepares a Docker context directory to build {github_project} and
its dependencies. Prints to stdout the path to the context
directory.
Pass --option {github_project}:git_hash SHA1 to build something
other than the master branch from Github.
Or, pass --option {github_project}:local_repo_dir LOCAL_PATH to
build from a local repo instead of cloning from Github.
Usage:
(cd $(./make_docker_context.py) && docker build . 2>&1 | tee log)
'''.format(github_project=github_project)),
)
# This allows travis_docker_build.sh not to know the main Github project.
local_repo_dir = opts.pop('local_repo_dir', None)
if local_repo_dir is not None:
opts['{0}:local_repo_dir'.format(github_project)] = local_repo_dir
if (opts.get('os_image'), opts.get('gcc_version')) not in valid_versions:
raise Exception(
'Due to 4/5 ABI changes (std::string), we can only use {0}'.format(
' / '.join('GCC {1} on {0}'.format(*p) for p in valid_versions)
)
)
if opts.get('docker_context_dir') is None:
opts['docker_context_dir'] = tempfile.mkdtemp(prefix='docker-context-')
elif not os.path.exists(opts.get('docker_context_dir')):
os.makedirs(opts.get('docker_context_dir'))
builder = DockerFBCodeBuilder(**opts)
context_dir = builder.option('docker_context_dir') # Mark option "in-use"
# The renderer may also populate some files into the context_dir.
dockerfile = builder.render(get_steps_fn(builder))
with os.fdopen(os.open(
os.path.join(context_dir, 'Dockerfile'),
os.O_RDWR | os.O_CREAT | os.O_EXCL, # Do not overwrite existing files
0o644,
), 'w') as f:
f.write(dockerfile)
return context_dir
if __name__ == '__main__':
from utils import read_fbcode_builder_config, build_fbcode_builder_config
# Load a spec from the current directory
config = read_fbcode_builder_config('fbcode_builder_config.py')
print(make_docker_context(
build_fbcode_builder_config(config),
config['github_project'],
))

83
build/fbcode_builder/parse_args.py Normal file
View File

@ -0,0 +1,83 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'Argument parsing logic shared by all fbcode_builder CLI tools.'
import argparse
import logging
from shell_quoting import raw_shell, ShellQuoted
def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help):
'''
Provides some standard arguments: --debug, --option, --shell-quoted-option
Then, calls `add_args_fn(parser)` to add application-specific arguments.
`opts` are first used as defaults for the various command-line
arguments. Then, the parsed arguments are mapped back into `opts`,
which then become the values for `FBCodeBuilder.option()`, to be used
both by the builder and by `get_steps_fn()`.
`help` is printed in response to the `--help` argument.
'''
top_level_opts = set(top_level_opts)
parser = argparse.ArgumentParser(
description=help,
formatter_class=argparse.RawDescriptionHelpFormatter
)
add_args_fn(parser)
parser.add_argument(
'--option', nargs=2, metavar=('KEY', 'VALUE'), action='append',
default=[
(k, v) for k, v in opts.items()
if k not in top_level_opts and not isinstance(v, ShellQuoted)
],
help='Set project-specific options. These are assumed to be raw '
'strings, to be shell-escaped as needed. Default: %(default)s.',
)
parser.add_argument(
'--shell-quoted-option', nargs=2, metavar=('KEY', 'VALUE'),
action='append',
default=[
(k, raw_shell(v)) for k, v in opts.items()
if k not in top_level_opts and isinstance(v, ShellQuoted)
],
help='Set project-specific options. These are assumed to be shell-'
'quoted, and may be used in commands as-is. Default: %(default)s.',
)
parser.add_argument('--debug', action='store_true', help='Log more')
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.debug else logging.INFO,
format='%(levelname)s: %(message)s'
)
# Map command-line args back into opts.
logging.debug('opts before command-line arguments: {0}'.format(opts))
new_opts = {}
for key in top_level_opts:
val = getattr(args, key)
# Allow clients to unset a default by passing a value of None in opts
if val is not None:
new_opts[key] = val
for key, val in args.option:
new_opts[key] = val
for key, val in args.shell_quoted_option:
new_opts[key] = ShellQuoted(val)
logging.debug('opts after command-line arguments: {0}'.format(new_opts))
return new_opts

112
build/fbcode_builder/shell_builder.py Normal file
View File

@ -0,0 +1,112 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'''
shell_builder.py allows running the fbcode_builder logic
on the host rather than in a container.
It emits a bash script with set -exo pipefail configured such that
any failing step will cause the script to exit with failure.
== How to run it? ==
cd build
python fbcode_builder/shell_builder.py > ~/run.sh
bash ~/run.sh
'''
import os
import distutils.spawn
from fbcode_builder import FBCodeBuilder
from shell_quoting import (
raw_shell, shell_comment, shell_join, ShellQuoted
)
from utils import recursively_flatten_list
class ShellFBCodeBuilder(FBCodeBuilder):
def _render_impl(self, steps):
return raw_shell(shell_join('\n', recursively_flatten_list(steps)))
def workdir(self, dir):
return [
ShellQuoted('mkdir -p {d} && cd {d}').format(
d=dir
),
]
def run(self, shell_cmd):
return ShellQuoted('{cmd}').format(cmd=shell_cmd)
def step(self, name, actions):
assert '\n' not in name, 'Name {0} would span > 1 line'.format(name)
b = ShellQuoted('')
return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b]
def setup(self):
steps = [
ShellQuoted('set -exo pipefail'),
] + [self.create_python_venv(), self.python_venv()]
if self.has_option('ccache_dir'):
ccache_dir = self.option('ccache_dir')
steps += [
ShellQuoted(
# Set CCACHE_DIR before the `ccache` invocations below.
'export CCACHE_DIR={ccache_dir} '
'CC="ccache ${{CC:-gcc}}" CXX="ccache ${{CXX:-g++}}"'
).format(ccache_dir=ccache_dir)
]
return steps
def comment(self, comment):
return shell_comment(comment)
def copy_local_repo(self, dir, dest_name):
return [
ShellQuoted('cp -r {dir} {dest_name}').format(
dir=dir,
dest_name=dest_name
),
]
def find_project_root():
here = os.path.dirname(os.path.realpath(__file__))
maybe_root = os.path.dirname(os.path.dirname(here))
if os.path.isdir(os.path.join(maybe_root, '.git')):
return maybe_root
raise RuntimeError(
"I expected shell_builder.py to be in the "
"build/fbcode_builder subdir of a git repo")
def persistent_temp_dir(repo_root):
escaped = repo_root.replace('/', 'sZs').replace('\\', 'sZs').replace(':', '')
return os.path.join(os.path.expandvars("$HOME"), '.fbcode_builder-' + escaped)
if __name__ == '__main__':
from utils import read_fbcode_builder_config, build_fbcode_builder_config
repo_root = find_project_root()
temp = persistent_temp_dir(repo_root)
config = read_fbcode_builder_config('fbcode_builder_config.py')
builder = ShellFBCodeBuilder()
builder.add_option('projects_dir', temp)
if distutils.spawn.find_executable('ccache'):
builder.add_option('ccache_dir',
os.environ.get('CCACHE_DIR', os.path.join(temp, '.ccache')))
builder.add_option('prefix', os.path.join(temp, 'installed'))
builder.add_option('make_parallelism', 4)
builder.add_option(
'{project}:local_repo_dir'.format(project=config['github_project']),
repo_root)
make_steps = build_fbcode_builder_config(config)
steps = make_steps(builder)
print(builder.render(steps))

99
build/fbcode_builder/shell_quoting.py Normal file
View File

@ -0,0 +1,99 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'''
Almost every FBCodeBuilder string is ultimately passed to a shell. Escaping
too little or too much tends to be the most common error. The utilities in
this file give a systematic way of avoiding such bugs:
- When you write literal strings destined for the shell, use `ShellQuoted`.
- When these literal strings are parameterized, use `ShellQuoted.format`.
- Any parameters that are raw strings get `shell_quote`d automatically,
while any ShellQuoted parameters will be left intact.
- Use `path_join` to join path components.
- Use `shell_join` to join already-quoted command arguments or shell lines.
'''
import os
from collections import namedtuple
class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))):
'''
Wrap a string with this to make it transparent to shell_quote(). It
will almost always suffice to use ShellQuoted.format(), path_join(),
or shell_join().
If you really must, use raw_shell() to access the raw string.
'''
def __new__(cls, s):
'No need to nest ShellQuoted.'
return super(ShellQuoted, cls).__new__(
cls, s.do_not_use_raw_str if isinstance(s, ShellQuoted) else s
)
def __str__(self):
raise RuntimeError(
'One does not simply convert {0} to a string -- use path_join() '
'or ShellQuoted.format() instead'.format(repr(self))
)
def __repr__(self):
return '{0}({1})'.format(
self.__class__.__name__, repr(self.do_not_use_raw_str)
)
def format(self, **kwargs):
'''
Use instead of str.format() when the arguments are either
`ShellQuoted()` or raw strings needing to be `shell_quote()`d.
Positional args are deliberately not supported since they are more
error-prone.
'''
return ShellQuoted(self.do_not_use_raw_str.format(**dict(
(k, shell_quote(v).do_not_use_raw_str) for k, v in kwargs.items()
)))
def shell_quote(s):
'Quotes a string if it is not already quoted'
return s if isinstance(s, ShellQuoted) \
else ShellQuoted("'" + str(s).replace("'", "'\\''") + "'")
def raw_shell(s):
'Not a member of ShellQuoted so we get a useful error for raw strings'
if isinstance(s, ShellQuoted):
return s.do_not_use_raw_str
raise RuntimeError('{0} should have been ShellQuoted'.format(s))
def shell_join(delim, it):
'Joins an iterable of ShellQuoted with a delimiter between each two'
return ShellQuoted(delim.join(raw_shell(s) for s in it))
def path_join(*args):
'Joins ShellQuoted and raw pieces of paths to make a shell-quoted path'
return ShellQuoted(os.path.join(*[
raw_shell(shell_quote(s)) for s in args
]))
def shell_comment(c):
'Do not shell-escape raw strings in comments, but do handle line breaks.'
return ShellQuoted('# {c}').format(c=ShellQuoted(
(raw_shell(c) if isinstance(c, ShellQuoted) else c)
.replace('\n', '\n# ')
))

0
build/fbcode_builder/specs/__init__.py Normal file
View File

33
build/fbcode_builder/specs/fbthrift.py Normal file
View File

@ -0,0 +1,33 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import specs.folly as folly
import specs.fizz as fizz
import specs.fmt as fmt
import specs.rsocket as rsocket
import specs.sodium as sodium
import specs.wangle as wangle
import specs.zstd as zstd
from shell_quoting import ShellQuoted
def fbcode_builder_spec(builder):
# This API should change rarely, so build the latest tag instead of master.
builder.add_option(
'no1msd/mstch:git_hash',
ShellQuoted('$(git describe --abbrev=0 --tags)')
)
return {
'depends_on': [folly, fizz, fmt, sodium, rsocket, wangle, zstd],
'steps': [
# This isn't a separete spec, since only fbthrift uses mstch.
builder.github_project_workdir('no1msd/mstch', 'build'),
builder.cmake_install('no1msd/mstch'),
builder.fb_github_cmake_install('fbthrift/thrift'),
],
}

40
build/fbcode_builder/specs/fbzmq.py Normal file
View File

@ -0,0 +1,40 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import specs.fbthrift as fbthrift
import specs.folly as folly
import specs.gmock as gmock
import specs.sodium as sodium
import specs.sigar as sigar
from shell_quoting import ShellQuoted
def fbcode_builder_spec(builder):
builder.add_option('zeromq/libzmq:git_hash', 'v4.2.2')
return {
'depends_on': [folly, fbthrift, gmock, sodium, sigar],
'steps': [
builder.github_project_workdir('zeromq/libzmq', '.'),
builder.step('Build and install zeromq/libzmq', [
builder.run(ShellQuoted('./autogen.sh')),
builder.configure(),
builder.make_and_install(),
]),
builder.fb_github_project_workdir('fbzmq/fbzmq/build', 'facebook'),
builder.step('Build and install fbzmq/fbzmq/build', [
builder.cmake_configure('fbzmq/fbzmq/build'),
# we need the pythonpath to find the thrift compiler
builder.run(ShellQuoted(
'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages '
'make -j {n}'
).format(p=builder.option('prefix'), n=builder.option('make_parallelism'))),
builder.run(ShellQuoted('make install')),
]),
],
}

21
build/fbcode_builder/specs/fizz.py Normal file
View File

@ -0,0 +1,21 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import specs.folly as folly
import specs.sodium as sodium
def fbcode_builder_spec(builder):
return {
'depends_on': [folly, sodium],
'steps': [
builder.fb_github_cmake_install(
'fizz/fizz/build',
github_org='facebookincubator',
),
],
}

16
build/fbcode_builder/specs/fmt.py Normal file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
def fbcode_builder_spec(builder):
builder.add_option('fmtlib/fmt:git_hash', '5.3.0')
return {
'steps': [
builder.github_project_workdir('fmtlib/fmt', 'build'),
builder.cmake_install('fmtlib/fmt'),
],
}

20
build/fbcode_builder/specs/folly.py Normal file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
def fbcode_builder_spec(builder):
return {
'steps': [
# on macOS the filesystem is typically case insensitive.
# We need to ensure that the CWD is not the folly source
# dir when we build, otherwise the system will decide
# that `folly/String.h` is the file it wants when including
# `string.h` and the build will fail.
builder.fb_github_project_workdir('folly/_build'),
builder.cmake_install('facebook/folly'),
],
}

24
build/fbcode_builder/specs/gmock.py Normal file
View File

@ -0,0 +1,24 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
def fbcode_builder_spec(builder):
builder.add_option('google/googletest:git_hash', 'release-1.8.1')
builder.add_option(
'google/googletest:cmake_defines',
{
'BUILD_GTEST': 'ON',
# Avoid problems with MACOSX_RPATH
'BUILD_SHARED_LIBS': 'OFF',
}
)
return {
'steps': [
builder.github_project_workdir('google/googletest', 'build'),
builder.cmake_install('google/googletest'),
],
}

20
build/fbcode_builder/specs/proxygen.py Normal file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import specs.folly as folly
import specs.fizz as fizz
import specs.sodium as sodium
import specs.wangle as wangle
def fbcode_builder_spec(builder):
return {
'depends_on': [folly, wangle, fizz, sodium],
'steps': [
builder.fb_github_autoconf_install('proxygen/proxygen'),
],
}

15
build/fbcode_builder/specs/re2.py Normal file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
def fbcode_builder_spec(builder):
return {
'steps': [
builder.github_project_workdir('google/re2', 'build'),
builder.cmake_install('google/re2'),
],
}

18
build/fbcode_builder/specs/rocksdb.py Normal file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
def fbcode_builder_spec(builder):
builder.add_option("rocksdb/_build:cmake_defines", {
"USE_RTTI": "1",
"PORTABLE": "ON",
})
return {
"steps": [
builder.fb_github_cmake_install("rocksdb/_build"),
],
}

20
build/fbcode_builder/specs/rsocket.py Normal file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import specs.gmock as gmock
import specs.folly as folly
def fbcode_builder_spec(builder):
return {
'depends_on': [folly, gmock],
'steps': [
builder.fb_github_cmake_install(
'rsocket-cpp/rsocket',
github_org='rsocket'),
],
}

23
build/fbcode_builder/specs/sigar.py Normal file
View File

@ -0,0 +1,23 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from shell_quoting import ShellQuoted
def fbcode_builder_spec(builder):
builder.add_option(
'hyperic/sigar:autoconf_options', {'CFLAGS' : '-fgnu89-inline'})
return {
'steps': [
builder.github_project_workdir('hyperic/sigar', '.'),
builder.step('Build and install sigar', [
builder.run(ShellQuoted('./autogen.sh')),
builder.configure('hyperic/sigar'),
builder.make_and_install(),
]),
],
}

22
build/fbcode_builder/specs/sodium.py Normal file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from shell_quoting import ShellQuoted
def fbcode_builder_spec(builder):
builder.add_option('jedisct1/libsodium:git_hash', 'stable')
return {
'steps': [
builder.github_project_workdir('jedisct1/libsodium', '.'),
builder.step('Build and install jedisct1/libsodium', [
builder.run(ShellQuoted('./autogen.sh')),
builder.configure(),
builder.make_and_install(),
]),
],
}

21
build/fbcode_builder/specs/wangle.py Normal file
View File

@ -0,0 +1,21 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import specs.folly as folly
import specs.fizz as fizz
import specs.sodium as sodium
def fbcode_builder_spec(builder):
# Projects that simply depend on Wangle need not spend time on tests.
builder.add_option('wangle/wangle/build:cmake_defines', {'BUILD_TESTS': 'OFF'})
return {
'depends_on': [folly, fizz, sodium],
'steps': [
builder.fb_github_cmake_install('wangle/wangle/build'),
],
}

26
build/fbcode_builder/specs/zstd.py Normal file
View File

@ -0,0 +1,26 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from shell_quoting import ShellQuoted
def fbcode_builder_spec(builder):
# This API should change rarely, so build the latest tag instead of master.
builder.add_option(
'facebook/zstd:git_hash',
ShellQuoted('$(git describe --abbrev=0 --tags origin/master)')
)
return {
'steps': [
builder.github_project_workdir('facebook/zstd', '.'),
builder.step('Build and install zstd', [
builder.make_and_install(make_vars={
'PREFIX': builder.option('prefix'),
})
]),
],
}

42
build/fbcode_builder/travis_docker_build.sh Executable file
View File

@ -0,0 +1,42 @@
#!/bin/bash -uex
# Copyright (c) Facebook, Inc. and its affiliates.
# .travis.yml in the top-level dir explains why this is a separate script.
# Read the docs: ./make_docker_context.py --help
os_image=${os_image?Must be set by Travis}
gcc_version=${gcc_version?Must be set by Travis}
make_parallelism=${make_parallelism:-4}
# ccache is off unless requested
travis_cache_dir=${travis_cache_dir:-}
# The docker build never times out, unless specified
docker_build_timeout=${docker_build_timeout:-}
cur_dir="$(readlink -f "$(dirname "$0")")"
if [[ "$travis_cache_dir" == "" ]]; then
echo "ccache disabled, enable by setting env. var. travis_cache_dir"
ccache_tgz=""
elif [[ -e "$travis_cache_dir/ccache.tgz" ]]; then
ccache_tgz="$travis_cache_dir/ccache.tgz"
else
echo "$travis_cache_dir/ccache.tgz does not exist, starting with empty cache"
ccache_tgz=$(mktemp)
tar -T /dev/null -czf "$ccache_tgz"
fi
docker_context_dir=$(
cd "$cur_dir/.." # Let the script find our fbcode_builder_config.py
"$cur_dir/make_docker_context.py" \
--os-image "$os_image" \
--gcc-version "$gcc_version" \
--make-parallelism "$make_parallelism" \
--local-repo-dir "$cur_dir/../.." \
--ccache-tgz "$ccache_tgz"
)
cd "${docker_context_dir?Failed to make Docker context directory}"
# Make it safe to iterate on the .sh in the tree while the script runs.
cp "$cur_dir/docker_build_with_ccache.sh" .
exec ./docker_build_with_ccache.sh \
--build-timeout "$docker_build_timeout" \
"$travis_cache_dir"

96
build/fbcode_builder/utils.py Normal file
View File

@ -0,0 +1,96 @@
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
'Miscellaneous utility functions.'
import itertools
import logging
import os
import shutil
import subprocess
import sys
from contextlib import contextmanager
def recursively_flatten_list(l):
return itertools.chain.from_iterable(
(recursively_flatten_list(i) if type(i) is list else (i,))
for i in l
)
def run_command(*cmd, **kwargs):
'The stdout of most fbcode_builder utilities is meant to be parsed.'
logging.debug('Running: {0} with {1}'.format(cmd, kwargs))
kwargs['stdout'] = sys.stderr
subprocess.check_call(cmd, **kwargs)
@contextmanager
def make_temp_dir(d):
os.mkdir(d)
try:
yield d
finally:
shutil.rmtree(d, ignore_errors=True)
def _inner_read_config(path):
'''
Helper to read a named config file.
The grossness with the global is a workaround for this python bug:
https://bugs.python.org/issue21591
The bug prevents us from defining either a local function or a lambda
in the scope of read_fbcode_builder_config below.
'''
global _project_dir
full_path = os.path.join(_project_dir, path)
return read_fbcode_builder_config(full_path)
def read_fbcode_builder_config(filename):
# Allow one spec to read another
# When doing so, treat paths as relative to the config's project directory.
# _project_dir is a "local" for _inner_read_config; see the comments
# in that function for an explanation of the use of global.
global _project_dir
_project_dir = os.path.dirname(filename)
scope = {'read_fbcode_builder_config': _inner_read_config}
with open(filename) as config_file:
code = compile(config_file.read(), filename, mode='exec')
exec(code, scope)
return scope['config']
def steps_for_spec(builder, spec, processed_modules=None):
'''
Sets `builder` configuration, and returns all the builder steps
necessary to build `spec` and its dependencies.
Traverses the dependencies in depth-first order, honoring the sequencing
in each 'depends_on' list.
'''
if processed_modules is None:
processed_modules = set()
steps = []
for module in spec.get('depends_on', []):
if module not in processed_modules:
processed_modules.add(module)
steps.extend(steps_for_spec(
builder,
module.fbcode_builder_spec(builder),
processed_modules
))
steps.extend(spec.get('steps', []))
return steps
def build_fbcode_builder_config(config):
return lambda builder: builder.build(
steps_for_spec(builder, config['fbcode_builder_spec'](builder))
)