diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..66e05c24fe --- /dev/null +++ b/.travis.yml @@ -0,0 +1,45 @@ +# Facebook projects that use `fbcode_builder` for continuous integration +# share this Travis configuration to run builds via Docker. + +sudo: required + +# Docker disables IPv6 in containers by default. Enable it for unit tests that need [::1]. +before_script: + - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; + then + sudo build/fbcode_builder/docker_enable_ipv6.sh; + fi + +env: + global: + - travis_cache_dir=$HOME/travis_ccache + # Travis times out after 50 minutes. Very generously leave 10 minutes + # for setup (e.g. cache download, compression, and upload), so we never + # fail to cache the progress we made. + - docker_build_timeout=40m + +cache: + # Our build caches can be 200-300MB, so increase the timeout to 7 minutes + # to make sure we never fail to cache the progress we made. + timeout: 420 + directories: + - $HOME/travis_ccache # see docker_build_with_ccache.sh + +# Ugh, `services:` must be in the matrix, or we get `docker: command not found` +# https://github.com/travis-ci/travis-ci/issues/5142 +matrix: + include: + - env: ['os_image=ubuntu:16.04', gcc_version=5] + services: [docker] + +addons: + apt: + packages: python2.7 + +script: + # We don't want to write the script inline because of Travis kludginess -- + # it looks like it escapes " and \ in scripts when using `matrix:`. + - ./build/fbcode_builder/travis_docker_build.sh + +notifications: + webhooks: https://code.facebook.com/travis/webhook/ diff --git a/build/deps/github_hashes/facebook/fbthrift-rev.txt b/build/deps/github_hashes/facebook/fbthrift-rev.txt new file mode 100644 index 0000000000..a0f37d3cdc --- /dev/null +++ b/build/deps/github_hashes/facebook/fbthrift-rev.txt @@ -0,0 +1 @@ +Subproject commit 9ec0a9819c2fcab1a9d61ade27d0ca93d30f0aa1 diff --git a/build/deps/github_hashes/facebook/folly-rev.txt b/build/deps/github_hashes/facebook/folly-rev.txt new file mode 100644 index 0000000000..27defd1b4b --- /dev/null +++ b/build/deps/github_hashes/facebook/folly-rev.txt @@ -0,0 +1 @@ +Subproject commit d52f0f52c7e442db6c66254953d353796bcf1011 diff --git a/build/deps/github_hashes/facebook/wangle-rev.txt b/build/deps/github_hashes/facebook/wangle-rev.txt new file mode 100644 index 0000000000..2baaa7601e --- /dev/null +++ b/build/deps/github_hashes/facebook/wangle-rev.txt @@ -0,0 +1 @@ +Subproject commit 540095b482bbe3b8d00a93a1055e33c3847bbf22 diff --git a/build/deps/github_hashes/facebookincubator/fizz-rev.txt b/build/deps/github_hashes/facebookincubator/fizz-rev.txt new file mode 100644 index 0000000000..ae9fc4148d --- /dev/null +++ b/build/deps/github_hashes/facebookincubator/fizz-rev.txt @@ -0,0 +1 @@ +Subproject commit 7a70bc5af71d7ce1103ebdfaa4b3d9c20f69f183 diff --git a/build/fbcode_builder/.gitignore b/build/fbcode_builder/.gitignore new file mode 100644 index 0000000000..b98f3edfa6 --- /dev/null +++ b/build/fbcode_builder/.gitignore @@ -0,0 +1,5 @@ +# Facebook-internal CI builds don't have write permission outside of the +# source tree, so we install all projects into this directory. +/facebook_ci +__pycache__/ +*.pyc diff --git a/build/fbcode_builder/CMake/FindGMock.cmake b/build/fbcode_builder/CMake/FindGMock.cmake new file mode 100644 index 0000000000..cd042dd9c4 --- /dev/null +++ b/build/fbcode_builder/CMake/FindGMock.cmake @@ -0,0 +1,80 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Find libgmock +# +# LIBGMOCK_DEFINES - List of defines when using libgmock. +# LIBGMOCK_INCLUDE_DIR - where to find gmock/gmock.h, etc. +# LIBGMOCK_LIBRARIES - List of libraries when using libgmock. +# LIBGMOCK_FOUND - True if libgmock found. + +IF (LIBGMOCK_INCLUDE_DIR) + # Already in cache, be silent + SET(LIBGMOCK_FIND_QUIETLY TRUE) +ENDIF () + +find_package(GTest CONFIG QUIET) +if (TARGET GTest::gmock) + get_target_property(LIBGMOCK_DEFINES GTest::gtest INTERFACE_COMPILE_DEFINITIONS) + if (NOT ${LIBGMOCK_DEFINES}) + # Explicitly set to empty string if not found to avoid it being + # set to NOTFOUND and breaking compilation + set(LIBGMOCK_DEFINES "") + endif() + get_target_property(LIBGMOCK_INCLUDE_DIR GTest::gtest INTERFACE_INCLUDE_DIRECTORIES) + set(LIBGMOCK_LIBRARIES GTest::gmock_main GTest::gmock GTest::gtest) + set(LIBGMOCK_FOUND ON) + message(STATUS "Found gmock via config, defines=${LIBGMOCK_DEFINES}, include=${LIBGMOCK_INCLUDE_DIR}, libs=${LIBGMOCK_LIBRARIES}") +else() + + FIND_PATH(LIBGMOCK_INCLUDE_DIR gmock/gmock.h) + + FIND_LIBRARY(LIBGMOCK_MAIN_LIBRARY_DEBUG NAMES gmock_maind) + FIND_LIBRARY(LIBGMOCK_MAIN_LIBRARY_RELEASE NAMES gmock_main) + FIND_LIBRARY(LIBGMOCK_LIBRARY_DEBUG NAMES gmockd) + FIND_LIBRARY(LIBGMOCK_LIBRARY_RELEASE NAMES gmock) + FIND_LIBRARY(LIBGTEST_LIBRARY_DEBUG NAMES gtestd) + FIND_LIBRARY(LIBGTEST_LIBRARY_RELEASE NAMES gtest) + + find_package(Threads REQUIRED) + INCLUDE(SelectLibraryConfigurations) + SELECT_LIBRARY_CONFIGURATIONS(LIBGMOCK_MAIN) + SELECT_LIBRARY_CONFIGURATIONS(LIBGMOCK) + SELECT_LIBRARY_CONFIGURATIONS(LIBGTEST) + + set(LIBGMOCK_LIBRARIES + ${LIBGMOCK_MAIN_LIBRARY} + ${LIBGMOCK_LIBRARY} + ${LIBGTEST_LIBRARY} + Threads::Threads + ) + + if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + # The GTEST_LINKED_AS_SHARED_LIBRARY macro must be set properly on Windows. + # + # There isn't currently an easy way to determine if a library was compiled as + # a shared library on Windows, so just assume we've been built against a + # shared build of gmock for now. + SET(LIBGMOCK_DEFINES "GTEST_LINKED_AS_SHARED_LIBRARY=1" CACHE STRING "") + endif() + + # handle the QUIETLY and REQUIRED arguments and set LIBGMOCK_FOUND to TRUE if + # all listed variables are TRUE + INCLUDE(FindPackageHandleStandardArgs) + FIND_PACKAGE_HANDLE_STANDARD_ARGS( + GMock + DEFAULT_MSG + LIBGMOCK_MAIN_LIBRARY + LIBGMOCK_LIBRARY + LIBGTEST_LIBRARY + LIBGMOCK_LIBRARIES + LIBGMOCK_INCLUDE_DIR + ) + + MARK_AS_ADVANCED( + LIBGMOCK_DEFINES + LIBGMOCK_MAIN_LIBRARY + LIBGMOCK_LIBRARY + LIBGTEST_LIBRARY + LIBGMOCK_LIBRARIES + LIBGMOCK_INCLUDE_DIR + ) +endif() diff --git a/build/fbcode_builder/CMake/FindGflags.cmake b/build/fbcode_builder/CMake/FindGflags.cmake new file mode 100644 index 0000000000..246ceacdd0 --- /dev/null +++ b/build/fbcode_builder/CMake/FindGflags.cmake @@ -0,0 +1,81 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Find libgflags. +# There's a lot of compatibility cruft going on in here, both +# to deal with changes across the FB consumers of this and also +# to deal with variances in behavior of cmake itself. +# +# Since this file is named FindGflags.cmake the cmake convention +# is for the module to export both GFLAGS_FOUND and Gflags_FOUND. +# The convention expected by consumers is that we export the +# following variables, even though these do not match the cmake +# conventions: +# +# LIBGFLAGS_INCLUDE_DIR - where to find gflags/gflags.h, etc. +# LIBGFLAGS_LIBRARY - List of libraries when using libgflags. +# LIBGFLAGS_FOUND - True if libgflags found. +# +# We need to be able to locate gflags both from an installed +# cmake config file and just from the raw headers and libs, so +# test for the former and then the latter, and then stick +# the results together and export them into the variables +# listed above. +# +# For forwards compatibility, we export the following variables: +# +# gflags_INCLUDE_DIR - where to find gflags/gflags.h, etc. +# gflags_TARGET / GFLAGS_TARGET / gflags_LIBRARIES +# - List of libraries when using libgflags. +# gflags_FOUND - True if libgflags found. +# + +IF (LIBGFLAGS_INCLUDE_DIR) + # Already in cache, be silent + SET(Gflags_FIND_QUIETLY TRUE) +ENDIF () + +find_package(gflags CONFIG QUIET) +if (gflags_FOUND) + if (NOT Gflags_FIND_QUIETLY) + message(STATUS "Found gflags from package config ${gflags_CONFIG}") + endif() + # Re-export the config-specified libs with our local names + set(LIBGFLAGS_LIBRARY ${gflags_LIBRARIES}) + set(LIBGFLAGS_INCLUDE_DIR ${gflags_INCLUDE_DIR}) + set(LIBGFLAGS_FOUND ${gflags_FOUND}) + # cmake module compat + set(GFLAGS_FOUND ${gflags_FOUND}) + set(Gflags_FOUND ${gflags_FOUND}) +else() + FIND_PATH(LIBGFLAGS_INCLUDE_DIR gflags/gflags.h) + + FIND_LIBRARY(LIBGFLAGS_LIBRARY_DEBUG NAMES gflagsd gflags_staticd) + FIND_LIBRARY(LIBGFLAGS_LIBRARY_RELEASE NAMES gflags gflags_static) + + INCLUDE(SelectLibraryConfigurations) + SELECT_LIBRARY_CONFIGURATIONS(LIBGFLAGS) + + # handle the QUIETLY and REQUIRED arguments and set LIBGFLAGS_FOUND to TRUE if + # all listed variables are TRUE + INCLUDE(FindPackageHandleStandardArgs) + FIND_PACKAGE_HANDLE_STANDARD_ARGS(gflags DEFAULT_MSG LIBGFLAGS_LIBRARY LIBGFLAGS_INCLUDE_DIR) + # cmake module compat + set(Gflags_FOUND ${GFLAGS_FOUND}) + # compat with some existing FindGflags consumers + set(LIBGFLAGS_FOUND ${GFLAGS_FOUND}) + + # Compat with the gflags CONFIG based detection + set(gflags_FOUND ${GFLAGS_FOUND}) + set(gflags_INCLUDE_DIR ${LIBGFLAGS_INCLUDE_DIR}) + set(gflags_LIBRARIES ${LIBGFLAGS_LIBRARY}) + set(GFLAGS_TARGET ${LIBGFLAGS_LIBRARY}) + set(gflags_TARGET ${LIBGFLAGS_LIBRARY}) + + MARK_AS_ADVANCED(LIBGFLAGS_LIBRARY LIBGFLAGS_INCLUDE_DIR) +endif() + +# Compat with the gflags CONFIG based detection +if (LIBGFLAGS_FOUND AND NOT TARGET gflags) + add_library(gflags UNKNOWN IMPORTED) + set_target_properties(gflags PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LIBGFLAGS_INCLUDE_DIR}") + set_target_properties(gflags PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" IMPORTED_LOCATION "${LIBGFLAGS_LIBRARY}") +endif() diff --git a/build/fbcode_builder/CMake/FindGlog.cmake b/build/fbcode_builder/CMake/FindGlog.cmake new file mode 100644 index 0000000000..a589b2e37d --- /dev/null +++ b/build/fbcode_builder/CMake/FindGlog.cmake @@ -0,0 +1,32 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# - Try to find Glog +# Once done, this will define +# +# GLOG_FOUND - system has Glog +# GLOG_INCLUDE_DIRS - the Glog include directories +# GLOG_LIBRARIES - link these to use Glog + +include(FindPackageHandleStandardArgs) + +find_library(GLOG_LIBRARY glog + PATHS ${GLOG_LIBRARYDIR}) + +find_path(GLOG_INCLUDE_DIR glog/logging.h + PATHS ${GLOG_INCLUDEDIR}) + +find_package_handle_standard_args(glog DEFAULT_MSG + GLOG_LIBRARY + GLOG_INCLUDE_DIR) + +mark_as_advanced( + GLOG_LIBRARY + GLOG_INCLUDE_DIR) + +set(GLOG_LIBRARIES ${GLOG_LIBRARY}) +set(GLOG_INCLUDE_DIRS ${GLOG_INCLUDE_DIR}) + +if (NOT TARGET glog::glog) + add_library(glog::glog UNKNOWN IMPORTED) + set_target_properties(glog::glog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${GLOG_INCLUDE_DIRS}") + set_target_properties(glog::glog PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" IMPORTED_LOCATION "${GLOG_LIBRARIES}") +endif() diff --git a/build/fbcode_builder/CMake/FindLibEvent.cmake b/build/fbcode_builder/CMake/FindLibEvent.cmake new file mode 100644 index 0000000000..dd11ebd843 --- /dev/null +++ b/build/fbcode_builder/CMake/FindLibEvent.cmake @@ -0,0 +1,77 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# - Find LibEvent (a cross event library) +# This module defines +# LIBEVENT_INCLUDE_DIR, where to find LibEvent headers +# LIBEVENT_LIB, LibEvent libraries +# LibEvent_FOUND, If false, do not try to use libevent + +set(LibEvent_EXTRA_PREFIXES /usr/local /opt/local "$ENV{HOME}") +foreach(prefix ${LibEvent_EXTRA_PREFIXES}) + list(APPEND LibEvent_INCLUDE_PATHS "${prefix}/include") + list(APPEND LibEvent_LIB_PATHS "${prefix}/lib") +endforeach() + +find_package(Libevent CONFIG QUIET) +if (TARGET event) + # Re-export the config under our own names + + # Somewhat gross, but some vcpkg installed libevents have a relative + # `include` path exported into LIBEVENT_INCLUDE_DIRS, which triggers + # a cmake error because it resolves to the `include` dir within the + # folly repo, which is not something cmake allows to be in the + # INTERFACE_INCLUDE_DIRECTORIES. Thankfully on such a system the + # actual include directory is already part of the global include + # directories, so we can just skip it. + if (NOT "${LIBEVENT_INCLUDE_DIRS}" STREQUAL "include") + set(LIBEVENT_INCLUDE_DIR ${LIBEVENT_INCLUDE_DIRS}) + else() + set(LIBEVENT_INCLUDE_DIR) + endif() + + # Unfortunately, with a bare target name `event`, downstream consumers + # of the package that depends on `Libevent` located via CONFIG end + # up exporting just a bare `event` in their libraries. This is problematic + # because this in interpreted as just `-levent` with no library path. + # When libevent is not installed in the default installation prefix + # this results in linker errors. + # To resolve this, we ask cmake to lookup the full path to the library + # and use that instead. + cmake_policy(PUSH) + if(POLICY CMP0026) + # Allow reading the LOCATION property + cmake_policy(SET CMP0026 OLD) + endif() + get_target_property(LIBEVENT_LIB event LOCATION) + cmake_policy(POP) + + set(LibEvent_FOUND ${Libevent_FOUND}) + if (NOT LibEvent_FIND_QUIETLY) + message(STATUS "Found libevent from package config include=${LIBEVENT_INCLUDE_DIRS} lib=${LIBEVENT_LIB}") + endif() +else() + find_path(LIBEVENT_INCLUDE_DIR event.h PATHS ${LibEvent_INCLUDE_PATHS}) + find_library(LIBEVENT_LIB NAMES event PATHS ${LibEvent_LIB_PATHS}) + + if (LIBEVENT_LIB AND LIBEVENT_INCLUDE_DIR) + set(LibEvent_FOUND TRUE) + set(LIBEVENT_LIB ${LIBEVENT_LIB}) + else () + set(LibEvent_FOUND FALSE) + endif () + + if (LibEvent_FOUND) + if (NOT LibEvent_FIND_QUIETLY) + message(STATUS "Found libevent: ${LIBEVENT_LIB}") + endif () + else () + if (LibEvent_FIND_REQUIRED) + message(FATAL_ERROR "Could NOT find libevent.") + endif () + message(STATUS "libevent NOT found.") + endif () + + mark_as_advanced( + LIBEVENT_LIB + LIBEVENT_INCLUDE_DIR + ) +endif() diff --git a/build/fbcode_builder/CMake/FindPCRE.cmake b/build/fbcode_builder/CMake/FindPCRE.cmake new file mode 100644 index 0000000000..32ccb37253 --- /dev/null +++ b/build/fbcode_builder/CMake/FindPCRE.cmake @@ -0,0 +1,11 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +include(FindPackageHandleStandardArgs) +find_path(PCRE_INCLUDE_DIR NAMES pcre.h) +find_library(PCRE_LIBRARY NAMES pcre) +find_package_handle_standard_args( + PCRE + DEFAULT_MSG + PCRE_LIBRARY + PCRE_INCLUDE_DIR +) +mark_as_advanced(PCRE_INCLUDE_DIR PCRE_LIBRARY) diff --git a/build/fbcode_builder/CMake/ThriftCppLibrary.cmake b/build/fbcode_builder/CMake/ThriftCppLibrary.cmake new file mode 100644 index 0000000000..2613e888a4 --- /dev/null +++ b/build/fbcode_builder/CMake/ThriftCppLibrary.cmake @@ -0,0 +1,120 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# NOTE: If you change this file, fbcode/fboss/github/ThriftCppLibrary.cmake also +# needs to be changed. TODO: this should be handled via shipit. +function(add_thrift_cpp2_library LIB_NAME THRIFT_FILE) + # Parse the arguments + set(SERVICES) + set(DEPENDS) + set(GEN_ARGS) + set(mode "UNSET") + foreach(arg IN LISTS ARGN) + if("${arg}" STREQUAL "SERVICES") + set(mode "SERVICES") + elseif("${arg}" STREQUAL "DEPENDS") + set(mode "DEPENDS") + elseif("${arg}" STREQUAL "OPTIONS") + set(mode "OPTIONS") + else() + if("${mode}" STREQUAL "SERVICES") + list(APPEND SERVICES "${arg}") + elseif("${mode}" STREQUAL "DEPENDS") + list(APPEND DEPENDS "${arg}") + elseif("${mode}" STREQUAL "OPTIONS") + list(APPEND GEN_ARGS "${arg}") + else() + message( + FATAL_ERROR + "expected SERVICES, DEPENDS, or OPTIONS argument, found ${arg}" + ) + endif() + endif() + endforeach() + + get_filename_component(base ${THRIFT_FILE} NAME_WE) + get_filename_component( + output_dir + ${CMAKE_CURRENT_BINARY_DIR}/${THRIFT_FILE} + DIRECTORY + ) + + list(APPEND GEN_ARGS "include_prefix=${output_dir}") + # CMake 3.12 is finally getting a list(JOIN) function, but until then + # treating the list as a string and replacing the semicolons is good enough. + string(REPLACE ";" "," GEN_ARG_STR "${GEN_ARGS}") + + # Compute the list of generated files + list(APPEND generated_headers + ${output_dir}/gen-cpp2/${base}_constants.h + ${output_dir}/gen-cpp2/${base}_constants.cpp + ${output_dir}/gen-cpp2/${base}_types.h + ${output_dir}/gen-cpp2/${base}_types.tcc + ${output_dir}/gen-cpp2/${base}_types_custom_protocol.h + ) + list(APPEND generated_sources + ${output_dir}/gen-cpp2/${base}_data.h + ${output_dir}/gen-cpp2/${base}_data.cpp + ${output_dir}/gen-cpp2/${base}_types.cpp + ) + foreach(service IN LISTS SERVICES) + list(APPEND generated_headers + ${output_dir}/gen-cpp2/${service}.h + ${output_dir}/gen-cpp2/${service}.tcc + ${output_dir}/gen-cpp2/${service}AsyncClient.h + ${output_dir}/gen-cpp2/${service}_custom_protocol.h + ) + list(APPEND generated_sources + ${output_dir}/gen-cpp2/${service}.cpp + ${output_dir}/gen-cpp2/${service}AsyncClient.cpp + ${output_dir}/gen-cpp2/${service}_processmap_binary.cpp + ${output_dir}/gen-cpp2/${service}_processmap_compact.cpp + ) + endforeach() + + # Emit the rule to run the thrift compiler + add_custom_command( + OUTPUT + ${generated_headers} + ${generated_sources} + COMMAND + ${CMAKE_COMMAND} -E make_directory ${output_dir} + COMMAND + ${FBTHRIFT_COMPILER} + --strict + --templates ${FBTHRIFT_TEMPLATES_DIR} + --gen "mstch_cpp2:${GEN_ARG_STR}" + -I ${CMAKE_SOURCE_DIR} + -o ${output_dir} + ${CMAKE_CURRENT_SOURCE_DIR}/${THRIFT_FILE} + WORKING_DIRECTORY + ${CMAKE_BINARY_DIR} + MAIN_DEPENDENCY + ${THRIFT_FILE} + DEPENDS + ${DEPENDS} + ) + + # Now emit the library rule to compile the sources + add_library(${LIB_NAME} STATIC + ${generated_sources} + ) + set_property( + TARGET ${LIB_NAME} + PROPERTY PUBLIC_HEADER + ${generated_headers} + ) + target_include_directories( + ${LIB_NAME} + PUBLIC + ${CMAKE_SOURCE_DIR} + ${CMAKE_BINARY_DIR} + ${FOLLY_INCLUDE_DIR} + ${FBTHRIFT_INCLUDE_DIR} + ) + target_link_libraries( + ${LIB_NAME} + PUBLIC + ${DEPENDS} + FBThrift::thriftcpp2 + Folly::folly + ) +endfunction() diff --git a/build/fbcode_builder/README.docker b/build/fbcode_builder/README.docker new file mode 100644 index 0000000000..4e9fa8a294 --- /dev/null +++ b/build/fbcode_builder/README.docker @@ -0,0 +1,44 @@ +## Debugging Docker builds + +To debug a a build failure, start up a shell inside the just-failed image as +follows: + +``` +docker ps -a | head # Grab the container ID +docker commit CONTAINER_ID # Grab the SHA string +docker run -it SHA_STRING /bin/bash +# Debug as usual, e.g. `./run-cmake.sh Debug`, `make`, `apt-get install gdb` +``` + +## A note on Docker security + +While the Dockerfile generated above is quite simple, you must be aware that +using Docker to run arbitrary code can present significant security risks: + + - Code signature validation is off by default (as of 2016), exposing you to + man-in-the-middle malicious code injection. + + - You implicitly trust the world -- a Dockerfile cannot annotate that + you trust the image `debian:8.6` because you trust a particular + certificate -- rather, you trust the name, and that it will never be + hijacked. + + - Sandboxing in the Linux kernel is not perfect, and the builds run code as + root. Any compromised code can likely escalate to the host system. + +Specifically, you must be very careful only to add trusted OS images to the +build flow. + +Consider setting this variable before running any Docker container -- this +will validate a signature on the base image before running code from it: + +``` +export DOCKER_CONTENT_TRUST=1 +``` + +Note that unless you go through the extra steps of notarizing the resulting +images, you will have to disable trust to enter intermediate images, e.g. + +``` +DOCKER_CONTENT_TRUST= docker run -it YOUR_IMAGE_ID /bin/bash +``` diff --git a/build/fbcode_builder/README.md b/build/fbcode_builder/README.md new file mode 100644 index 0000000000..084601eebf --- /dev/null +++ b/build/fbcode_builder/README.md @@ -0,0 +1,60 @@ +# Easy builds for Facebook projects + +This is a Python 2.6+ library designed to simplify continuous-integration +(and other builds) of Facebook projects. + +For external Travis builds, the entry point is `travis_docker_build.sh`. + + +## Using Docker to reproduce a CI build + +If you are debugging or enhancing a CI build, you will want to do so from +host or virtual machine that can run a reasonably modern version of Docker: + +``` sh +./make_docker_context.py --help # See available options for OS & compiler +# Tiny wrapper that starts a Travis-like build with compile caching: +os_image=ubuntu:16.04 \ + gcc_version=5 \ + make_parallelism=2 \ + travis_cache_dir=~/travis_ccache \ + ./travis_docker_build.sh &> build_at_$(date +'%Y%m%d_%H%M%S').log +``` + +**IMPORTANT**: Read `fbcode_builder/README.docker` before diving in! + +Setting `travis_cache_dir` turns on [ccache](https://ccache.samba.org/), +saving a fresh copy of `ccache.tgz` after every build. This will invalidate +Docker's layer cache, foring it to rebuild starting right after OS package +setup, but the builds will be fast because all the compiles will be cached. +To iterate without invalidating the Docker layer cache, just `cd +/tmp/docker-context-*` and interact with the `Dockerfile` normally. Note +that the `docker-context-*` dirs preserve a copy of `ccache.tgz` as they +first used it. + + +# What to read next + +The *.py files are fairly well-documented. You might want to peruse them +in this order: + - shell_quoting.py + - fbcode_builder.py + - docker_builder.py + - make_docker_context.py + +As far as runs on Travis go, the control flow is: + - .travis.yml calls + - travis_docker_build.sh calls + - docker_build_with_ccache.sh + +This library also has an (unpublished) component targeting Facebook's +internal continuous-integration platform using the same build-step DSL. + + +# Contributing + +Please follow the ambient style (or PEP-8), and keep the code Python 2.6 +compatible -- since `fbcode_builder`'s only dependency is Docker, we want to +allow building projects on even fairly ancient base systems. We also wish +to be compatible with Python 3, and would appreciate it if you kept that +in mind while making changes also. diff --git a/build/fbcode_builder/docker_build_with_ccache.sh b/build/fbcode_builder/docker_build_with_ccache.sh new file mode 100755 index 0000000000..e922810d59 --- /dev/null +++ b/build/fbcode_builder/docker_build_with_ccache.sh @@ -0,0 +1,219 @@ +#!/bin/bash -uex +# Copyright (c) Facebook, Inc. and its affiliates. +set -o pipefail # Be sure to `|| :` commands that are allowed to fail. + +# +# Future: port this to Python if you are making significant changes. +# + +# Parse command-line arguments +build_timeout="" # Default to no time-out +print_usage() { + echo "Usage: $0 [--build-timeout TIMEOUT_VAL] SAVE-CCACHE-TO-DIR" + echo "SAVE-CCACHE-TO-DIR is required. An empty string discards the ccache." +} +while [[ $# -gt 0 ]]; do + case "$1" in + --build-timeout) + shift + build_timeout="$1" + if [[ "$build_timeout" != "" ]] ; then + timeout "$build_timeout" true # fail early on invalid timeouts + fi + ;; + -h|--help) + print_usage + exit + ;; + *) + break + ;; + esac + shift +done +# There is one required argument, but an empty string is allowed. +if [[ "$#" != 1 ]] ; then + print_usage + exit 1 +fi +save_ccache_to_dir="$1" +if [[ "$save_ccache_to_dir" != "" ]] ; then + mkdir -p "$save_ccache_to_dir" # fail early if there's nowhere to save +else + echo "WARNING: Will not save /ccache from inside the Docker container" +fi + +rand_guid() { + echo "$(date +%s)_${RANDOM}_${RANDOM}_${RANDOM}_${RANDOM}" +} + +id=fbcode_builder_image_id=$(rand_guid) +logfile=$(mktemp) + +echo " + + +Running build with timeout '$build_timeout', label $id, and log in $logfile + + +" + +if [[ "$build_timeout" != "" ]] ; then + # Kill the container after $build_timeout. Using `/bin/timeout` would cause + # Docker to destroy the most recent container and lose its cache. + ( + sleep "$build_timeout" + echo "Build timed out after $build_timeout" 1>&2 + while true; do + maybe_container=$( + grep -E '^( ---> Running in [0-9a-f]+|FBCODE_BUILDER_EXIT)$' "$logfile" | + tail -n 1 | awk '{print $NF}' + ) + if [[ "$maybe_container" == "FBCODE_BUILDER_EXIT" ]] ; then + echo "Time-out successfully terminated build" 1>&2 + break + fi + echo "Time-out: trying to kill $maybe_container" 1>&2 + # This kill fail if we get unlucky, try again soon. + docker kill "$maybe_container" || sleep 5 + done + ) & +fi + +build_exit_code=0 +# `docker build` is allowed to fail, and `pipefail` means we must check the +# failure explicitly. +if ! docker build --label="$id" . 2>&1 | tee "$logfile" ; then + build_exit_code="${PIPESTATUS[0]}" + # NB: We are going to deliberately forge ahead even if `tee` failed. + # If it did, we have a problem with tempfile creation, and all is sad. + echo "Build failed with code $build_exit_code, trying to save ccache" 1>&2 +fi +# Stop trying to kill the container. +echo $'\nFBCODE_BUILDER_EXIT' >> "$logfile" + +if [[ "$save_ccache_to_dir" == "" ]] ; then + echo "Not inspecting Docker build, since saving the ccache wasn't requested." + exit "$build_exit_code" +fi + +img=$(docker images --filter "label=$id" -a -q) +if [[ "$img" == "" ]] ; then + docker images -a + echo "In the above list, failed to find most recent image with $id" 1>&2 + # Usually, the above `docker kill` will leave us with an up-to-the-second + # container, from which we can extract the cache. However, if that fails + # for any reason, this loop will instead grab the latest available image. + # + # It's possible for this log search to get confused due to the output of + # the build command itself, but since our builds aren't **trying** to + # break cache, we probably won't randomly hit an ID from another build. + img=$( + grep -E '^ ---> (Running in [0-9a-f]+|[0-9a-f]+)$' "$logfile" | tac | + sed 's/Running in /container_/;s/ ---> //;' | ( + while read -r x ; do + # Both docker commands below print an image ID to stdout on + # success, so we just need to know when to stop. + if [[ "$x" =~ container_.* ]] ; then + if docker commit "${x#container_}" ; then + break + fi + elif docker inspect --type image -f '{{.Id}}' "$x" ; then + break + fi + done + ) + ) + if [[ "$img" == "" ]] ; then + echo "Failed to find valid container or image ID in log $logfile" 1>&2 + exit 1 + fi +elif [[ "$(echo "$img" | wc -l)" != 1 ]] ; then + # Shouldn't really happen, but be explicit if it does. + echo "Multiple images with label $id, taking the latest of:" + echo "$img" + img=$(echo "$img" | head -n 1) +fi + +container_name="fbcode_builder_container_$(rand_guid)" +echo "Starting $container_name from latest image of the build with $id --" +echo "$img" + +# ccache collection must be done outside of the Docker build steps because +# we need to be able to kill it on timeout. +# +# This step grows the max cache size to slightly exceed than the working set +# of a successful build. This simple design persists the max size in the +# cache directory itself (the env var CCACHE_MAXSIZE does not even work with +# older ccaches like the one on 14.04). +# +# Future: copy this script into the Docker image via Dockerfile. +( + # By default, fbcode_builder creates an unsigned image, so the `docker + # run` below would fail if DOCKER_CONTENT_TRUST were set. So we unset it + # just for this one run. + export DOCKER_CONTENT_TRUST= + # CAUTION: The inner bash runs without -uex, so code accordingly. + docker run --user root --name "$container_name" "$img" /bin/bash -c ' + build_exit_code='"$build_exit_code"' + + # Might be useful if debugging whether max cache size is too small? + grep " Cleaning up cache directory " /tmp/ccache.log + + export CCACHE_DIR=/ccache + ccache -s + + echo "Total bytes in /ccache:"; + total_bytes=$(du -sb /ccache | awk "{print \$1}") + echo "$total_bytes" + + echo "Used bytes in /ccache:"; + used_bytes=$( + du -sb $(find /ccache -type f -newermt @$( + cat /FBCODE_BUILDER_CCACHE_START_TIME + )) | awk "{t += \$1} END {print t}" + ) + echo "$used_bytes" + + # Goal: set the max cache to 750MB over 125% of the usage of a + # successful build. If this is too small, it takes too long to get a + # cache fully warmed up. Plus, ccache cleans 100-200MB before reaching + # the max cache size, so a large margin is essential to prevent misses. + desired_mb=$(( 750 + used_bytes / 800000 )) # 125% in decimal MB: 1e6/1.25 + if [[ "$build_exit_code" != "0" ]] ; then + # For a bad build, disallow shrinking the max cache size. Instead of + # the max cache size, we use on-disk size, which ccache keeps at least + # 150MB under the actual max size, hence the 400MB safety margin. + cur_max_mb=$(( 400 + total_bytes / 1000000 )) # ccache uses decimal MB + if [[ "$desired_mb" -le "$cur_max_mb" ]] ; then + desired_mb="" + fi + fi + + if [[ "$desired_mb" != "" ]] ; then + echo "Updating cache size to $desired_mb MB" + ccache -M "${desired_mb}M" + ccache -s + fi + + # Subshell because `time` the binary may not be installed. + if (time tar czf /ccache.tgz /ccache) ; then + ls -l /ccache.tgz + else + # This `else` ensures we never overwrite the current cache with + # partial data in case of error, even if somebody adds code below. + rm /ccache.tgz + exit 1 + fi + ' +) + +echo "Updating $save_ccache_to_dir/ccache.tgz" +# This will not delete the existing cache if `docker run` didn't make one +docker cp "$container_name:/ccache.tgz" "$save_ccache_to_dir/" + +# Future: it'd be nice if Travis allowed us to retry if the build timed out, +# since we'll make more progress thanks to the cache. As-is, we have to +# wait for the next commit to land. +echo "Build exited with code $build_exit_code" +exit "$build_exit_code" diff --git a/build/fbcode_builder/docker_builder.py b/build/fbcode_builder/docker_builder.py new file mode 100644 index 0000000000..aa251f8a46 --- /dev/null +++ b/build/fbcode_builder/docker_builder.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +''' + +Extends FBCodeBuilder to produce Docker context directories. + +In order to get the largest iteration-time savings from Docker's build +caching, you will want to: + - Use fine-grained steps as appropriate (e.g. separate make & make install), + - Start your action sequence with the lowest-risk steps, and with the steps + that change the least often, and + - Put the steps that you are debugging towards the very end. + +''' +import logging +import os +import shutil +import tempfile + +from fbcode_builder import FBCodeBuilder +from shell_quoting import ( + raw_shell, shell_comment, shell_join, ShellQuoted, path_join +) +from utils import recursively_flatten_list, run_command + + +class DockerFBCodeBuilder(FBCodeBuilder): + + def _user(self): + return self.option('user', 'root') + + def _change_user(self): + return ShellQuoted('USER {u}').format(u=self._user()) + + def setup(self): + # Please add RPM-based OSes here as appropriate. + # + # To allow exercising non-root installs -- we change users after the + # system packages are installed. TODO: For users not defined in the + # image, we should probably `useradd`. + return self.step('Setup', [ + # Docker's FROM does not understand shell quoting. + ShellQuoted('FROM {}'.format(self.option('os_image'))), + # /bin/sh syntax is a pain + ShellQuoted('SHELL ["/bin/bash", "-c"]'), + ] + self.install_debian_deps() + [self._change_user()] + + [self.workdir(self.option('prefix')), + self.create_python_venv(), + self.python_venv()]) + + def python_venv(self): + # To both avoid calling venv activate on each RUN command AND to ensure + # it is present when the resulting container is run add to PATH + actions = [] + if self.option("PYTHON_VENV", "OFF") == "ON": + actions = ShellQuoted('ENV PATH={p}:$PATH').format( + p=path_join(self.option('prefix'), "venv", "bin")) + return(actions) + + def step(self, name, actions): + assert '\n' not in name, 'Name {0} would span > 1 line'.format(name) + b = ShellQuoted('') + return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b] + + def run(self, shell_cmd): + return ShellQuoted('RUN {cmd}').format(cmd=shell_cmd) + + def workdir(self, dir): + return [ + # As late as Docker 1.12.5, this results in `build` being owned + # by root:root -- the explicit `mkdir` works around the bug: + # USER nobody + # WORKDIR build + ShellQuoted('USER root'), + ShellQuoted('RUN mkdir -p {d} && chown {u} {d}').format( + d=dir, u=self._user() + ), + self._change_user(), + ShellQuoted('WORKDIR {dir}').format(dir=dir), + ] + + def comment(self, comment): + # This should not be a command since we don't want comment changes + # to invalidate the Docker build cache. + return shell_comment(comment) + + def copy_local_repo(self, repo_dir, dest_name): + fd, archive_path = tempfile.mkstemp( + prefix='local_repo_{0}_'.format(dest_name), + suffix='.tgz', + dir=os.path.abspath(self.option('docker_context_dir')), + ) + os.close(fd) + run_command('tar', 'czf', archive_path, '.', cwd=repo_dir) + return [ + ShellQuoted('ADD {archive} {dest_name}').format( + archive=os.path.basename(archive_path), dest_name=dest_name + ), + # Docker permissions make very little sense... see also workdir() + ShellQuoted('USER root'), + ShellQuoted('RUN chown -R {u} {d}').format( + d=dest_name, u=self._user() + ), + self._change_user(), + ] + + def _render_impl(self, steps): + return raw_shell(shell_join('\n', recursively_flatten_list(steps))) + + def debian_ccache_setup_steps(self): + source_ccache_tgz = self.option('ccache_tgz', '') + if not source_ccache_tgz: + logging.info('Docker ccache not enabled') + return [] + + dest_ccache_tgz = os.path.join( + self.option('docker_context_dir'), 'ccache.tgz' + ) + + try: + try: + os.link(source_ccache_tgz, dest_ccache_tgz) + except OSError: + logging.exception( + 'Hard-linking {s} to {d} failed, falling back to copy' + .format(s=source_ccache_tgz, d=dest_ccache_tgz) + ) + shutil.copyfile(source_ccache_tgz, dest_ccache_tgz) + except Exception: + logging.exception( + 'Failed to copy or link {s} to {d}, aborting' + .format(s=source_ccache_tgz, d=dest_ccache_tgz) + ) + raise + + return [ + # Separate layer so that in development we avoid re-downloads. + self.run(ShellQuoted('apt-get install -yq ccache')), + ShellQuoted('ADD ccache.tgz /'), + ShellQuoted( + # Set CCACHE_DIR before the `ccache` invocations below. + 'ENV CCACHE_DIR=/ccache ' + # No clang support for now, so it's easiest to hardcode gcc. + 'CC="ccache gcc" CXX="ccache g++" ' + # Always log for ease of debugging. For real FB projects, + # this log is several megabytes, so dumping it to stdout + # would likely exceed the Travis log limit of 4MB. + # + # On a local machine, `docker cp` will get you the data. To + # get the data out from Travis, I would compress and dump + # uuencoded bytes to the log -- for Bistro this was about + # 600kb or 8000 lines: + # + # apt-get install sharutils + # bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2 + 'CCACHE_LOGFILE=/tmp/ccache.log' + ), + self.run(ShellQuoted( + # Future: Skipping this part made this Docker step instant, + # saving ~1min of build time. It's unclear if it is the + # chown or the du, but probably the chown -- since a large + # part of the cost is incurred at image save time. + # + # ccache.tgz may be empty, or may have the wrong + # permissions. + 'mkdir -p /ccache && time chown -R nobody /ccache && ' + 'time du -sh /ccache && ' + # Reset stats so `docker_build_with_ccache.sh` can print + # useful values at the end of the run. + 'echo === Prev run stats === && ccache -s && ccache -z && ' + # Record the current time to let travis_build.sh figure out + # the number of bytes in the cache that are actually used -- + # this is crucial for tuning the maximum cache size. + 'date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && ' + # The build running as `nobody` should be able to write here + 'chown nobody /tmp/ccache.log' + )), + ] diff --git a/build/fbcode_builder/docker_enable_ipv6.sh b/build/fbcode_builder/docker_enable_ipv6.sh new file mode 100755 index 0000000000..3752f6f5e6 --- /dev/null +++ b/build/fbcode_builder/docker_enable_ipv6.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Copyright (c) Facebook, Inc. and its affiliates. + + +# `daemon.json` is normally missing, but let's log it in case that changes. +touch /etc/docker/daemon.json +service docker stop +echo '{"ipv6": true, "fixed-cidr-v6": "2001:db8:1::/64"}' > /etc/docker/daemon.json +service docker start +# Fail early if docker failed on start -- add `- sudo dockerd` to debug. +docker info +# Paranoia log: what if our config got overwritten? +cat /etc/docker/daemon.json diff --git a/build/fbcode_builder/fbcode_builder.py b/build/fbcode_builder/fbcode_builder.py new file mode 100644 index 0000000000..75b0631672 --- /dev/null +++ b/build/fbcode_builder/fbcode_builder.py @@ -0,0 +1,384 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +''' + +This is a small DSL to describe builds of Facebook's open-source projects +that are published to Github from a single internal repo, including projects +that depend on folly, wangle, proxygen, fbthrift, etc. + +This file defines the interface of the DSL, and common utilieis, but you +will have to instantiate a specific builder, with specific options, in +order to get work done -- see e.g. make_docker_context.py. + +== Design notes == + +Goals: + + - A simple declarative language for what needs to be checked out & built, + how, in what order. + + - The same specification should work for external continuous integration + builds (e.g. Travis + Docker) and for internal VM-based continuous + integration builds. + + - One should be able to build without root, and to install to a prefix. + +Non-goals: + + - General usefulness. The only point of this is to make it easier to build + and test Facebook's open-source services. + +Ideas for the future -- these may not be very good :) + + - Especially on Ubuntu 14.04 the current initial setup is inefficient: + we add PPAs after having installed a bunch of packages -- this prompts + reinstalls of large amounts of code. We also `apt-get update` a few + times. + + - A "shell script" builder. Like DockerFBCodeBuilder, but outputs a + shell script that runs outside of a container. Or maybe even + synchronously executes the shell commands, `make`-style. + + - A "Makefile" generator. That might make iterating on builds even quicker + than what you can currently get with Docker build caching. + + - Generate a rebuild script that can be run e.g. inside the built Docker + container by tagging certain steps with list-inheriting Python objects: + * do change directories + * do NOT `git clone` -- if we want to update code this should be a + separate script that e.g. runs rebase on top of specific targets + across all the repos. + * do NOT install software (most / all setup can be skipped) + * do NOT `autoreconf` or `configure` + * do `make` and `cmake` + + - If we get non-Debian OSes, part of ccache setup should be factored out. +''' + +import os +import re + +from shell_quoting import path_join, shell_join, ShellQuoted + + +def _read_project_github_hashes(): + base_dir = 'deps/github_hashes/' # trailing slash used in regex below + for dirname, _, files in os.walk(base_dir): + for filename in files: + path = os.path.join(dirname, filename) + with open(path) as f: + m_proj = re.match('^' + base_dir + '(.*)-rev\.txt$', path) + if m_proj is None: + raise RuntimeError('Not a hash file? {0}'.format(path)) + m_hash = re.match('^Subproject commit ([0-9a-f]+)\n$', f.read()) + if m_hash is None: + raise RuntimeError('No hash in {0}'.format(path)) + yield m_proj.group(1), m_hash.group(1) + + +class FBCodeBuilder(object): + + def __init__(self, **kwargs): + self._options_do_not_access = kwargs # Use .option() instead. + # This raises upon detecting options that are specified but unused, + # because otherwise it is very easy to make a typo in option names. + self.options_used = set() + self._github_hashes = dict(_read_project_github_hashes()) + + def __repr__(self): + return '{0}({1})'.format( + self.__class__.__name__, + ', '.join( + '{0}={1}'.format(k, repr(v)) + for k, v in self._options_do_not_access.items() + ) + ) + + def option(self, name, default=None): + value = self._options_do_not_access.get(name, default) + if value is None: + raise RuntimeError('Option {0} is required'.format(name)) + self.options_used.add(name) + return value + + def has_option(self, name): + return name in self._options_do_not_access + + def add_option(self, name, value): + if name in self._options_do_not_access: + raise RuntimeError('Option {0} already set'.format(name)) + self._options_do_not_access[name] = value + + # + # Abstract parts common to every installation flow + # + + def render(self, steps): + ''' + + Converts nested actions to your builder's expected output format. + Typically takes the output of build(). + + ''' + res = self._render_impl(steps) # Implementation-dependent + # Now that the output is rendered, we expect all options to have + # been used. + unused_options = set(self._options_do_not_access) + unused_options -= self.options_used + if unused_options: + raise RuntimeError( + 'Unused options: {0} -- please check if you made a typo ' + 'in any of them. Those that are truly not useful should ' + 'be not be set so that this typo detection can be useful.' + .format(unused_options) + ) + return res + + def build(self, steps): + if not steps: + raise RuntimeError('Please ensure that the config you are passing ' + 'contains steps') + return [self.setup(), self.diagnostics()] + steps + + def setup(self): + 'Your builder may want to install packages here.' + raise NotImplementedError + + def diagnostics(self): + 'Log some system diagnostics before/after setup for ease of debugging' + # The builder's repr is not used in a command to avoid pointlessly + # invalidating Docker's build cache. + return self.step('Diagnostics', [ + self.comment('Builder {0}'.format(repr(self))), + self.run(ShellQuoted('hostname')), + self.run(ShellQuoted('cat /etc/issue || echo no /etc/issue')), + self.run(ShellQuoted('g++ --version || echo g++ not installed')), + self.run(ShellQuoted('cmake --version || echo cmake not installed')), + ]) + + def step(self, name, actions): + 'A labeled collection of actions or other steps' + raise NotImplementedError + + def run(self, shell_cmd): + 'Run this bash command' + raise NotImplementedError + + def workdir(self, dir): + 'Create this directory if it does not exist, and change into it' + raise NotImplementedError + + def copy_local_repo(self, dir, dest_name): + ''' + Copy the local repo at `dir` into this step's `workdir()`, analog of: + cp -r /path/to/folly folly + ''' + raise NotImplementedError + + def debian_deps(self): + return [ + 'autoconf-archive', + 'bison', + 'build-essential', + 'cmake', + 'curl', + 'flex', + 'git', + 'gperf', + 'joe', + 'libboost-all-dev', + 'libcap-dev', + 'libdouble-conversion-dev', + 'libevent-dev', + 'libgflags-dev', + 'libgoogle-glog-dev', + 'libkrb5-dev', + 'libpcre3-dev', + 'libpthread-stubs0-dev', + 'libnuma-dev', + 'libsasl2-dev', + 'libsnappy-dev', + 'libsqlite3-dev', + 'libssl-dev', + 'libtool', + 'netcat-openbsd', + 'pkg-config', + 'sudo', + 'unzip', + 'wget', + 'python3-venv', + ] + + # + # Specific build helpers + # + + def install_debian_deps(self): + actions = [ + self.run( + ShellQuoted('apt-get update && apt-get install -yq {deps}').format( + deps=shell_join(' ', ( + ShellQuoted(dep) for dep in self.debian_deps()))) + ), + ] + gcc_version = self.option('gcc_version') + + # Make the selected GCC the default before building anything + actions.extend([ + self.run(ShellQuoted('apt-get install -yq {c} {cpp}').format( + c=ShellQuoted('gcc-{v}').format(v=gcc_version), + cpp=ShellQuoted('g++-{v}').format(v=gcc_version), + )), + self.run(ShellQuoted( + 'update-alternatives --install /usr/bin/gcc gcc {c} 40 ' + '--slave /usr/bin/g++ g++ {cpp}' + ).format( + c=ShellQuoted('/usr/bin/gcc-{v}').format(v=gcc_version), + cpp=ShellQuoted('/usr/bin/g++-{v}').format(v=gcc_version), + )), + self.run(ShellQuoted('update-alternatives --config gcc')), + ]) + + actions.extend(self.debian_ccache_setup_steps()) + + return self.step('Install packages for Debian-based OS', actions) + + def create_python_venv(self): + action = [] + if self.option("PYTHON_VENV", "OFF") == "ON": + action = self.run(ShellQuoted("python3 -m venv {p}").format( + p=path_join(self.option('prefix'), "venv"))) + return(action) + + def python_venv(self): + action = [] + if self.option("PYTHON_VENV", "OFF") == "ON": + action = ShellQuoted("source {p}").format( + p=path_join(self.option('prefix'), "venv", "bin", "activate")) + return(action) + + def debian_ccache_setup_steps(self): + return [] # It's ok to ship a renderer without ccache support. + + def github_project_workdir(self, project, path): + # Only check out a non-default branch if requested. This especially + # makes sense when building from a local repo. + git_hash = self.option( + '{0}:git_hash'.format(project), + # Any repo that has a hash in deps/github_hashes defaults to + # that, with the goal of making builds maximally consistent. + self._github_hashes.get(project, '') + ) + maybe_change_branch = [ + self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)), + ] if git_hash else [] + + base_dir = self.option('projects_dir') + + local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '') + return self.step('Check out {0}, workdir {1}'.format(project, path), [ + self.workdir(base_dir), + self.run( + ShellQuoted('git clone https://github.com/{p}').format(p=project) + ) if not local_repo_dir else self.copy_local_repo( + local_repo_dir, os.path.basename(project) + ), + self.workdir(path_join(base_dir, os.path.basename(project), path)), + ] + maybe_change_branch) + + def fb_github_project_workdir(self, project_and_path, github_org='facebook'): + 'This helper lets Facebook-internal CI special-cases FB projects' + project, path = project_and_path.split('/', 1) + return self.github_project_workdir(github_org + '/' + project, path) + + def _make_vars(self, make_vars): + return shell_join(' ', ( + ShellQuoted('{k}={v}').format(k=k, v=v) + for k, v in ({} if make_vars is None else make_vars).items() + )) + + def parallel_make(self, make_vars=None): + return self.run(ShellQuoted('make -j {n} VERBOSE=1 {vars}').format( + n=self.option('make_parallelism'), + vars=self._make_vars(make_vars), + )) + + def make_and_install(self, make_vars=None): + return [ + self.parallel_make(make_vars), + self.run(ShellQuoted('make install VERBOSE=1 {vars}').format( + vars=self._make_vars(make_vars), + )), + ] + + def configure(self, name=None): + autoconf_options = {} + if name is not None: + autoconf_options.update( + self.option('{0}:autoconf_options'.format(name), {}) + ) + return [ + self.run(ShellQuoted( + 'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" ' + 'CFLAGS="$CFLAGS -I"{p}"/include" ' + 'CPPFLAGS="$CPPFLAGS -I"{p}"/include" ' + 'PY_PREFIX={p} ' + './configure --prefix={p} {args}' + ).format( + p=self.option('prefix'), + args=shell_join(' ', ( + ShellQuoted('{k}={v}').format(k=k, v=v) + for k, v in autoconf_options.items() + )), + )), + ] + + def autoconf_install(self, name): + return self.step('Build and install {0}'.format(name), [ + self.run(ShellQuoted('autoreconf -ivf')), + ] + self.configure() + self.make_and_install()) + + def cmake_configure(self, name, cmake_path='..'): + cmake_defines = { + 'BUILD_SHARED_LIBS': 'ON', + 'CMAKE_INSTALL_PREFIX': self.option('prefix'), + } + cmake_defines.update( + self.option('{0}:cmake_defines'.format(name), {}) + ) + return [ + self.run(ShellQuoted( + 'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" ' + 'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" ' + 'cmake {args} {cmake_path}' + ).format( + p=self.option('prefix'), + args=shell_join(' ', ( + ShellQuoted('-D{k}={v}').format(k=k, v=v) + for k, v in cmake_defines.items() + )), + cmake_path=cmake_path, + )), + ] + + def cmake_install(self, name, cmake_path='..'): + return self.step( + 'Build and install {0}'.format(name), + self.cmake_configure(name, cmake_path) + self.make_and_install() + ) + + def fb_github_autoconf_install(self, project_and_path, github_org='facebook'): + return [ + self.fb_github_project_workdir(project_and_path, github_org), + self.autoconf_install(project_and_path), + ] + + def fb_github_cmake_install(self, project_and_path, cmake_path='..', github_org='facebook'): + return [ + self.fb_github_project_workdir(project_and_path, github_org), + self.cmake_install(project_and_path, cmake_path), + ] diff --git a/build/fbcode_builder/fbcode_builder_config.py b/build/fbcode_builder/fbcode_builder_config.py new file mode 100644 index 0000000000..c8f868a519 --- /dev/null +++ b/build/fbcode_builder/fbcode_builder_config.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +'Demo config, so that `make_docker_context.py --help` works in this directory.' + +config = { + 'fbcode_builder_spec': lambda _builder: { + 'depends_on': [], + 'steps': [], + }, + 'github_project': 'demo/project', +} diff --git a/build/fbcode_builder/make_docker_context.py b/build/fbcode_builder/make_docker_context.py new file mode 100755 index 0000000000..30aad9e821 --- /dev/null +++ b/build/fbcode_builder/make_docker_context.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +''' +Reads `fbcode_builder_config.py` from the current directory, and prepares a +Docker context directory to build this project. Prints to stdout the path +to the context directory. + +Try `.../make_docker_context.py --help` from a project's `build/` directory. + +By default, the Docker context directory will be in /tmp. It will always +contain a Dockerfile, and might also contain copies of your local repos, and +other data needed for the build container. +''' + +import os +import tempfile +import textwrap + +from docker_builder import DockerFBCodeBuilder +from parse_args import parse_args_to_fbcode_builder_opts + + +def make_docker_context( + get_steps_fn, github_project, opts=None, default_context_dir=None +): + ''' + Returns a path to the Docker context directory. See parse_args.py. + + Helper for making a command-line utility that writes your project's + Dockerfile and associated data into a (temporary) directory. Your main + program might look something like this: + + print(make_docker_context( + lambda builder: [builder.step(...), ...], + 'facebook/your_project', + )) + ''' + + if opts is None: + opts = {} + + valid_versions = ( + ('ubuntu:16.04', '5'), + ) + + def add_args(parser): + parser.add_argument( + '--docker-context-dir', metavar='DIR', + default=default_context_dir, + help='Write the Dockerfile and its context into this directory. ' + 'If empty, make a temporary directory. Default: %(default)s.', + ) + parser.add_argument( + '--user', metavar='NAME', default=opts.get('user', 'nobody'), + help='Build and install as this user. Default: %(default)s.', + ) + parser.add_argument( + '--prefix', metavar='DIR', + default=opts.get('prefix', '/home/install'), + help='Install all libraries in this prefix. Default: %(default)s.', + ) + parser.add_argument( + '--projects-dir', metavar='DIR', + default=opts.get('projects_dir', '/home'), + help='Place project code directories here. Default: %(default)s.', + ) + parser.add_argument( + '--os-image', metavar='IMG', choices=zip(*valid_versions)[0], + default=opts.get('os_image', valid_versions[0][0]), + help='Docker OS image -- be sure to use only ones you trust (See ' + 'README.docker). Choices: %(choices)s. Default: %(default)s.', + ) + parser.add_argument( + '--gcc-version', metavar='VER', + choices=set(zip(*valid_versions)[1]), + default=opts.get('gcc_version', valid_versions[0][1]), + help='Choices: %(choices)s. Default: %(default)s.', + ) + parser.add_argument( + '--make-parallelism', metavar='NUM', type=int, + default=opts.get('make_parallelism', 1), + help='Use `make -j` on multi-CPU systems with lots of RAM. ' + 'Default: %(default)s.', + ) + parser.add_argument( + '--local-repo-dir', metavar='DIR', + help='If set, build {0} from a local directory instead of Github.' + .format(github_project), + ) + parser.add_argument( + '--ccache-tgz', metavar='PATH', + help='If set, enable ccache for the build. To initialize the ' + 'cache, first try to hardlink, then to copy --cache-tgz ' + 'as ccache.tgz into the --docker-context-dir.' + ) + + opts = parse_args_to_fbcode_builder_opts( + add_args, + # These have add_argument() calls, others are set via --option. + ( + 'docker_context_dir', + 'user', + 'prefix', + 'projects_dir', + 'os_image', + 'gcc_version', + 'make_parallelism', + 'local_repo_dir', + 'ccache_tgz', + ), + opts, + help=textwrap.dedent(''' + + Reads `fbcode_builder_config.py` from the current directory, and + prepares a Docker context directory to build {github_project} and + its dependencies. Prints to stdout the path to the context + directory. + + Pass --option {github_project}:git_hash SHA1 to build something + other than the master branch from Github. + + Or, pass --option {github_project}:local_repo_dir LOCAL_PATH to + build from a local repo instead of cloning from Github. + + Usage: + (cd $(./make_docker_context.py) && docker build . 2>&1 | tee log) + + '''.format(github_project=github_project)), + ) + + # This allows travis_docker_build.sh not to know the main Github project. + local_repo_dir = opts.pop('local_repo_dir', None) + if local_repo_dir is not None: + opts['{0}:local_repo_dir'.format(github_project)] = local_repo_dir + + if (opts.get('os_image'), opts.get('gcc_version')) not in valid_versions: + raise Exception( + 'Due to 4/5 ABI changes (std::string), we can only use {0}'.format( + ' / '.join('GCC {1} on {0}'.format(*p) for p in valid_versions) + ) + ) + + if opts.get('docker_context_dir') is None: + opts['docker_context_dir'] = tempfile.mkdtemp(prefix='docker-context-') + elif not os.path.exists(opts.get('docker_context_dir')): + os.makedirs(opts.get('docker_context_dir')) + + builder = DockerFBCodeBuilder(**opts) + context_dir = builder.option('docker_context_dir') # Mark option "in-use" + # The renderer may also populate some files into the context_dir. + dockerfile = builder.render(get_steps_fn(builder)) + + with os.fdopen(os.open( + os.path.join(context_dir, 'Dockerfile'), + os.O_RDWR | os.O_CREAT | os.O_EXCL, # Do not overwrite existing files + 0o644, + ), 'w') as f: + f.write(dockerfile) + + return context_dir + + +if __name__ == '__main__': + from utils import read_fbcode_builder_config, build_fbcode_builder_config + + # Load a spec from the current directory + config = read_fbcode_builder_config('fbcode_builder_config.py') + print(make_docker_context( + build_fbcode_builder_config(config), + config['github_project'], + )) diff --git a/build/fbcode_builder/parse_args.py b/build/fbcode_builder/parse_args.py new file mode 100644 index 0000000000..def9e504de --- /dev/null +++ b/build/fbcode_builder/parse_args.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +'Argument parsing logic shared by all fbcode_builder CLI tools.' + +import argparse +import logging + +from shell_quoting import raw_shell, ShellQuoted + + +def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help): + ''' + + Provides some standard arguments: --debug, --option, --shell-quoted-option + + Then, calls `add_args_fn(parser)` to add application-specific arguments. + + `opts` are first used as defaults for the various command-line + arguments. Then, the parsed arguments are mapped back into `opts`, + which then become the values for `FBCodeBuilder.option()`, to be used + both by the builder and by `get_steps_fn()`. + + `help` is printed in response to the `--help` argument. + + ''' + top_level_opts = set(top_level_opts) + + parser = argparse.ArgumentParser( + description=help, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + add_args_fn(parser) + + parser.add_argument( + '--option', nargs=2, metavar=('KEY', 'VALUE'), action='append', + default=[ + (k, v) for k, v in opts.items() + if k not in top_level_opts and not isinstance(v, ShellQuoted) + ], + help='Set project-specific options. These are assumed to be raw ' + 'strings, to be shell-escaped as needed. Default: %(default)s.', + ) + parser.add_argument( + '--shell-quoted-option', nargs=2, metavar=('KEY', 'VALUE'), + action='append', + default=[ + (k, raw_shell(v)) for k, v in opts.items() + if k not in top_level_opts and isinstance(v, ShellQuoted) + ], + help='Set project-specific options. These are assumed to be shell-' + 'quoted, and may be used in commands as-is. Default: %(default)s.', + ) + + parser.add_argument('--debug', action='store_true', help='Log more') + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO, + format='%(levelname)s: %(message)s' + ) + + # Map command-line args back into opts. + logging.debug('opts before command-line arguments: {0}'.format(opts)) + + new_opts = {} + for key in top_level_opts: + val = getattr(args, key) + # Allow clients to unset a default by passing a value of None in opts + if val is not None: + new_opts[key] = val + for key, val in args.option: + new_opts[key] = val + for key, val in args.shell_quoted_option: + new_opts[key] = ShellQuoted(val) + + logging.debug('opts after command-line arguments: {0}'.format(new_opts)) + + return new_opts diff --git a/build/fbcode_builder/shell_builder.py b/build/fbcode_builder/shell_builder.py new file mode 100644 index 0000000000..5bb41fe57e --- /dev/null +++ b/build/fbcode_builder/shell_builder.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +''' +shell_builder.py allows running the fbcode_builder logic +on the host rather than in a container. + +It emits a bash script with set -exo pipefail configured such that +any failing step will cause the script to exit with failure. + +== How to run it? == + +cd build +python fbcode_builder/shell_builder.py > ~/run.sh +bash ~/run.sh +''' + +import os +import distutils.spawn + +from fbcode_builder import FBCodeBuilder +from shell_quoting import ( + raw_shell, shell_comment, shell_join, ShellQuoted +) +from utils import recursively_flatten_list + + +class ShellFBCodeBuilder(FBCodeBuilder): + def _render_impl(self, steps): + return raw_shell(shell_join('\n', recursively_flatten_list(steps))) + + def workdir(self, dir): + return [ + ShellQuoted('mkdir -p {d} && cd {d}').format( + d=dir + ), + ] + + def run(self, shell_cmd): + return ShellQuoted('{cmd}').format(cmd=shell_cmd) + + def step(self, name, actions): + assert '\n' not in name, 'Name {0} would span > 1 line'.format(name) + b = ShellQuoted('') + return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b] + + def setup(self): + steps = [ + ShellQuoted('set -exo pipefail'), + ] + [self.create_python_venv(), self.python_venv()] + if self.has_option('ccache_dir'): + ccache_dir = self.option('ccache_dir') + steps += [ + ShellQuoted( + # Set CCACHE_DIR before the `ccache` invocations below. + 'export CCACHE_DIR={ccache_dir} ' + 'CC="ccache ${{CC:-gcc}}" CXX="ccache ${{CXX:-g++}}"' + ).format(ccache_dir=ccache_dir) + ] + return steps + + def comment(self, comment): + return shell_comment(comment) + + def copy_local_repo(self, dir, dest_name): + return [ + ShellQuoted('cp -r {dir} {dest_name}').format( + dir=dir, + dest_name=dest_name + ), + ] + + +def find_project_root(): + here = os.path.dirname(os.path.realpath(__file__)) + maybe_root = os.path.dirname(os.path.dirname(here)) + if os.path.isdir(os.path.join(maybe_root, '.git')): + return maybe_root + raise RuntimeError( + "I expected shell_builder.py to be in the " + "build/fbcode_builder subdir of a git repo") + + +def persistent_temp_dir(repo_root): + escaped = repo_root.replace('/', 'sZs').replace('\\', 'sZs').replace(':', '') + return os.path.join(os.path.expandvars("$HOME"), '.fbcode_builder-' + escaped) + + +if __name__ == '__main__': + from utils import read_fbcode_builder_config, build_fbcode_builder_config + repo_root = find_project_root() + temp = persistent_temp_dir(repo_root) + + config = read_fbcode_builder_config('fbcode_builder_config.py') + builder = ShellFBCodeBuilder() + + builder.add_option('projects_dir', temp) + if distutils.spawn.find_executable('ccache'): + builder.add_option('ccache_dir', + os.environ.get('CCACHE_DIR', os.path.join(temp, '.ccache'))) + builder.add_option('prefix', os.path.join(temp, 'installed')) + builder.add_option('make_parallelism', 4) + builder.add_option( + '{project}:local_repo_dir'.format(project=config['github_project']), + repo_root) + make_steps = build_fbcode_builder_config(config) + steps = make_steps(builder) + print(builder.render(steps)) diff --git a/build/fbcode_builder/shell_quoting.py b/build/fbcode_builder/shell_quoting.py new file mode 100644 index 0000000000..f3b968a6db --- /dev/null +++ b/build/fbcode_builder/shell_quoting.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +''' + +Almost every FBCodeBuilder string is ultimately passed to a shell. Escaping +too little or too much tends to be the most common error. The utilities in +this file give a systematic way of avoiding such bugs: + - When you write literal strings destined for the shell, use `ShellQuoted`. + - When these literal strings are parameterized, use `ShellQuoted.format`. + - Any parameters that are raw strings get `shell_quote`d automatically, + while any ShellQuoted parameters will be left intact. + - Use `path_join` to join path components. + - Use `shell_join` to join already-quoted command arguments or shell lines. + +''' + +import os + +from collections import namedtuple + + +class ShellQuoted(namedtuple('ShellQuoted', ('do_not_use_raw_str',))): + ''' + + Wrap a string with this to make it transparent to shell_quote(). It + will almost always suffice to use ShellQuoted.format(), path_join(), + or shell_join(). + + If you really must, use raw_shell() to access the raw string. + + ''' + + def __new__(cls, s): + 'No need to nest ShellQuoted.' + return super(ShellQuoted, cls).__new__( + cls, s.do_not_use_raw_str if isinstance(s, ShellQuoted) else s + ) + + def __str__(self): + raise RuntimeError( + 'One does not simply convert {0} to a string -- use path_join() ' + 'or ShellQuoted.format() instead'.format(repr(self)) + ) + + def __repr__(self): + return '{0}({1})'.format( + self.__class__.__name__, repr(self.do_not_use_raw_str) + ) + + def format(self, **kwargs): + ''' + + Use instead of str.format() when the arguments are either + `ShellQuoted()` or raw strings needing to be `shell_quote()`d. + + Positional args are deliberately not supported since they are more + error-prone. + + ''' + return ShellQuoted(self.do_not_use_raw_str.format(**dict( + (k, shell_quote(v).do_not_use_raw_str) for k, v in kwargs.items() + ))) + + +def shell_quote(s): + 'Quotes a string if it is not already quoted' + return s if isinstance(s, ShellQuoted) \ + else ShellQuoted("'" + str(s).replace("'", "'\\''") + "'") + + +def raw_shell(s): + 'Not a member of ShellQuoted so we get a useful error for raw strings' + if isinstance(s, ShellQuoted): + return s.do_not_use_raw_str + raise RuntimeError('{0} should have been ShellQuoted'.format(s)) + + +def shell_join(delim, it): + 'Joins an iterable of ShellQuoted with a delimiter between each two' + return ShellQuoted(delim.join(raw_shell(s) for s in it)) + + +def path_join(*args): + 'Joins ShellQuoted and raw pieces of paths to make a shell-quoted path' + return ShellQuoted(os.path.join(*[ + raw_shell(shell_quote(s)) for s in args + ])) + + +def shell_comment(c): + 'Do not shell-escape raw strings in comments, but do handle line breaks.' + return ShellQuoted('# {c}').format(c=ShellQuoted( + (raw_shell(c) if isinstance(c, ShellQuoted) else c) + .replace('\n', '\n# ') + )) diff --git a/build/fbcode_builder/specs/__init__.py b/build/fbcode_builder/specs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/build/fbcode_builder/specs/fbthrift.py b/build/fbcode_builder/specs/fbthrift.py new file mode 100644 index 0000000000..d982fc7981 --- /dev/null +++ b/build/fbcode_builder/specs/fbthrift.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.folly as folly +import specs.fizz as fizz +import specs.fmt as fmt +import specs.rsocket as rsocket +import specs.sodium as sodium +import specs.wangle as wangle +import specs.zstd as zstd + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + # This API should change rarely, so build the latest tag instead of master. + builder.add_option( + 'no1msd/mstch:git_hash', + ShellQuoted('$(git describe --abbrev=0 --tags)') + ) + return { + 'depends_on': [folly, fizz, fmt, sodium, rsocket, wangle, zstd], + 'steps': [ + # This isn't a separete spec, since only fbthrift uses mstch. + builder.github_project_workdir('no1msd/mstch', 'build'), + builder.cmake_install('no1msd/mstch'), + builder.fb_github_cmake_install('fbthrift/thrift'), + ], + } diff --git a/build/fbcode_builder/specs/fbzmq.py b/build/fbcode_builder/specs/fbzmq.py new file mode 100644 index 0000000000..588b349676 --- /dev/null +++ b/build/fbcode_builder/specs/fbzmq.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.fbthrift as fbthrift +import specs.folly as folly +import specs.gmock as gmock +import specs.sodium as sodium +import specs.sigar as sigar + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + builder.add_option('zeromq/libzmq:git_hash', 'v4.2.2') + return { + 'depends_on': [folly, fbthrift, gmock, sodium, sigar], + 'steps': [ + builder.github_project_workdir('zeromq/libzmq', '.'), + builder.step('Build and install zeromq/libzmq', [ + builder.run(ShellQuoted('./autogen.sh')), + builder.configure(), + builder.make_and_install(), + ]), + + builder.fb_github_project_workdir('fbzmq/fbzmq/build', 'facebook'), + builder.step('Build and install fbzmq/fbzmq/build', [ + builder.cmake_configure('fbzmq/fbzmq/build'), + # we need the pythonpath to find the thrift compiler + builder.run(ShellQuoted( + 'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages ' + 'make -j {n}' + ).format(p=builder.option('prefix'), n=builder.option('make_parallelism'))), + builder.run(ShellQuoted('make install')), + ]), + ], + } diff --git a/build/fbcode_builder/specs/fizz.py b/build/fbcode_builder/specs/fizz.py new file mode 100644 index 0000000000..20fc321aa6 --- /dev/null +++ b/build/fbcode_builder/specs/fizz.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.folly as folly +import specs.sodium as sodium + + +def fbcode_builder_spec(builder): + return { + 'depends_on': [folly, sodium], + 'steps': [ + builder.fb_github_cmake_install( + 'fizz/fizz/build', + github_org='facebookincubator', + ), + ], + } diff --git a/build/fbcode_builder/specs/fmt.py b/build/fbcode_builder/specs/fmt.py new file mode 100644 index 0000000000..b68534c0dd --- /dev/null +++ b/build/fbcode_builder/specs/fmt.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + builder.add_option('fmtlib/fmt:git_hash', '5.3.0') + return { + 'steps': [ + builder.github_project_workdir('fmtlib/fmt', 'build'), + builder.cmake_install('fmtlib/fmt'), + ], + } diff --git a/build/fbcode_builder/specs/folly.py b/build/fbcode_builder/specs/folly.py new file mode 100644 index 0000000000..3d128b9c07 --- /dev/null +++ b/build/fbcode_builder/specs/folly.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + return { + 'steps': [ + # on macOS the filesystem is typically case insensitive. + # We need to ensure that the CWD is not the folly source + # dir when we build, otherwise the system will decide + # that `folly/String.h` is the file it wants when including + # `string.h` and the build will fail. + builder.fb_github_project_workdir('folly/_build'), + builder.cmake_install('facebook/folly'), + ], + } diff --git a/build/fbcode_builder/specs/gmock.py b/build/fbcode_builder/specs/gmock.py new file mode 100644 index 0000000000..8b0562f7e6 --- /dev/null +++ b/build/fbcode_builder/specs/gmock.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + builder.add_option('google/googletest:git_hash', 'release-1.8.1') + builder.add_option( + 'google/googletest:cmake_defines', + { + 'BUILD_GTEST': 'ON', + # Avoid problems with MACOSX_RPATH + 'BUILD_SHARED_LIBS': 'OFF', + } + ) + return { + 'steps': [ + builder.github_project_workdir('google/googletest', 'build'), + builder.cmake_install('google/googletest'), + ], + } diff --git a/build/fbcode_builder/specs/proxygen.py b/build/fbcode_builder/specs/proxygen.py new file mode 100644 index 0000000000..b937e425ab --- /dev/null +++ b/build/fbcode_builder/specs/proxygen.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.folly as folly +import specs.fizz as fizz +import specs.sodium as sodium +import specs.wangle as wangle + + +def fbcode_builder_spec(builder): + return { + 'depends_on': [folly, wangle, fizz, sodium], + 'steps': [ + builder.fb_github_autoconf_install('proxygen/proxygen'), + ], + } diff --git a/build/fbcode_builder/specs/re2.py b/build/fbcode_builder/specs/re2.py new file mode 100644 index 0000000000..b6b81ab942 --- /dev/null +++ b/build/fbcode_builder/specs/re2.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + return { + 'steps': [ + builder.github_project_workdir('google/re2', 'build'), + builder.cmake_install('google/re2'), + ], + } diff --git a/build/fbcode_builder/specs/rocksdb.py b/build/fbcode_builder/specs/rocksdb.py new file mode 100644 index 0000000000..c7d7c6ac2b --- /dev/null +++ b/build/fbcode_builder/specs/rocksdb.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +def fbcode_builder_spec(builder): + builder.add_option("rocksdb/_build:cmake_defines", { + "USE_RTTI": "1", + "PORTABLE": "ON", + }) + return { + "steps": [ + builder.fb_github_cmake_install("rocksdb/_build"), + ], + } diff --git a/build/fbcode_builder/specs/rsocket.py b/build/fbcode_builder/specs/rsocket.py new file mode 100644 index 0000000000..7409585268 --- /dev/null +++ b/build/fbcode_builder/specs/rsocket.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.gmock as gmock +import specs.folly as folly + + +def fbcode_builder_spec(builder): + return { + 'depends_on': [folly, gmock], + 'steps': [ + builder.fb_github_cmake_install( + 'rsocket-cpp/rsocket', + github_org='rsocket'), + ], + } diff --git a/build/fbcode_builder/specs/sigar.py b/build/fbcode_builder/specs/sigar.py new file mode 100644 index 0000000000..acb64e63f0 --- /dev/null +++ b/build/fbcode_builder/specs/sigar.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + builder.add_option( + 'hyperic/sigar:autoconf_options', {'CFLAGS' : '-fgnu89-inline'}) + return { + 'steps': [ + builder.github_project_workdir('hyperic/sigar', '.'), + builder.step('Build and install sigar', [ + builder.run(ShellQuoted('./autogen.sh')), + builder.configure('hyperic/sigar'), + builder.make_and_install(), + ]), + ], + } diff --git a/build/fbcode_builder/specs/sodium.py b/build/fbcode_builder/specs/sodium.py new file mode 100644 index 0000000000..52bb0006ec --- /dev/null +++ b/build/fbcode_builder/specs/sodium.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + builder.add_option('jedisct1/libsodium:git_hash', 'stable') + return { + 'steps': [ + builder.github_project_workdir('jedisct1/libsodium', '.'), + builder.step('Build and install jedisct1/libsodium', [ + builder.run(ShellQuoted('./autogen.sh')), + builder.configure(), + builder.make_and_install(), + ]), + ], + } diff --git a/build/fbcode_builder/specs/wangle.py b/build/fbcode_builder/specs/wangle.py new file mode 100644 index 0000000000..15cf3ff468 --- /dev/null +++ b/build/fbcode_builder/specs/wangle.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import specs.folly as folly +import specs.fizz as fizz +import specs.sodium as sodium + + +def fbcode_builder_spec(builder): + # Projects that simply depend on Wangle need not spend time on tests. + builder.add_option('wangle/wangle/build:cmake_defines', {'BUILD_TESTS': 'OFF'}) + return { + 'depends_on': [folly, fizz, sodium], + 'steps': [ + builder.fb_github_cmake_install('wangle/wangle/build'), + ], + } diff --git a/build/fbcode_builder/specs/zstd.py b/build/fbcode_builder/specs/zstd.py new file mode 100644 index 0000000000..d24385dd7d --- /dev/null +++ b/build/fbcode_builder/specs/zstd.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from shell_quoting import ShellQuoted + + +def fbcode_builder_spec(builder): + # This API should change rarely, so build the latest tag instead of master. + builder.add_option( + 'facebook/zstd:git_hash', + ShellQuoted('$(git describe --abbrev=0 --tags origin/master)') + ) + return { + 'steps': [ + builder.github_project_workdir('facebook/zstd', '.'), + builder.step('Build and install zstd', [ + builder.make_and_install(make_vars={ + 'PREFIX': builder.option('prefix'), + }) + ]), + ], + } diff --git a/build/fbcode_builder/travis_docker_build.sh b/build/fbcode_builder/travis_docker_build.sh new file mode 100755 index 0000000000..9733bb206b --- /dev/null +++ b/build/fbcode_builder/travis_docker_build.sh @@ -0,0 +1,42 @@ +#!/bin/bash -uex +# Copyright (c) Facebook, Inc. and its affiliates. +# .travis.yml in the top-level dir explains why this is a separate script. +# Read the docs: ./make_docker_context.py --help + +os_image=${os_image?Must be set by Travis} +gcc_version=${gcc_version?Must be set by Travis} +make_parallelism=${make_parallelism:-4} +# ccache is off unless requested +travis_cache_dir=${travis_cache_dir:-} +# The docker build never times out, unless specified +docker_build_timeout=${docker_build_timeout:-} + +cur_dir="$(readlink -f "$(dirname "$0")")" + +if [[ "$travis_cache_dir" == "" ]]; then + echo "ccache disabled, enable by setting env. var. travis_cache_dir" + ccache_tgz="" +elif [[ -e "$travis_cache_dir/ccache.tgz" ]]; then + ccache_tgz="$travis_cache_dir/ccache.tgz" +else + echo "$travis_cache_dir/ccache.tgz does not exist, starting with empty cache" + ccache_tgz=$(mktemp) + tar -T /dev/null -czf "$ccache_tgz" +fi + +docker_context_dir=$( + cd "$cur_dir/.." # Let the script find our fbcode_builder_config.py + "$cur_dir/make_docker_context.py" \ + --os-image "$os_image" \ + --gcc-version "$gcc_version" \ + --make-parallelism "$make_parallelism" \ + --local-repo-dir "$cur_dir/../.." \ + --ccache-tgz "$ccache_tgz" +) +cd "${docker_context_dir?Failed to make Docker context directory}" + +# Make it safe to iterate on the .sh in the tree while the script runs. +cp "$cur_dir/docker_build_with_ccache.sh" . +exec ./docker_build_with_ccache.sh \ + --build-timeout "$docker_build_timeout" \ + "$travis_cache_dir" diff --git a/build/fbcode_builder/utils.py b/build/fbcode_builder/utils.py new file mode 100644 index 0000000000..bdf7b01d52 --- /dev/null +++ b/build/fbcode_builder/utils.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +'Miscellaneous utility functions.' + +import itertools +import logging +import os +import shutil +import subprocess +import sys + +from contextlib import contextmanager + + +def recursively_flatten_list(l): + return itertools.chain.from_iterable( + (recursively_flatten_list(i) if type(i) is list else (i,)) + for i in l + ) + + +def run_command(*cmd, **kwargs): + 'The stdout of most fbcode_builder utilities is meant to be parsed.' + logging.debug('Running: {0} with {1}'.format(cmd, kwargs)) + kwargs['stdout'] = sys.stderr + subprocess.check_call(cmd, **kwargs) + + +@contextmanager +def make_temp_dir(d): + os.mkdir(d) + try: + yield d + finally: + shutil.rmtree(d, ignore_errors=True) + + +def _inner_read_config(path): + ''' + Helper to read a named config file. + The grossness with the global is a workaround for this python bug: + https://bugs.python.org/issue21591 + The bug prevents us from defining either a local function or a lambda + in the scope of read_fbcode_builder_config below. + ''' + global _project_dir + full_path = os.path.join(_project_dir, path) + return read_fbcode_builder_config(full_path) + + +def read_fbcode_builder_config(filename): + # Allow one spec to read another + # When doing so, treat paths as relative to the config's project directory. + # _project_dir is a "local" for _inner_read_config; see the comments + # in that function for an explanation of the use of global. + global _project_dir + _project_dir = os.path.dirname(filename) + + scope = {'read_fbcode_builder_config': _inner_read_config} + with open(filename) as config_file: + code = compile(config_file.read(), filename, mode='exec') + exec(code, scope) + return scope['config'] + + +def steps_for_spec(builder, spec, processed_modules=None): + ''' + Sets `builder` configuration, and returns all the builder steps + necessary to build `spec` and its dependencies. + + Traverses the dependencies in depth-first order, honoring the sequencing + in each 'depends_on' list. + ''' + if processed_modules is None: + processed_modules = set() + steps = [] + for module in spec.get('depends_on', []): + if module not in processed_modules: + processed_modules.add(module) + steps.extend(steps_for_spec( + builder, + module.fbcode_builder_spec(builder), + processed_modules + )) + steps.extend(spec.get('steps', [])) + return steps + + +def build_fbcode_builder_config(config): + return lambda builder: builder.build( + steps_for_spec(builder, config['fbcode_builder_spec'](builder)) + )