facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-04 22:07:44 +03:00
Code Issues Projects Releases Wiki Activity

fbcode_builder: CMake functions for building standalone python programs

Browse Source 
Summary:
Add some CMake functions for building standalone executables from Python
source files.  This generates executables similar to PEX
(https://github.com/pantsbuild/pex).

In the future this could potentially be leveraged to directly build XAR files
(https://github.com/facebookincubator/xar).

The main advantages of these functions is that they allow easily defining
"libraries" of python files, and their dependencies, which can then be used
and packaged as part of multiple different standalone executables.

Reviewed By: wez

Differential Revision: D16722499

fbshipit-source-id: e1d829b911dc428e5438b5cf9cebf99b3fb6ce24
This commit is contained in:
Adam Simpkins 2019-08-19 11:06:17 -07:00 committed by Facebook Github Bot
parent 4534e72815
commit 53f8ded39e
3 changed files with 875 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
build/fbcode_builder/CMake/FBCMakeParseArgs.cmake Normal file
View File

@ -0,0 +1,141 @@
#
# Copyright (c) Facebook, Inc. and its affiliates.
#
# Helper function for parsing arguments to a CMake function.
#
# This function is very similar to CMake's built-in cmake_parse_arguments()
# function, with some improvements:
# - This function correctly handles empty arguments. (cmake_parse_arguments()
# ignores empty arguments.)
# - If a multi-value argument is specified more than once, the subsequent
# arguments are appended to the original list rather than replacing it. e.g.
# if "SOURCES" is a multi-value argument, and the argument list contains
# "SOURCES a b c SOURCES x y z" then the resulting value for SOURCES will be
# "a;b;c;x;y;z" rather than "x;y;z"
# - This function errors out by default on unrecognized arguments. You can
# pass in an extra "ALLOW_UNPARSED_ARGS" argument to make it behave like
# cmake_parse_arguments(), and return the unparsed arguments in a
# <prefix>_UNPARSED_ARGUMENTS variable instead.
#
# It does look like cmake_parse_arguments() handled empty arguments correctly
# from CMake 3.0 through 3.3, but it seems like this was probably broken when
# it was turned into a built-in function in CMake 3.4. Here is discussion and
# patches that fixed this behavior prior to CMake 3.0:
# https://cmake.org/pipermail/cmake-developers/2013-November/020607.html
#
# The one downside to this function over the built-in cmake_parse_arguments()
# is that I don't think we can achieve the PARSE_ARGV behavior in a non-builtin
# function, so we can't properly handle arguments that contain ";". CMake will
# treat the ";" characters as list element separators, and treat it as multiple
# separate arguments.
#
function(fb_cmake_parse_args PREFIX OPTIONS ONE_VALUE_ARGS MULTI_VALUE_ARGS ARGS)
foreach(option IN LISTS ARGN)
if ("${option}" STREQUAL "ALLOW_UNPARSED_ARGS")
set(ALLOW_UNPARSED_ARGS TRUE)
else()
message(
FATAL_ERROR
"unknown optional argument for fb_cmake_parse_args(): ${option}"
)
endif()
endforeach()
# Define all options as FALSE in the parent scope to start with
foreach(var_name IN LISTS OPTIONS)
set("${PREFIX}_${var_name}" "FALSE" PARENT_SCOPE)
endforeach()
# TODO: We aren't extremely strict about error checking for one-value
# arguments here. e.g., we don't complain if a one-value argument is
# followed by another option/one-value/multi-value name rather than an
# argument. We also don't complain if a one-value argument is the last
# argument and isn't followed by a value.
list(APPEND all_args ${ONE_VALUE_ARGS})
list(APPEND all_args ${MULTI_VALUE_ARGS})
set(current_variable)
set(unparsed_args)
foreach(arg IN LISTS ARGS)
list(FIND OPTIONS "${arg}" opt_index)
if("${opt_index}" EQUAL -1)
list(FIND all_args "${arg}" arg_index)
if("${arg_index}" EQUAL -1)
# This argument does not match an argument name,
# must be an argument value
if("${current_variable}" STREQUAL "")
list(APPEND unparsed_args "${arg}")
else()
# Ugh, CMake lists have a pretty fundamental flaw: they cannot
# distinguish between an empty list and a list with a single empty
# element. We track our own SEEN_VALUES_arg setting to help
# distinguish this and behave properly here.
if ("${SEEN_${current_variable}}" AND "${${current_variable}}" STREQUAL "")
set("${current_variable}" ";${arg}")
else()
list(APPEND "${current_variable}" "${arg}")
endif()
set("SEEN_${current_variable}" TRUE)
endif()
else()
# We found a single- or multi-value argument name
set(current_variable "VALUES_${arg}")
set("SEEN_${arg}" TRUE)
endif()
else()
# We found an option variable
set("${PREFIX}_${arg}" "TRUE" PARENT_SCOPE)
set(current_variable)
endif()
endforeach()
foreach(arg_name IN LISTS ONE_VALUE_ARGS)
if(NOT "${SEEN_${arg_name}}")
unset("${PREFIX}_${arg_name}" PARENT_SCOPE)
elseif(NOT "${SEEN_VALUES_${arg_name}}")
# If the argument was seen but a value wasn't specified, error out.
# We require exactly one value to be specified.
message(
FATAL_ERROR "argument ${arg_name} was specified without a value"
)
else()
list(LENGTH "VALUES_${arg_name}" num_args)
if("${num_args}" EQUAL 0)
# We know an argument was specified and that we called list(APPEND).
# If CMake thinks the list is empty that means there is really a single
# empty element in the list.
set("${PREFIX}_${arg_name}" "" PARENT_SCOPE)
elseif("${num_args}" EQUAL 1)
list(GET "VALUES_${arg_name}" 0 arg_value)
set("${PREFIX}_${arg_name}" "${arg_value}" PARENT_SCOPE)
else()
message(
FATAL_ERROR "too many arguments specified for ${arg_name}: "
"${VALUES_${arg_name}}"
)
endif()
endif()
endforeach()
foreach(arg_name IN LISTS MULTI_VALUE_ARGS)
# If this argument name was never seen, then unset the parent scope
if (NOT "${SEEN_${arg_name}}")
unset("${PREFIX}_${arg_name}" PARENT_SCOPE)
else()
# TODO: Our caller still won't be able to distinguish between an empty
# list and a list with a single empty element. We can tell which is
# which, but CMake lists don't make it easy to show this to our caller.
set("${PREFIX}_${arg_name}" "${VALUES_${arg_name}}" PARENT_SCOPE)
endif()
endforeach()
# By default we fatal out on unparsed arguments, but return them to the
# caller if ALLOW_UNPARSED_ARGS was specified.
if (DEFINED unparsed_args)
if ("${ALLOW_UNPARSED_ARGS}")
set("${PREFIX}_UNPARSED_ARGUMENTS" "${unparsed_args}" PARENT_SCOPE)
else()
message(FATAL_ERROR "unrecognized arguments: ${unparsed_args}")
endif()
endif()
endfunction()

407
build/fbcode_builder/CMake/FBPythonBinary.cmake Normal file
View File

@ -0,0 +1,407 @@
# Copyright (c) Facebook, Inc. and its affiliates.
include(FBCMakeParseArgs)
#
# This file contains helper functions for building self-executing Python
# binaries.
#
# This is somewhat different than typical python installation with
# distutils/pip/virtualenv/etc. We primarily want to build a standalone
# executable, isolated from other Python packages on the system. We don't want
# to install files into the standard library python paths. This is more
# similar to PEX (https://github.com/pantsbuild/pex) and XAR
# (https://github.com/facebookincubator/xar). (In the future it would be nice
# to update this code to also support directly generating XAR files if XAR is
# available.)
#
# We also want to be able to easily define "libraries" of python files that can
# be shared and re-used between these standalone python executables, and can be
# shared across projects in different repositories. This means that we do need
# a way to "install" libraries so that they are visible to CMake builds in
# other repositories, without actually installing them in the standard python
# library paths.
#
# Find our helper program.
# We typically install this in the same directory as this .cmake file.
find_program(
MAKE_PYTHON_ARCHIVE "make_fbpy_archive.py"
PATHS ${CMAKE_MODULE_PATH}
)
if (NOT MAKE_PYTHON_ARCHIVE)
message(
FATAL_ERROR "unable to find make_fbpy_archive.py helper program (it "
"should be located in the same directory as FBPythonRules.cmake)"
)
endif()
# An option to control the default installation location for
# install_fb_python_library(). This is relative to ${CMAKE_INSTALL_PREFIX}
set(
FBPY_LIB_INSTALL_DIR "lib/fb-py-libs" CACHE STRING
"The subdirectory where FB python libraries should be installed"
)
#
# Build a self-executing python binary.
#
# This accepts the same arguments as add_fb_python_library().
# In addition, a MAIN_MODULE argument is required. This argument specifies
# which module should be started as the __main__ module when the executable is
# run.
#
function(add_fb_python_executable EXE_NAME)
# Parse the arguments
set(one_value_args BASE_DIR NAMESPACE MAIN_MODULE TYPE)
set(multi_value_args SOURCES DEPENDS)
fb_cmake_parse_args(
ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
)
fb_py_process_default_args(ARG_NAMESPACE ARG_BASE_DIR)
# Use add_fb_python_library() to perform most of our source handling
add_fb_python_library(
"${EXE_NAME}.main_lib"
BASE_DIR "${ARG_BASE_DIR}"
NAMESPACE "${ARG_NAMESPACE}"
SOURCES ${ARG_SOURCES}
DEPENDS ${ARG_DEPENDS}
)
set(
manifest_files
"$<TARGET_PROPERTY:${EXE_NAME}.main_lib.py_lib,INTERFACE_INCLUDE_DIRECTORIES>"
)
set(
source_files
"$<TARGET_PROPERTY:${EXE_NAME}.main_lib.py_lib,INTERFACE_SOURCES>"
)
# The command to build the executable archive.
#
# If we are using CMake 3.8+ we can use COMMAND_EXPAND_LISTS.
# CMP0067 isn't really the policy we care about, but seems like the best way
# to check if we are running 3.8+.
if (POLICY CMP0067)
set(extra_cmd_params COMMAND_EXPAND_LISTS)
set(make_py_args "${manifest_files}")
else()
set(extra_cmd_params)
set(make_py_args --manifest-separator "::" "$<JOIN:${manifest_files},::>")
endif()
set(output_file "${EXE_NAME}")
if(DEFINED ARG_TYPE)
list(APPEND make_py_args "--type" "${ARG_TYPE}")
if ("${ARG_TYPE}" STREQUAL "dir")
# CMake doesn't really seem to like having a directory specified as an
# output; specify the __main__.py file as the output instead.
set(output_file "${EXE_NAME}/__main__.py")
list(APPEND
extra_cmd_params
COMMAND "${CMAKE_COMMAND}" -E remove_directory "${EXE_NAME}"
)
endif()
endif()
add_custom_command(
OUTPUT "${output_file}"
${extra_cmd_params}
COMMAND
"${MAKE_PYTHON_ARCHIVE}" -o "${EXE_NAME}" --main "${ARG_MAIN_MODULE}"
${make_py_args}
DEPENDS
${source_files}
"${EXE_NAME}.main_lib.py_sources_built"
"${MAKE_PYTHON_ARCHIVE}"
)
# Add an "ALL" target that depends on force ${EXE_NAME},
# so that ${EXE_NAME} will be included in the default list of build targets.
add_custom_target("${EXE_NAME}.GEN_PY_EXE" ALL DEPENDS "${output_file}")
endfunction()
#
# Define a python library.
#
# If you want to install a python library generated from this rule note that
# you need to use install_fb_python_library() rather than CMake's built-in
# install() function. This will make it available for other downstream
# projects to use in their add_fb_python_executable() and
# add_fb_python_library() calls. (You do still need to use `install(EXPORT)`
# later to install the CMake exports.)
#
# Parameters:
# - BASE_DIR <dir>:
# The base directory path to strip off from each source path. All source
# files must be inside this directory. If not specified it defaults to
# ${CMAKE_CURRENT_SOURCE_DIR}.
# - NAMESPACE <namespace>:
# The destination namespace where these files should be installed in python
# binaries. If not specified, this defaults to the current relative path of
# ${CMAKE_CURRENT_SOURCE_DIR} inside ${CMAKE_SOURCE_DIR}. e.g., a python
# library defined in the directory repo_root/foo/bar will use a default
# namespace of "foo.bar"
# - SOURCES <src1> <...>:
# The python source files.
# - DEPENDS <target1> <...>:
# Other python libraries that this one depends on.
# - INSTALL_DIR <dir>:
# The directory where this library should be installed.
# install_fb_python_library() must still be called later to perform the
# installation. If a relative path is given it will be treated relative to
# ${CMAKE_INSTALL_PREFIX}
#
# CMake is unfortunately pretty crappy at being able to define custom build
# rules & behaviors. It doesn't support transitive property propagation
# between custom targets; only the built-in add_executable() and add_library()
# targets support transitive properties.
#
# We hack around this janky CMake behavior by (ab)using interface libraries to
# propagate some of the data we want between targets, without actually
# generating a C library.
#
# add_fb_python_library(SOMELIB) generates the following things:
# - An INTERFACE library rule named SOMELIB.py_lib which tracks some
# information about transitive dependencies:
# - the transitive set of source files in the INTERFACE_SOURCES property
# - the transitive set of manifest files that this library depends on in
# the INTERFACE_INCLUDE_DIRECTORIES property.
# - A custom command that generates a SOMELIB.manifest file.
# This file contains the mapping of source files to desired destination
# locations in executables that depend on this library. This manifest file
# will then be read at build-time in order to build executables.
#
function(add_fb_python_library LIB_NAME)
# Parse the arguments
# We use fb_cmake_parse_args() rather than cmake_parse_arguments() since
# cmake_parse_arguments() does not handle empty arguments, and it is common
# for callers to want to specify an empty NAMESPACE parameter.
set(one_value_args BASE_DIR NAMESPACE INSTALL_DIR)
set(multi_value_args SOURCES DEPENDS)
fb_cmake_parse_args(
ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
)
fb_py_process_default_args(ARG_NAMESPACE ARG_BASE_DIR)
string(REPLACE "." "/" namespace_dir "${ARG_NAMESPACE}")
if (NOT "${namespace_dir}" STREQUAL "")
set(namespace_dir "${namespace_dir}/")
endif()
if(NOT DEFINED ARG_INSTALL_DIR)
set(install_dir "${FBPY_LIB_INSTALL_DIR}/")
elseif("${ARG_INSTALL_DIR}" STREQUAL "")
set(install_dir "")
else()
set(install_dir "${ARG_INSTALL_DIR}/")
endif()
# message(STATUS "fb py library ${LIB_NAME}: "
# "NS=${namespace_dir} BASE=${ARG_BASE_DIR}")
# TODO: In the future it would be nice to support pre-compiling the source
# files. We could emit a rule to compile each source file and emit a
# .pyc/.pyo file here, and then have the manifest reference the pyc/pyo
# files.
# Define a library target to help pass around information about the library,
# and propagate dependency information.
#
# CMake make a lot of assumptions that libraries are C++ libraries. To help
# avoid confusion we name our target "${LIB_NAME}.py_lib" rather than just
# "${LIB_NAME}". This helps avoid confusion if callers try to use
# "${LIB_NAME}" on their own as a target name. (e.g., attempting to install
# it directly with install(TARGETS) won't work. Callers must use
# install_fb_python_library() instead.)
add_library("${LIB_NAME}.py_lib" INTERFACE)
# Emit the manifest file.
#
# We write the manifest file to a temporary path first, then copy it with
# configure_file(COPYONLY). This is necessary to get CMake to understand
# that "${manifest_path}" is generated by the CMake configure phase,
# and allow using it as a dependency for add_custom_command().
# (https://gitlab.kitware.com/cmake/cmake/issues/16367)
set(manifest_path "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}.manifest")
set(tmp_manifest "${manifest_path}.tmp")
file(WRITE "${tmp_manifest}" "FBPY_MANIFEST 1\n")
set(abs_sources)
foreach(src_path IN LISTS ARG_SOURCES)
get_filename_component(abs_source "${src_path}" ABSOLUTE)
list(APPEND abs_sources "${abs_source}")
file(RELATIVE_PATH rel_src "${ARG_BASE_DIR}" "${abs_source}")
target_sources(
"${LIB_NAME}.py_lib" INTERFACE
"$<BUILD_INTERFACE:${abs_source}>"
"$<INSTALL_INTERFACE:${install_dir}${LIB_NAME}/${namespace_dir}${rel_src}>"
)
if("${rel_src}" MATCHES "^../")
message(
FATAL_ERROR "${LIB_NAME}: source file \"${abs_source}\" is not inside "
"the base directory ${ARG_BASE_DIR}"
)
endif()
file(
APPEND "${tmp_manifest}"
"${abs_source} :: ${namespace_dir}${rel_src}\n"
)
endforeach()
configure_file("${tmp_manifest}" "${manifest_path}" COPYONLY)
target_include_directories(
"${LIB_NAME}.py_lib" INTERFACE
"$<BUILD_INTERFACE:${manifest_path}>"
"$<INSTALL_INTERFACE:${install_dir}${LIB_NAME}.manifest>"
)
# Add a target that depends on all of the source files.
# This is needed in case some of the source files are generated. This will
# ensure that these source files are brought up-to-date before we build
# any python binaries that depend on this library.
add_custom_target("${LIB_NAME}.py_sources_built" DEPENDS ${ARG_SOURCES})
add_dependencies("${LIB_NAME}.py_lib" "${LIB_NAME}.py_sources_built")
# Hook up library dependencies, and also make the *.py_sources_built target
# depend on the sources for all of our dependencies also being up-to-date.
foreach(dep IN LISTS ARG_DEPENDS)
target_link_libraries("${LIB_NAME}.py_lib" INTERFACE "${dep}.py_lib")
# Mark that our .py_sources_built target depends on each our our dependent
# libraries. This serves two functions:
# - This causes CMake to generate an error message if one of the
# dependencies is never defined. The target_link_libraries() call above
# won't complain if one of the dependencies doesn't exist (since it is
# intended to allow passing in file names for plain library files rather
# than just targets).
# - It ensures that sources for our depencencies are built before any
# executable that depends on us. Note that we depend on "${dep}.py_lib"
# rather than "${dep}.py_sources_built" for this purpose because the
# ".py_sources_built" target won't be available for imported targets.
add_dependencies("${LIB_NAME}.py_sources_built" "${dep}.py_lib")
endforeach()
# Add a custom command to help with library installation, in case
# install_fb_python_library() is called later for this library.
# add_custom_command() only works with file dependencies defined in the same
# CMakeLists.txt file, so we want to make sure this is defined here, rather
# then where install_fb_python_library() is called.
# This command won't be run by default, but will only be run if it is needed
# by a subsequent install_fb_python_library() call.
#
# This command copies the library contents into the build directory.
# It would be nicer if we could skip this intermediate copy, and just run
# make_fbpy_archive.py at install time to copy them directly to the desired
# installation directory. Unfortunately this is difficult to do, and seems
# to interfere with some of the CMake code that wants to generate a manifest
# of installed files.
set(build_install_dir "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}.lib_install")
add_custom_command(
OUTPUT
"${build_install_dir}/${LIB_NAME}.manifest"
COMMAND "${CMAKE_COMMAND}" -E remove_directory "${build_install_dir}"
COMMAND
"${MAKE_PYTHON_ARCHIVE}" --type lib-install
--install-dir "${LIB_NAME}"
-o "${build_install_dir}/${LIB_NAME}" "${manifest_path}"
DEPENDS
"${ARG_SOURCES}"
"${manifest_path}"
"${MAKE_PYTHON_ARCHIVE}"
)
add_custom_target(
"${LIB_NAME}.py_lib_install"
DEPENDS "${build_install_dir}/${LIB_NAME}.manifest"
)
# Set some properties to pass through the install paths to
# install_fb_python_library()
#
# Passing through ${build_install_dir} allows install_fb_python_library()
# to work even if used from a different CMakeLists.txt file than where
# add_fb_python_library() was called (i.e. such that
# ${CMAKE_CURRENT_BINARY_DIR} is different between the two calls).
set(abs_install_dir "${install_dir}")
if(NOT IS_ABSOLUTE "${abs_install_dir}")
set(abs_install_dir "${CMAKE_INSTALL_PREFIX}/${abs_install_dir}")
endif()
set_target_properties(
"${LIB_NAME}.py_lib_install"
PROPERTIES
INSTALL_DIR "${abs_install_dir}"
BUILD_INSTALL_DIR "${build_install_dir}"
)
endfunction()
#
# Install a python library.
#
# - EXPORT <export-name>:
# Associate the installed target files with the given export-name.
#
# Note that unlike the built-in CMake install() function we do not accept a
# DESTINATION parameter. Instead, use the INSTALL_DIR parameter to
# add_fb_python_library() to set the installation location.
#
function(install_fb_python_library LIB_NAME)
set(one_value_args EXPORT)
fb_cmake_parse_args(ARG "" "${one_value_args}" "" "${ARGN}")
# Export our "${LIB_NAME}.py_lib" target so that it will be available to
# downstream projects in our installed CMake config files.
if(DEFINED ARG_EXPORT)
install(TARGETS "${LIB_NAME}.py_lib" EXPORT "${ARG_EXPORT}")
endif()
# add_fb_python_library() emits a .py_lib_install target that will prepare
# the installation directory. However, it isn't part of the "ALL" target and
# therefore isn't built by default.
#
# Make sure the ALL target depends on it now. We have to do this by
# introducing yet another custom target.
# Add it as a dependency to the ALL target now.
add_custom_target("${LIB_NAME}.py_lib_install_all" ALL)
add_dependencies(
"${LIB_NAME}.py_lib_install_all" "${LIB_NAME}.py_lib_install"
)
# Copy the intermediate install directory generated at build time into
# the desired install location.
set(dest_dir "$<TARGET_PROPERTY:${LIB_NAME}.py_lib_install,INSTALL_DIR>")
set(
build_install_dir
"$<TARGET_PROPERTY:${LIB_NAME}.py_lib_install,BUILD_INSTALL_DIR>"
)
install(
DIRECTORY "${build_install_dir}/${LIB_NAME}"
DESTINATION "${dest_dir}"
)
install(
FILES "${build_install_dir}/${LIB_NAME}.manifest"
DESTINATION "${dest_dir}"
)
endfunction()
# Helper macro to process the BASE_DIR and NAMESPACE arguments for
# add_fb_python_executable() and add_fb_python_executable()
macro(fb_py_process_default_args NAMESPACE_VAR BASE_DIR_VAR)
# If the namespace was not specified, default to the relative path to the
# current directory (starting from the repository root).
if(NOT DEFINED "${NAMESPACE_VAR}")
file(
RELATIVE_PATH "${NAMESPACE_VAR}"
"${CMAKE_SOURCE_DIR}"
"${CMAKE_CURRENT_SOURCE_DIR}"
)
endif()
if(NOT DEFINED "${BASE_DIR_VAR}")
# If the base directory was not specified, default to the current directory
set("${BASE_DIR_VAR}" "${CMAKE_CURRENT_SOURCE_DIR}")
else()
# If the base directory was specified, always convert it to an
# absolute path.
get_filename_component("${BASE_DIR_VAR}" "${${BASE_DIR_VAR}}" ABSOLUTE)
endif()
endmacro()

327
build/fbcode_builder/CMake/make_fbpy_archive.py Executable file
View File

@ -0,0 +1,327 @@
#!/usr/bin/env python3
#
# Copyright (c) Facebook, Inc. and its affiliates.
#
import argparse
import collections
import errno
import os
import shutil
import sys
import tempfile
import zipapp
MANIFEST_SEPARATOR = " :: "
MANIFEST_HEADER_V1 = "FBPY_MANIFEST 1\n"
class UsageError(Exception):
def __init__(self, message):
self.message = message
def __str__(self):
return self.message
class BadManifestError(UsageError):
def __init__(self, path, line_num, message):
full_msg = "%s:%s: %s" % (path, line_num, message)
super().__init__(full_msg)
self.path = path
self.line_num = line_num
self.raw_message = message
PathInfo = collections.namedtuple(
"PathInfo", ("src", "dest", "manifest_path", "manifest_line")
)
def parse_manifest(manifest, path_map):
bad_prefix = ".." + os.path.sep
manifest_dir = os.path.dirname(manifest)
with open(manifest, "r") as f:
line_num = 1
line = f.readline()
if line != MANIFEST_HEADER_V1:
raise BadManifestError(
manifest, line_num, "Unexpected manifest file header"
)
for line in f:
line_num += 1
if line.startswith("#"):
continue
line = line.rstrip("\n")
parts = line.split(MANIFEST_SEPARATOR)
if len(parts) != 2:
msg = "line must be of the form SRC %s DEST" % MANIFEST_SEPARATOR
raise BadManifestError(manifest, line_num, msg)
src, dest = parts
dest = os.path.normpath(dest)
if dest.startswith(bad_prefix):
msg = "destination path starts with %s: %s" % (bad_prefix, dest)
raise BadManifestError(manifest, line_num, msg)
if not os.path.isabs(src):
src = os.path.normpath(os.path.join(manifest_dir, src))
if dest in path_map:
prev_info = path_map[dest]
msg = (
"multiple source paths specified for destination "
"path %s. Previous source was %s from %s:%s"
% (
dest,
prev_info.src,
prev_info.manifest_path,
prev_info.manifest_line,
)
)
raise BadManifestError(manifest, line_num, msg)
info = PathInfo(
src=src,
dest=dest,
manifest_path=manifest,
manifest_line=line_num,
)
path_map[dest] = info
def populate_install_tree(inst_dir, path_map):
os.mkdir(inst_dir)
dest_dirs = {"": False}
def make_dest_dir(path):
if path in dest_dirs:
return
parent = os.path.dirname(path)
make_dest_dir(parent)
abs_path = os.path.join(inst_dir, path)
os.mkdir(abs_path)
dest_dirs[path] = False
def install_file(info):
dir_name, base_name = os.path.split(info.dest)
make_dest_dir(dir_name)
if base_name == "__init__.py":
dest_dirs[dir_name] = True
abs_dest = os.path.join(inst_dir, info.dest)
shutil.copy2(info.src, abs_dest)
# Copy all of the destination files
for info in path_map.values():
install_file(info)
# Create __init__ files in any directories that don't have them.
for dir_path, has_init in dest_dirs.items():
if has_init:
continue
init_path = os.path.join(inst_dir, dir_path, "__init__.py")
with open(init_path, "w"):
pass
def build_zipapp(args, path_map):
""" Create a self executing python binary using Python 3's built-in
zipapp module.
This type of Python binary is relatively simple, as zipapp is part of the
standard library, but it does not support native language extensions
(.so/.dll files).
"""
dest_dir = os.path.dirname(args.output)
with tempfile.TemporaryDirectory(prefix="make_fbpy.", dir=dest_dir) as tmpdir:
inst_dir = os.path.join(tmpdir, "tree")
populate_install_tree(inst_dir, path_map)
tmp_output = os.path.join(tmpdir, "output.exe")
zipapp.create_archive(
inst_dir, target=tmp_output, interpreter=args.python, main=args.main
)
os.rename(tmp_output, args.output)
def create_main_module(args, inst_dir, path_map):
if not args.main:
assert "__main__.py" in path_map
return
dest_path = os.path.join(inst_dir, "__main__.py")
main_module, main_fn = args.main.split(":")
main_contents = """\
#!{python}
if __name__ == "__main__":
import {main_module}
{main_module}.{main_fn}()
""".format(
python=args.python, main_module=main_module, main_fn=main_fn
)
with open(dest_path, "w") as f:
f.write(main_contents)
os.chmod(dest_path, 0o755)
def build_install_dir(args, path_map):
""" Create a directory that contains all of the sources, with a __main__
module to run the program.
"""
# Populate a temporary directory first, then rename to the destination
# location. This ensures that we don't ever leave a halfway-built
# directory behind at the output path if something goes wrong.
dest_dir = os.path.dirname(args.output)
with tempfile.TemporaryDirectory(prefix="make_fbpy.", dir=dest_dir) as tmpdir:
inst_dir = os.path.join(tmpdir, "tree")
populate_install_tree(inst_dir, path_map)
create_main_module(args, inst_dir, path_map)
os.rename(inst_dir, args.output)
def ensure_directory(path):
try:
os.makedirs(path)
except OSError as ex:
if ex.errno != errno.EEXIST:
raise
def install_library(args, path_map):
""" Create an installation directory a python library. """
out_dir = args.output
out_manifest = args.output + ".manifest"
install_dir = args.install_dir
if not install_dir:
install_dir = out_dir
os.makedirs(out_dir)
with open(out_manifest, "w") as manifest:
manifest.write(MANIFEST_HEADER_V1)
for info in path_map.values():
abs_dest = os.path.join(out_dir, info.dest)
ensure_directory(os.path.dirname(abs_dest))
print("copy %r --> %r" % (info.src, abs_dest))
shutil.copy2(info.src, abs_dest)
installed_dest = os.path.join(install_dir, info.dest)
manifest.write("%s%s%s\n" % (installed_dest, MANIFEST_SEPARATOR, info.dest))
def parse_manifests(args):
# Process args.manifest_separator to help support older versions of CMake
if args.manifest_separator:
manifests = []
for manifest_arg in args.manifests:
split_arg = manifest_arg.split(args.manifest_separator)
manifests.extend(split_arg)
args.manifests = manifests
path_map = {}
for manifest in args.manifests:
parse_manifest(manifest, path_map)
return path_map
def check_main_module(args, path_map):
# Translate an empty string in the --main argument to None,
# just to allow the CMake logic to be slightly simpler and pass in an
# empty string when it really wants the default __main__.py module to be
# used.
if args.main == "":
args.main = None
if args.type == "lib-install":
if args.main is not None:
raise UsageError("cannot specify a --main argument with --type=lib-install")
return
main_info = path_map.get("__main__.py")
if args.main:
if main_info is not None:
msg = (
"specified an explicit main module with --main, "
"but the file listing already includes __main__.py"
)
raise BadManifestError(
main_info.manifest_path, main_info.manifest_line, msg
)
parts = args.main.split(":")
if len(parts) != 2:
raise UsageError(
"argument to --main must be of the form MODULE:CALLABLE "
"(received %s)" % (args.main,)
)
else:
if main_info is None:
raise UsageError(
"no main module specified with --main, "
"and no __main__.py module present"
)
BUILD_TYPES = {
"zipapp": build_zipapp,
"dir": build_install_dir,
"lib-install": install_library,
}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("-o", "--output", required=True, help="The output file path")
ap.add_argument(
"--install-dir",
help="When used with --type=lib-install, this parameter specifies the "
"final location where the library where be installed. This can be "
"used to generate the library in one directory first, when you plan "
"to move or copy it to another final location later.",
)
ap.add_argument(
"--manifest-separator",
help="Split manifest arguments around this separator. This is used "
"to support older versions of CMake that cannot supply the manifests "
"as separate arguments.",
)
ap.add_argument(
"--main",
help="The main module to run, specified as <module>:<callable>. "
"This must be specified if and only if the archive does not contain "
"a __main__.py file.",
)
ap.add_argument(
"--python",
help="Explicitly specify the python interpreter to use for the " "executable.",
)
ap.add_argument(
"--type", choices=BUILD_TYPES.keys(), help="The type of output to build."
)
ap.add_argument(
"manifests",
nargs="+",
help="The manifest files specifying how to construct the archive",
)
args = ap.parse_args()
if args.python is None:
args.python = sys.executable
if args.type is None:
# In the future we might want different default output types
# for different platforms.
args.type = "zipapp"
build_fn = BUILD_TYPES[args.type]
try:
path_map = parse_manifests(args)
check_main_module(args, path_map)
except UsageError as ex:
print("error: %s" % (ex,), file=sys.stderr)
sys.exit(1)
build_fn(args, path_map)
if __name__ == "__main__":
main()