facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-10 16:57:49 +03:00
Code Issues Projects Releases Wiki Activity
c578ffc214
sapling/setup.py
Augie Fackler bdaa8cfe88 setup: fix globbing on windows so modules get detected 
Differential Revision: https://phab.mercurial-scm.org/D1535
2017-11-28 17:57:34 -05:00

553 lines
18 KiB
Python
Raw Blame History

from distutils.version import LooseVersion
from distutils.cmd import Command
from distutils.core import setup, Extension
import distutils
from distutils_rust import RustExtension, BuildRustExt
import fnmatch
from glob import glob
import os, shutil, sys
iswindows = os.name == 'nt'
WERROR = "/WX" if iswindows else "-Werror"
WSTRICTPROTOTYPES = None if iswindows else "-Werror=strict-prototypes"
WALL = "/Wall" if iswindows else "-Wall"
STDC99 = "" if iswindows else "-std=c99"
STDCPP0X = "" if iswindows else "-std=c++0x"
WEXTRA = "" if iswindows else "-Wextra"
WCONVERSION = "" if iswindows else "-Wconversion"
PEDANTIC = "" if iswindows else "-pedantic"
SHA1LIB_DEFINE = "/DSHA1_USE_SHA1DC" if iswindows else "-DSHA1_USE_SHA1DC"
SHA1_LIBRARY = "sha1detectcoll"
NOOPTIMIZATION = "/Od" if iswindows else "-O0"
OPTIMIZATION = "" if iswindows else "-O2"
PRODUCEDEBUGSYMBOLS = "/DEBUG:FULL" if iswindows else "-g"
if 'USERUST' in os.environ:
USERUST = int(os.environ['USERUST'])
else:
import subprocess
USERUST = False
try:
cargo_version = subprocess.check_output(
['cargo', '--version']).split()[1]
except Exception:
sys.stderr.write(
"not compiling Rust extensions: cargo is not available\n")
else:
required_cargo_version = '0.21'
if (LooseVersion(cargo_version) >=
LooseVersion(required_cargo_version)):
USERUST = True
else:
sys.stderr.write(
"not compiling Rust extensions: cargo is too old " +
"(found %s, need %s or higher)\n"
% (cargo_version, required_cargo_version))
# whether to use Cython to recompile .pyx to .c/.cpp at build time.
# if False, fallback to .c/.cpp in the repo and .pyx files are ignored.
# if True, re-compile .c/.cpp from .pyx files, require cython at build time.
if 'USECYTHON' in os.environ:
USECYTHON = int(os.environ['USECYTHON'])
else:
try:
import Cython
except ImportError:
USECYTHON = False
else:
USECYTHON = (LooseVersion(Cython.__version__) >= LooseVersion('0.22'))
# --component allows the caller to specify what components they want. We can't
# use argparse like normal, because setup() at the bottom has it's own argument
# logic.
components = []
args = []
skip = False
for i, arg in enumerate(sys.argv):
if skip:
skip = False
continue
if arg == '--component' and len(sys.argv) > i + 1:
components.extend(sys.argv[i + 1].split(','))
skip = True
else:
args.append(arg)
sys.argv = args
cflags = [SHA1LIB_DEFINE]
# if this is set, compile all C extensions with -O0 -g for easy debugging. note
# that this is not manifested in any way in the Makefile dependencies.
# therefore, if you already have build products, they won't be rebuilt!
if os.getenv('FB_HGEXT_CDEBUG') is not None:
cflags.extend([NOOPTIMIZATION, PRODUCEDEBUGSYMBOLS])
else:
cflags.append(WERROR)
def get_env_path_list(var_name, default=None):
'''Get a path list from an environment variable. The variable is parsed as
a colon-separated list.'''
value = os.environ.get(var_name)
if not value:
return default
return value.split(os.path.pathsep)
include_dirs = get_env_path_list('INCLUDE_DIRS')
library_dirs = get_env_path_list('LIBRARY_DIRS')
def filter_existing_dirs(dirs):
'''Filters the given list and keeps only existing directory names.'''
return [d for d in dirs if os.path.isdir(d)]
# Historical default values.
# We should perhaps clean these up in the future after verifying that it
# doesn't break the build on any platforms.
#
# The /usr/local/* directories shouldn't actually be needed--the compiler
# should already use these directories when appropriate (e.g., if we are
# using the standard system compiler that has them in its default paths).
#
# The /opt/local paths may be necessary on Darwin builds.
if include_dirs is None:
if iswindows:
include_dirs = []
else:
include_dirs = filter_existing_dirs([
'/usr/local/include',
'/opt/local/include',
'/opt/homebrew/include/',
])
def distutils_dir_name(dname):
"""Returns the name of a distutils build directory"""
f = "{dirname}.{platform}-{version}"
return f.format(dirname=dname,
platform=distutils.util.get_platform(),
version=sys.version[:3])
if library_dirs is None:
if iswindows:
library_dirs = []
else:
library_dirs = filter_existing_dirs([
'/usr/local/lib',
'/opt/local/lib',
'/opt/homebrew/lib/',
])
library_dirs.append('build/' + distutils_dir_name('lib'))
# Override the default c static library building code in distutils since it
# doesn't pass enough args, like libraries and extra args.
import distutils.command.build_clib
from distutils.errors import DistutilsSetupError
def build_libraries(self, libraries):
for (lib_name, build_info) in libraries:
sources = build_info.get('sources')
if sources is None or not isinstance(sources, (list, tuple)):
raise DistutilsSetupError(
"in 'libraries' option (library '%s'), " +
"'sources' must be present and must be " +
"a list of source filenames") % lib_name
sources = list(sources)
# First, compile the source code to object files in the library
# directory. (This should probably change to putting object
# files in a temporary build directory.)
macros = build_info.get('macros')
include_dirs = build_info.get('include_dirs')
extra_args = build_info.get('extra_args')
objects = self.compiler.compile(sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_args)
# Now "link" the object files together into a static library.
# (On Unix at least, this isn't really linking -- it just
# builds an archive. Whatever.)
libraries = build_info.get('libraries', [])
for lib in libraries:
self.compiler.add_library(lib)
self.compiler.create_static_lib(objects, lib_name,
output_dir=self.build_clib,
debug=self.debug)
distutils.command.build_clib.build_clib.build_libraries = build_libraries
# Static c libaries
if iswindows:
availablelibraries = {}
else:
availablelibraries = {
'datapack': {
"sources" : ["cdatapack/cdatapack.c"],
"include_dirs" : ["."] + include_dirs,
"libraries" : ["lz4", SHA1_LIBRARY],
"extra_args" : filter(None,
[STDC99, WALL, WERROR, WSTRICTPROTOTYPES] + cflags),
},
'mpatch': {
"sources": ["cstore/mpatch.c"],
"include_dirs" : ["."] + include_dirs,
},
"sha1detectcoll": {
"sources" : [
"third-party/sha1dc/sha1.c",
"third-party/sha1dc/ubc_check.c",
],
"include_dirs" : ["third-party"] + include_dirs,
"extra_args" : filter(None,
[STDC99, WALL, WERROR, WSTRICTPROTOTYPES] + cflags),
},
}
# modules that are single files in hgext3rd
hgext3rd = [
p[:-3].replace('\\', '/').replace('/', '.')
for p in glob('hgext3rd/*.py')
if p != 'hgext3rd/__init__.py'
]
# packages that are directories in hgext3rd
hgext3rdpkgs = [
p[:-12].replace('\\', '/').replace('/', '.')
for p in glob('hgext3rd/*/__init__.py')
]
availablepymodules = hgext3rd
availablepackages = hgext3rdpkgs + [
'infinitepush',
'phabricator',
'remotefilelog',
]
if iswindows:
availablepackages += [
'linelog',
]
else:
availablepackages += [
'fastmanifest',
'treemanifest',
'linelog',
]
def distutils_dir_name(dname):
"""Returns the name of a distutils build directory"""
f = "{dirname}.{platform}-{version}"
return f.format(dirname=dname,
platform=distutils.util.get_platform(),
version=sys.version[:3])
if iswindows:
# The modules that successfully compile on Windows
availableextmodules = {
'linelog' : [
Extension('linelog',
sources=['linelog/pyext/linelog.pyx'],
extra_compile_args=filter(None, [
STDC99, WALL, WEXTRA, WCONVERSION, PEDANTIC,
]),
),
],
}
else:
availableextmodules = {
'cstore' : [
Extension('cstore',
sources=[
'cstore/datapackstore.cpp',
'cstore/deltachain.cpp',
'cstore/py-cstore.cpp',
'cstore/pythonutil.cpp',
'cstore/pythondatastore.cpp',
'cstore/uniondatapackstore.cpp',
'ctreemanifest/manifest.cpp',
'ctreemanifest/manifest_entry.cpp',
'ctreemanifest/manifest_fetcher.cpp',
'ctreemanifest/manifest_ptr.cpp',
'ctreemanifest/treemanifest.cpp',
],
include_dirs=[
'.',
'third-party',
] + include_dirs,
library_dirs=[
'build/' + distutils_dir_name('lib'),
] + library_dirs,
libraries=[
'datapack',
'lz4',
'mpatch',
SHA1_LIBRARY,
],
extra_compile_args=filter(None, [STDCPP0X, WALL] + cflags),
),
],
'cfastmanifest' : [
Extension('cfastmanifest',
sources=['cfastmanifest.c',
'cfastmanifest/bsearch.c',
'clib/buffer.c',
'cfastmanifest/checksum.c',
'cfastmanifest/node.c',
'cfastmanifest/tree.c',
'cfastmanifest/tree_arena.c',
'cfastmanifest/tree_convert.c',
'cfastmanifest/tree_copy.c',
'cfastmanifest/tree_diff.c',
'cfastmanifest/tree_disk.c',
'cfastmanifest/tree_iterator.c',
'cfastmanifest/tree_path.c',
],
include_dirs=[
'.',
'cfastmanifest',
'clib',
'third-party',
] + include_dirs,
library_dirs=library_dirs,
libraries=[SHA1_LIBRARY],
extra_compile_args=filter(None, [
STDC99,
WALL,
WSTRICTPROTOTYPES,
] + cflags),
),
],
'clindex': [
Extension('hgext3rd.clindex',
sources=['hgext3rd/clindex.pyx'],
include_dirs=['hgext3rd'],
extra_compile_args=filter(None, [
STDC99, WALL, WEXTRA, WCONVERSION, PEDANTIC,
]),
),
],
'linelog' : [
Extension('linelog',
sources=['linelog/pyext/linelog.pyx'],
extra_compile_args=filter(None, [
STDC99, WALL, WEXTRA, WCONVERSION, PEDANTIC,
]),
),
],
'patchrmdir': [
Extension('hgext3rd.patchrmdir',
sources=['hgext3rd/patchrmdir.pyx'],
extra_compile_args=filter(None, [
STDC99, WALL, WEXTRA, WCONVERSION, PEDANTIC,
]),
),
],
'traceprof': [
Extension('hgext3rd.traceprof',
sources=['hgext3rd/traceprof.pyx'],
include_dirs=['hgext3rd'],
extra_compile_args=filter(None, [
OPTIMIZATION, STDCPP0X, WALL, WEXTRA, WCONVERSION, PEDANTIC,
PRODUCEDEBUGSYMBOLS
]),
),
],
}
allnames = availablepackages + availableextmodules.keys() + availablepymodules
COMPONENTS = sorted(name.split('.')[-1] for name in allnames)
if not components:
components = COMPONENTS
dependencies = {
'absorb' : ['linelog'],
'cstore' : ['ctreemanifest', 'cdatapack'],
'fastannotate' : ['linelog'],
'infinitepush' : ['extutil'],
'remotefilelog' : ['cstore', 'extutil'],
'treemanifest' : ['cstore'],
}
processdep = True
while processdep:
processdep = False
for name, deps in dependencies.iteritems():
if name in components:
for dep in deps:
if dep not in components:
components.append(dep)
processdep = True
if iswindows:
# The modules that successfully compile on Windows
cythonmodules = ['linelog']
else:
cythonmodules = [
'clindex',
'linelog',
'patchrmdir',
'traceprof',
]
if USECYTHON:
# see http://cython.readthedocs.io/en/latest/src/reference/compilation.html
compileroptions = {
'unraisable_tracebacks': False,
'c_string_type': 'bytes',
}
for cythonmodule in cythonmodules:
if cythonmodule in components:
module = availableextmodules[cythonmodule]
try:
from Cython.Build import cythonize
availableextmodules[cythonmodule] = cythonize(
module,
compiler_directives=compileroptions,
)
except Exception:
# ImportError or Cython.Compiler.Errors.CompileError
sys.stderr.write(
'+------------------------------------------------+\n'
'| Failed to run cythonize. |\n'
'| Make sure you have Cython >= 0.21.1 installed. |\n'
'+------------------------------------------------+\n')
raise SystemExit(255)
else:
# use prebuilt files under prebuilt/cython
# change module sources from .pyx to .c or .cpp files
for cythonmodule in cythonmodules:
for m in availableextmodules[cythonmodule]:
sources = m.sources
iscpp = 'c++' in open(sources[0]).readline()
ext = iscpp and '.cpp' or '.c'
dstpaths = []
for src in sources:
dst = src.replace('.pyx', ext)
dstpaths.append(dst)
shutil.copy(os.path.join('prebuilt', 'cython',
os.path.basename(dst)), dst)
m.sources = dstpaths
packages = []
for package in availablepackages:
if package.split('.')[-1] in components:
packages.append(package)
librarynames = set()
ext_modules = []
for ext_module in availableextmodules:
if ext_module in components:
modules = availableextmodules[ext_module]
ext_modules.extend(modules)
librarynames.update(l for m in modules for l in m.libraries)
libraries = [(n, availablelibraries[n])
for n in librarynames if n in availablelibraries]
# Dependencies between our native libraries means we need to build in order
ext_order = {
'libdatapack' : 0,
'cstore' : 3,
}
ext_modules = sorted(ext_modules, key=lambda k: ext_order.get(k.name, 999))
requires = []
requireslz4 = ['remotefilelog', 'cdatapack']
if any(c for c in components if c in requireslz4):
requires.append('lz4')
if 'phabricator' in components:
requires.append('urllib3')
py_modules = []
for module in availablepymodules:
if module.split('.')[-1] in components:
py_modules.append(module)
# Extra clean command cleaning up non-Python extensions
class CleanExtCommand(Command):
description = 'remove extra build files'
user_options = []
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
root = os.path.dirname(os.path.abspath(__file__))
os.chdir(root)
# removed counter (ext: count)
removed = {}
def removepath(path):
try:
os.unlink(path)
except OSError: # ENOENT
pass
else:
ext = path.split('.')[-1]
removed.setdefault(ext, 0)
removed[ext] += 1
# remove *.o not belonging to Python extensions, and .py[cdo], .so files
for pat in ['*.o', '*.py[cdo]', '*.so']:
for path in self._rglob(pat):
removepath(path)
# remove .c generated from Cython .pyx
for path in self._rglob('*.pyx'):
cpath = '%s.c' % path[:-4]
removepath(cpath)
cpppath = cpath + 'pp'
removepath(cpppath)
# print short summary
if removed:
summary = 'removed %s files' % (
', '.join('%s .%s' % (count, ext)
for ext, count in sorted(removed.iteritems())))
self.announce(summary, level=distutils.log.INFO)
def _rglob(self, patten):
# recursive glob
for dirname, dirs, files in os.walk('.'):
for name in fnmatch.filter(files, patten):
yield os.path.join(dirname, name)
rust_ext_modules = []
if USERUST:
rust_ext_modules.extend([
RustExtension('treedirstate',
package='hgext3rd.rust',
manifest='rust/treedirstate/Cargo.toml',
),
])
setup(
name='fbhgext',
version='1.0',
author='Facebook Source Control Team',
maintainer='Facebook Source Control Team',
maintainer_email='sourcecontrol-dev@fb.com',
url='https://bitbucket.org/facebook/hg-experimental/',
description='Facebook mercurial extensions',
long_description="",
keywords='facebook fb hg mercurial shallow remote filelog',
license='GPLv2+',
packages=packages,
install_requires=requires,
py_modules=py_modules,
ext_modules=ext_modules,
libraries=libraries,
rust_ext_modules=rust_ext_modules,
cmdclass={
'clean_ext': CleanExtCommand,
'build_rust_ext': BuildRustExt,
}
)
Reference in New Issue View Git Blame Copy Permalink