mirror of
https://github.com/browsermt/bergamot-translator.git
synced 2024-08-15 16:40:26 +03:00
Old GitHub CI using Ubuntu and MacOS explicitly and building wheels have been removed in favour of the more portable pypa specified builds. These wheels should work just as well across a wider range of distributions. pybind11:CMakeLists.txt requires Development.Module instead of Development.* to avoid Embed from getting in the way of manylinux builds. manylinux_x86_64 builds are added for cp3.6 - 3.10. The linux build uses an old image via docker. Since the docker images are able to use shared ccache folder, builds quite fast on warm starts. ccache usage in setup.py is now triggered by an environment variable. This allows for builds not to fail if ccache not present. On tag pushes corresponding to versions, CI is configured to deliver built wheels to PyPI, reading from repository secrets. Improves setup.py including documentation and some formatting, and additional links to source. Fixes: #315
249 lines
9.2 KiB
Python
249 lines
9.2 KiB
Python
import io
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
from setuptools import Command, Extension, find_packages, setup
|
|
from setuptools.command.build_ext import build_ext
|
|
from setuptools.command.build_py import build_py as _build_py
|
|
|
|
# Convert distutils Windows platform specifiers to CMake -A arguments
|
|
PLAT_TO_CMAKE = {
|
|
"win32": "Win32",
|
|
"win-amd64": "x64",
|
|
"win-arm32": "ARM",
|
|
"win-arm64": "ARM64",
|
|
}
|
|
|
|
# A CMakeExtension needs a sourcedir instead of a file list.
|
|
# The name must be the _single_ output extension from the CMake build.
|
|
# If you need multiple extensions, see scikit-build.
|
|
class CMakeExtension(Extension):
|
|
def __init__(self, name, sourcedir=""):
|
|
Extension.__init__(self, name, sources=[])
|
|
self.sourcedir = os.path.abspath(sourcedir)
|
|
|
|
|
|
class CMakeBuild(build_ext):
|
|
def build_extension(self, ext):
|
|
extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
|
|
|
|
# required for auto-detection & inclusion of auxiliary "native" libs
|
|
if not extdir.endswith(os.path.sep):
|
|
extdir += os.path.sep
|
|
|
|
debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug
|
|
cfg = "Debug" if debug else "Release"
|
|
|
|
# CMake lets you override the generator - we need to check this.
|
|
# Can be set with Conda-Build, for example.
|
|
cmake_generator = os.environ.get("CMAKE_GENERATOR", "")
|
|
build_arch = os.environ.get("BUILD_ARCH", "native")
|
|
|
|
# Set Python_EXECUTABLE instead if you use PYBIND11_FINDPYTHON
|
|
# EXAMPLE_VERSION_INFO shows you how to pass a value into the C++ code
|
|
# from Python.
|
|
cmake_args = [
|
|
f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}",
|
|
f"-DPYTHON_EXECUTABLE={sys.executable}",
|
|
f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm
|
|
f"-DCOMPILE_PYTHON=ON",
|
|
f"-DSSPLIT_USE_INTERNAL_PCRE2=ON",
|
|
f"-DBUILD_ARCH={build_arch}",
|
|
]
|
|
|
|
build_args = ["-t", "_bergamot"]
|
|
# Adding CMake arguments set as environment variable
|
|
# (needed e.g. to build for ARM OSx on conda-forge)
|
|
if "CMAKE_ARGS" in os.environ:
|
|
cmake_args += [item for item in os.environ["CMAKE_ARGS"].split(" ") if item]
|
|
|
|
# In this example, we pass in the version to C++. You might not need to.
|
|
cmake_args += [f"-DEXAMPLE_VERSION_INFO={self.distribution.get_version()}"]
|
|
|
|
use_ccache = os.environ.get("USE_CCACHE", "0") == "1"
|
|
if use_ccache:
|
|
cmake_args += [
|
|
f"-DCMAKE_CXX_COMPILER_LAUNCHER=ccache",
|
|
f"-DCMAKE_C_COMPILER_LAUNCHER=ccache",
|
|
]
|
|
|
|
if self.compiler.compiler_type != "msvc":
|
|
# Using Ninja-build since it a) is available as a wheel and b)
|
|
# multithreads automatically. MSVC would require all variables be
|
|
# exported for Ninja to pick it up, which is a little tricky to do.
|
|
# Users can override the generator with CMAKE_GENERATOR in CMake
|
|
# 3.15+.
|
|
if not cmake_generator:
|
|
try:
|
|
import ninja # noqa: F401
|
|
|
|
cmake_args += ["-GNinja"]
|
|
except ImportError:
|
|
pass
|
|
|
|
else:
|
|
|
|
# Single config generators are handled "normally"
|
|
single_config = any(x in cmake_generator for x in {"NMake", "Ninja"})
|
|
|
|
# CMake allows an arch-in-generator style for backward compatibility
|
|
contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"})
|
|
|
|
# Specify the arch if using MSVC generator, but only if it doesn't
|
|
# contain a backward-compatibility arch spec already in the
|
|
# generator name.
|
|
if not single_config and not contains_arch:
|
|
cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]]
|
|
|
|
# Multi-config generators have a different way to specify configs
|
|
if not single_config:
|
|
cmake_args += [
|
|
f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"
|
|
]
|
|
build_args += ["--config", cfg]
|
|
|
|
if sys.platform.startswith("darwin"):
|
|
# Cross-compile support for macOS - respect ARCHFLAGS if set
|
|
archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", ""))
|
|
if archs:
|
|
cmake_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))]
|
|
|
|
# Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level
|
|
# across all generators.
|
|
if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ:
|
|
# self.parallel is a Python 3 only way to set parallel jobs by hand
|
|
# using -j in the build_ext call, not supported by pip or PyPA-build.
|
|
if hasattr(self, "parallel") and self.parallel:
|
|
# CMake 3.12+ only.
|
|
build_args += [f"-j{self.parallel}"]
|
|
|
|
if not os.path.exists(self.build_temp):
|
|
os.makedirs(self.build_temp)
|
|
|
|
print("cmake", ext.sourcedir, " ".join(cmake_args))
|
|
|
|
subprocess.check_call(
|
|
["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp
|
|
)
|
|
subprocess.check_call(
|
|
["cmake", "--build", "."] + build_args, cwd=self.build_temp
|
|
)
|
|
|
|
|
|
here = os.path.abspath(os.path.dirname(__file__))
|
|
|
|
# Import the README and use it as the long-description.
|
|
# Note: this will only work if 'README.md' is present in your MANIFEST.in file!
|
|
long_description = ""
|
|
with io.open(os.path.join(here, "bindings/python/README.md"), encoding="utf-8") as f:
|
|
long_description = "\n" + f.read()
|
|
|
|
version = None
|
|
with open(os.path.join(here, "BERGAMOT_VERSION")) as f:
|
|
version = f.read().strip()
|
|
suffix = os.environ.get("PYTHON_LOCAL_VERSION_IDENTIFIER", None)
|
|
if suffix:
|
|
version = "{}+{}".format(version, suffix)
|
|
|
|
|
|
class UploadCommand(Command):
|
|
"""Support setup.py upload."""
|
|
|
|
description = "Build and publish the package."
|
|
user_options = []
|
|
|
|
@staticmethod
|
|
def status(s):
|
|
"""Prints things in bold."""
|
|
print("\033[1m{0}\033[0m".format(s))
|
|
|
|
def initialize_options(self):
|
|
pass
|
|
|
|
def finalize_options(self):
|
|
pass
|
|
|
|
def run(self):
|
|
try:
|
|
self.status("Removing previous builds…")
|
|
rmtree(os.path.join(here, "dist"))
|
|
except OSError:
|
|
pass
|
|
|
|
self.status("Building Source and Wheel (universal) distribution…")
|
|
os.system("{0} setup.py sdist bdist_wheel --universal".format(sys.executable))
|
|
|
|
self.status("Pushing git tags…")
|
|
os.system("git push --tags")
|
|
|
|
self.status("Uploading the package to PyPI via Twine…")
|
|
os.system("twine upload dist/*")
|
|
|
|
sys.exit()
|
|
|
|
|
|
class build_py(_build_py):
|
|
def run(self):
|
|
self.run_command("build_ext")
|
|
return super().run()
|
|
|
|
|
|
# The information here can also be placed in setup.cfg - better separation of
|
|
# logic and declaration, and simpler if you include description/version in a file.
|
|
setup(
|
|
name="bergamot",
|
|
version=version,
|
|
author="Jerin Philip",
|
|
author_email="jerinphilip@live.in",
|
|
url="https://github.com/browsermt/bergamot-translator/",
|
|
description="Translate text-content locally in your machine across langauges.",
|
|
long_description=long_description,
|
|
long_description_content_type="text/markdown",
|
|
ext_modules=[CMakeExtension("bergamot/_bergamot")],
|
|
cmdclass={"build_py": build_py, "build_ext": CMakeBuild},
|
|
zip_safe=False,
|
|
extras_require={"test": ["pytest>=6.0"]},
|
|
license_files=("LICENSE",),
|
|
python_requires=">=3.6",
|
|
packages=["bergamot"],
|
|
package_dir={"bergamot": "bindings/python"},
|
|
install_requires=["requests", "pyyaml", "appdirs"],
|
|
entry_points={
|
|
"console_scripts": [
|
|
"bergamot = bergamot.__main__:main",
|
|
],
|
|
},
|
|
# Classifiers help users find your project by categorizing it.
|
|
#
|
|
# For a list of valid classifiers, see https://pypi.org/classifiers/
|
|
classifiers=[ # Optional
|
|
# How mature is this project? Common values are
|
|
# 3 - Alpha
|
|
# 4 - Beta
|
|
# 5 - Production/Stable
|
|
"Development Status :: 3 - Alpha",
|
|
# Indicate who your project is intended for
|
|
"Intended Audience :: Developers",
|
|
"Topic :: Software Development :: Build Tools",
|
|
# Pick your license as you wish
|
|
"License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
|
|
# Specify the Python versions you support here. In particular, ensure
|
|
# that you indicate you support Python 3. These classifiers are *not*
|
|
# checked by 'pip install'. See instead 'python_requires' below.
|
|
"Programming Language :: Python :: 3",
|
|
"Programming Language :: Python :: 3.6",
|
|
"Programming Language :: Python :: 3.7",
|
|
"Programming Language :: Python :: 3.8",
|
|
"Programming Language :: Python :: 3.9",
|
|
"Programming Language :: Python :: 3.10",
|
|
"Programming Language :: Python :: 3 :: Only",
|
|
],
|
|
project_urls={
|
|
"Bug Reports": "https://github.com/browsermt/bergamot-transator/issues",
|
|
"Source": "https://github.com/browsermt/bergamot-translator/",
|
|
"Documentation": "https://browser.mt/docs/main/python.html",
|
|
},
|
|
)
|