mirror of
https://github.com/browsermt/bergamot-translator.git
synced 2024-08-15 08:30:46 +03:00
c0f311a8c0
Imports python bindings and associated sources incubated in https://github.com/jerinphilip/lemonade to bergamot-translator. Adds a pybind11 dependency for python bindings. Following the import, the python build is integrated into the existing CMake based build system here. There is a command-line application provided through python which provides the ability to fetch and prepare models from model-repositories (like browsermt/students or OPUS). Wheels built for a few common operating systems are provided via GitHub releases through automated actions configured to run at tagged semantic versions and pushes to main. The documentation for python is also integrated into our existing documentation setup. Previous documentation GitHub action is now configured to run behind python builds in Ubuntu 18.04 Python3.7, in order to pick up the packaged as a wheel bergamot module and the sphinx documentation using the python module. Formatting checks of black, isort with profile black and a pytype type checker is configured for the python component residing in this repository.
211 lines
7.5 KiB
Python
211 lines
7.5 KiB
Python
import io
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
from setuptools import Command, Extension, find_packages, setup
|
|
from setuptools.command.build_ext import build_ext
|
|
from setuptools.command.build_py import build_py as _build_py
|
|
|
|
# Convert distutils Windows platform specifiers to CMake -A arguments
|
|
PLAT_TO_CMAKE = {
|
|
"win32": "Win32",
|
|
"win-amd64": "x64",
|
|
"win-arm32": "ARM",
|
|
"win-arm64": "ARM64",
|
|
}
|
|
|
|
# A CMakeExtension needs a sourcedir instead of a file list.
|
|
# The name must be the _single_ output extension from the CMake build.
|
|
# If you need multiple extensions, see scikit-build.
|
|
class CMakeExtension(Extension):
|
|
def __init__(self, name, sourcedir=""):
|
|
Extension.__init__(self, name, sources=[])
|
|
self.sourcedir = os.path.abspath(sourcedir)
|
|
|
|
|
|
class CMakeBuild(build_ext):
|
|
def build_extension(self, ext):
|
|
extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
|
|
|
|
# required for auto-detection & inclusion of auxiliary "native" libs
|
|
if not extdir.endswith(os.path.sep):
|
|
extdir += os.path.sep
|
|
|
|
debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug
|
|
cfg = "Debug" if debug else "Release"
|
|
|
|
# CMake lets you override the generator - we need to check this.
|
|
# Can be set with Conda-Build, for example.
|
|
cmake_generator = os.environ.get("CMAKE_GENERATOR", "")
|
|
build_arch = os.environ.get("BUILD_ARCH", "native")
|
|
|
|
# Set Python_EXECUTABLE instead if you use PYBIND11_FINDPYTHON
|
|
# EXAMPLE_VERSION_INFO shows you how to pass a value into the C++ code
|
|
# from Python.
|
|
cmake_args = [
|
|
f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}",
|
|
f"-DPYTHON_EXECUTABLE={sys.executable}",
|
|
f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm
|
|
f"-DCMAKE_CXX_COMPILER_LAUNCHER=ccache",
|
|
f"-DCMAKE_C_COMPILER_LAUNCHER=ccache",
|
|
f"-DCOMPILE_PYTHON=ON",
|
|
f"-DSSPLIT_USE_INTERNAL_PCRE2=ON",
|
|
f"-DBUILD_ARCH={build_arch}",
|
|
]
|
|
build_args = ["-t", "_bergamot"]
|
|
# Adding CMake arguments set as environment variable
|
|
# (needed e.g. to build for ARM OSx on conda-forge)
|
|
if "CMAKE_ARGS" in os.environ:
|
|
cmake_args += [item for item in os.environ["CMAKE_ARGS"].split(" ") if item]
|
|
|
|
# In this example, we pass in the version to C++. You might not need to.
|
|
cmake_args += [f"-DEXAMPLE_VERSION_INFO={self.distribution.get_version()}"]
|
|
|
|
if self.compiler.compiler_type != "msvc":
|
|
# Using Ninja-build since it a) is available as a wheel and b)
|
|
# multithreads automatically. MSVC would require all variables be
|
|
# exported for Ninja to pick it up, which is a little tricky to do.
|
|
# Users can override the generator with CMAKE_GENERATOR in CMake
|
|
# 3.15+.
|
|
if not cmake_generator:
|
|
try:
|
|
import ninja # noqa: F401
|
|
|
|
cmake_args += ["-GNinja"]
|
|
except ImportError:
|
|
pass
|
|
|
|
else:
|
|
|
|
# Single config generators are handled "normally"
|
|
single_config = any(x in cmake_generator for x in {"NMake", "Ninja"})
|
|
|
|
# CMake allows an arch-in-generator style for backward compatibility
|
|
contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"})
|
|
|
|
# Specify the arch if using MSVC generator, but only if it doesn't
|
|
# contain a backward-compatibility arch spec already in the
|
|
# generator name.
|
|
if not single_config and not contains_arch:
|
|
cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]]
|
|
|
|
# Multi-config generators have a different way to specify configs
|
|
if not single_config:
|
|
cmake_args += [
|
|
f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"
|
|
]
|
|
build_args += ["--config", cfg]
|
|
|
|
if sys.platform.startswith("darwin"):
|
|
# Cross-compile support for macOS - respect ARCHFLAGS if set
|
|
archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", ""))
|
|
if archs:
|
|
cmake_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))]
|
|
|
|
# Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level
|
|
# across all generators.
|
|
if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ:
|
|
# self.parallel is a Python 3 only way to set parallel jobs by hand
|
|
# using -j in the build_ext call, not supported by pip or PyPA-build.
|
|
if hasattr(self, "parallel") and self.parallel:
|
|
# CMake 3.12+ only.
|
|
build_args += [f"-j{self.parallel}"]
|
|
|
|
if not os.path.exists(self.build_temp):
|
|
os.makedirs(self.build_temp)
|
|
|
|
print("cmake", ext.sourcedir, " ".join(cmake_args))
|
|
|
|
subprocess.check_call(
|
|
["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp
|
|
)
|
|
subprocess.check_call(
|
|
["cmake", "--build", "."] + build_args, cwd=self.build_temp
|
|
)
|
|
|
|
|
|
here = os.path.abspath(os.path.dirname(__file__))
|
|
|
|
# Import the README and use it as the long-description.
|
|
# Note: this will only work if 'README.md' is present in your MANIFEST.in file!
|
|
with io.open(os.path.join(here, "README.md"), encoding="utf-8") as f:
|
|
long_description = "\n" + f.read()
|
|
|
|
version = None
|
|
with open(os.path.join(here, "BERGAMOT_VERSION")) as f:
|
|
version = f.read().strip()
|
|
suffix = os.environ.get("PYTHON_LOCAL_VERSION_IDENTIFIER", None)
|
|
if suffix is not None:
|
|
version = "{}+{}".format(version, suffix)
|
|
|
|
|
|
class UploadCommand(Command):
|
|
"""Support setup.py upload."""
|
|
|
|
description = "Build and publish the package."
|
|
user_options = []
|
|
|
|
@staticmethod
|
|
def status(s):
|
|
"""Prints things in bold."""
|
|
print("\033[1m{0}\033[0m".format(s))
|
|
|
|
def initialize_options(self):
|
|
pass
|
|
|
|
def finalize_options(self):
|
|
pass
|
|
|
|
def run(self):
|
|
try:
|
|
self.status("Removing previous builds…")
|
|
rmtree(os.path.join(here, "dist"))
|
|
except OSError:
|
|
pass
|
|
|
|
self.status("Building Source and Wheel (universal) distribution…")
|
|
os.system("{0} setup.py sdist bdist_wheel --universal".format(sys.executable))
|
|
|
|
self.status("Pushing git tags…")
|
|
os.system("git push --tags")
|
|
|
|
self.status("Uploading the package to PyPI via Twine…")
|
|
os.system("twine upload dist/*")
|
|
|
|
sys.exit()
|
|
|
|
|
|
class build_py(_build_py):
|
|
def run(self):
|
|
self.run_command("build_ext")
|
|
return super().run()
|
|
|
|
|
|
# The information here can also be placed in setup.cfg - better separation of
|
|
# logic and declaration, and simpler if you include description/version in a file.
|
|
setup(
|
|
name="bergamot",
|
|
version=version,
|
|
author="Jerin Philip",
|
|
author_email="jerinphilip@live.in",
|
|
url="https://github.com/browsermt/bergamot-translator/",
|
|
description="Bergamot translator python binding.",
|
|
long_description="",
|
|
ext_modules=[CMakeExtension("bergamot/_bergamot")],
|
|
cmdclass={"build_py": build_py, "build_ext": CMakeBuild},
|
|
zip_safe=False,
|
|
extras_require={"test": ["pytest>=6.0"]},
|
|
license_files=("LICENSE",),
|
|
python_requires=">=3.6",
|
|
packages=["bergamot"],
|
|
package_dir={"bergamot": "bindings/python"},
|
|
install_requires=["requests", "pyyaml", "appdirs"],
|
|
entry_points={
|
|
"console_scripts": [
|
|
"bergamot = bergamot.__main__:main",
|
|
],
|
|
},
|
|
)
|