2017-09-25 20:17:43 +03:00
|
|
|
#!/usr/bin/env python3
|
2019-07-30 17:45:13 +03:00
|
|
|
# Copyright (c) Facebook, Inc. and its affiliates.
|
2017-09-15 03:22:43 +03:00
|
|
|
#
|
2019-07-30 17:45:13 +03:00
|
|
|
# This source code is licensed under the MIT license found in the
|
|
|
|
# LICENSE file in the root directory of this source tree.
|
2017-09-15 03:22:43 +03:00
|
|
|
|
2019-11-03 02:51:32 +03:00
|
|
|
import os
|
2020-10-22 22:45:19 +03:00
|
|
|
import subprocess
|
2017-09-15 03:22:43 +03:00
|
|
|
import sys
|
2020-10-22 22:45:19 +03:00
|
|
|
from setuptools import setup, find_packages, Extension
|
2017-09-15 03:22:43 +03:00
|
|
|
|
2020-10-19 04:13:29 +03:00
|
|
|
from setuptools import Extension, find_packages, setup
|
|
|
|
|
2017-09-15 03:22:43 +03:00
|
|
|
|
2019-12-17 06:46:03 +03:00
|
|
|
if sys.version_info < (3, 6):
|
2020-10-19 04:13:29 +03:00
|
|
|
sys.exit("Sorry, Python >= 3.6 is required for fairseq.")
|
2017-09-15 03:22:43 +03:00
|
|
|
|
2019-08-27 20:06:26 +03:00
|
|
|
|
2020-10-22 22:45:19 +03:00
|
|
|
def write_version_py():
|
|
|
|
with open(os.path.join("fairseq", "version.txt")) as f:
|
|
|
|
version = f.read().strip()
|
|
|
|
|
|
|
|
# append latest commit hash to version string
|
|
|
|
try:
|
2020-11-20 16:59:25 +03:00
|
|
|
sha = (
|
|
|
|
subprocess.check_output(["git", "rev-parse", "HEAD"])
|
|
|
|
.decode("ascii")
|
|
|
|
.strip()
|
|
|
|
)
|
2020-10-22 22:45:19 +03:00
|
|
|
version += "+" + sha[:7]
|
|
|
|
except Exception:
|
|
|
|
pass
|
|
|
|
|
|
|
|
# write version info to fairseq/version.py
|
|
|
|
with open(os.path.join("fairseq", "version.py"), "w") as f:
|
2020-11-20 16:59:25 +03:00
|
|
|
f.write('__version__ = "{}"\n'.format(version))
|
2020-10-22 22:45:19 +03:00
|
|
|
return version
|
|
|
|
|
|
|
|
|
|
|
|
version = write_version_py()
|
|
|
|
|
|
|
|
|
2020-10-19 04:13:29 +03:00
|
|
|
with open("README.md") as f:
|
2017-09-15 03:22:43 +03:00
|
|
|
readme = f.read()
|
|
|
|
|
2019-08-27 20:06:26 +03:00
|
|
|
|
2020-10-19 04:13:29 +03:00
|
|
|
if sys.platform == "darwin":
|
|
|
|
extra_compile_args = ["-stdlib=libc++", "-O3"]
|
2019-07-06 22:27:58 +03:00
|
|
|
else:
|
2020-10-19 04:13:29 +03:00
|
|
|
extra_compile_args = ["-std=c++11", "-O3"]
|
2017-09-15 03:22:43 +03:00
|
|
|
|
2019-08-27 17:10:35 +03:00
|
|
|
|
2019-09-01 02:52:03 +03:00
|
|
|
class NumpyExtension(Extension):
|
|
|
|
"""Source: https://stackoverflow.com/a/54128391"""
|
|
|
|
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
|
|
self.__include_dirs = []
|
|
|
|
super().__init__(*args, **kwargs)
|
|
|
|
|
|
|
|
@property
|
|
|
|
def include_dirs(self):
|
|
|
|
import numpy
|
2020-10-19 04:13:29 +03:00
|
|
|
|
2019-09-01 02:52:03 +03:00
|
|
|
return self.__include_dirs + [numpy.get_include()]
|
|
|
|
|
|
|
|
@include_dirs.setter
|
|
|
|
def include_dirs(self, dirs):
|
|
|
|
self.__include_dirs = dirs
|
|
|
|
|
|
|
|
|
2019-08-27 20:06:26 +03:00
|
|
|
extensions = [
|
|
|
|
Extension(
|
2020-10-19 04:13:29 +03:00
|
|
|
"fairseq.libbleu",
|
2019-08-27 20:06:26 +03:00
|
|
|
sources=[
|
2020-10-19 04:13:29 +03:00
|
|
|
"fairseq/clib/libbleu/libbleu.cpp",
|
|
|
|
"fairseq/clib/libbleu/module.cpp",
|
2019-08-27 20:06:26 +03:00
|
|
|
],
|
2019-08-27 17:10:35 +03:00
|
|
|
extra_compile_args=extra_compile_args,
|
2019-08-27 20:06:26 +03:00
|
|
|
),
|
2019-09-01 02:52:03 +03:00
|
|
|
NumpyExtension(
|
2020-10-19 04:13:29 +03:00
|
|
|
"fairseq.data.data_utils_fast",
|
|
|
|
sources=["fairseq/data/data_utils_fast.pyx"],
|
|
|
|
language="c++",
|
2019-08-27 17:10:35 +03:00
|
|
|
extra_compile_args=extra_compile_args,
|
2019-08-27 20:06:26 +03:00
|
|
|
),
|
2019-09-01 02:52:03 +03:00
|
|
|
NumpyExtension(
|
2020-10-19 04:13:29 +03:00
|
|
|
"fairseq.data.token_block_utils_fast",
|
|
|
|
sources=["fairseq/data/token_block_utils_fast.pyx"],
|
|
|
|
language="c++",
|
2019-08-27 20:06:26 +03:00
|
|
|
extra_compile_args=extra_compile_args,
|
|
|
|
),
|
|
|
|
]
|
2019-08-27 17:10:35 +03:00
|
|
|
|
|
|
|
|
2019-11-03 02:51:32 +03:00
|
|
|
cmdclass = {}
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
# torch is not available when generating docs
|
|
|
|
from torch.utils import cpp_extension
|
2019-12-21 13:43:47 +03:00
|
|
|
|
2020-10-19 04:13:29 +03:00
|
|
|
extensions.extend(
|
|
|
|
[
|
2019-12-21 13:43:47 +03:00
|
|
|
cpp_extension.CppExtension(
|
2020-10-19 04:13:29 +03:00
|
|
|
"fairseq.libnat",
|
2019-12-21 13:43:47 +03:00
|
|
|
sources=[
|
2020-10-19 04:13:29 +03:00
|
|
|
"fairseq/clib/libnat/edit_dist.cpp",
|
2019-12-21 13:43:47 +03:00
|
|
|
],
|
2020-10-19 04:13:29 +03:00
|
|
|
)
|
|
|
|
]
|
|
|
|
)
|
|
|
|
|
|
|
|
if "CUDA_HOME" in os.environ:
|
|
|
|
extensions.extend(
|
|
|
|
[
|
|
|
|
cpp_extension.CppExtension(
|
|
|
|
"fairseq.libnat_cuda",
|
|
|
|
sources=[
|
|
|
|
"fairseq/clib/libnat_cuda/edit_dist.cu",
|
|
|
|
"fairseq/clib/libnat_cuda/binding.cpp",
|
|
|
|
],
|
|
|
|
)
|
|
|
|
]
|
|
|
|
)
|
|
|
|
cmdclass["build_ext"] = cpp_extension.BuildExtension
|
2019-12-21 13:43:47 +03:00
|
|
|
|
2019-11-03 02:51:32 +03:00
|
|
|
except ImportError:
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
2020-10-19 04:13:29 +03:00
|
|
|
if "READTHEDOCS" in os.environ:
|
2019-11-03 02:51:32 +03:00
|
|
|
# don't build extensions when generating docs
|
|
|
|
extensions = []
|
2020-10-19 04:13:29 +03:00
|
|
|
if "build_ext" in cmdclass:
|
|
|
|
del cmdclass["build_ext"]
|
2019-11-03 02:51:32 +03:00
|
|
|
|
|
|
|
# use CPU build of PyTorch
|
|
|
|
dependency_links = [
|
2020-11-19 01:30:02 +03:00
|
|
|
"https://download.pytorch.org/whl/cpu/torch-1.7.0%2Bcpu-cp36-cp36m-linux_x86_64.whl"
|
2019-11-03 02:51:32 +03:00
|
|
|
]
|
|
|
|
else:
|
|
|
|
dependency_links = []
|
|
|
|
|
|
|
|
|
2020-10-19 04:13:29 +03:00
|
|
|
if "clean" in sys.argv[1:]:
|
2019-11-13 21:47:05 +03:00
|
|
|
# Source: https://bit.ly/2NLVsgE
|
|
|
|
print("deleting Cython files...")
|
|
|
|
import subprocess
|
2020-10-19 04:13:29 +03:00
|
|
|
|
|
|
|
subprocess.run(
|
|
|
|
["rm -f fairseq/*.so fairseq/**/*.so fairseq/*.pyd fairseq/**/*.pyd"],
|
|
|
|
shell=True,
|
|
|
|
)
|
2019-11-26 00:38:42 +03:00
|
|
|
|
|
|
|
|
2020-11-19 01:30:02 +03:00
|
|
|
extra_packages = []
|
|
|
|
if os.path.exists(os.path.join("fairseq", "model_parallel", "megatron", "mpu")):
|
|
|
|
extra_packages.append("fairseq.model_parallel.megatron.mpu")
|
|
|
|
|
|
|
|
|
2020-10-19 19:22:28 +03:00
|
|
|
def do_setup(package_data):
|
|
|
|
setup(
|
|
|
|
name="fairseq",
|
2020-10-22 22:45:19 +03:00
|
|
|
version=version,
|
2020-10-19 19:22:28 +03:00
|
|
|
description="Facebook AI Research Sequence-to-Sequence Toolkit",
|
|
|
|
url="https://github.com/pytorch/fairseq",
|
|
|
|
classifiers=[
|
|
|
|
"Intended Audience :: Science/Research",
|
|
|
|
"License :: OSI Approved :: MIT License",
|
|
|
|
"Programming Language :: Python :: 3.6",
|
|
|
|
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
2019-02-05 18:46:44 +03:00
|
|
|
],
|
2020-10-19 19:22:28 +03:00
|
|
|
long_description=readme,
|
|
|
|
long_description_content_type="text/markdown",
|
|
|
|
setup_requires=[
|
|
|
|
"cython",
|
2020-12-04 05:17:09 +03:00
|
|
|
'numpy<1.20.0; python_version<"3.7"',
|
|
|
|
'numpy; python_version>="3.7"',
|
2020-10-19 19:22:28 +03:00
|
|
|
"setuptools>=18.0",
|
|
|
|
],
|
|
|
|
install_requires=[
|
|
|
|
"cffi",
|
|
|
|
"cython",
|
2020-12-04 05:17:09 +03:00
|
|
|
'dataclasses; python_version<"3.7"',
|
2020-10-19 19:22:28 +03:00
|
|
|
"hydra-core",
|
2020-12-04 05:17:09 +03:00
|
|
|
'numpy<1.20.0; python_version<"3.7"',
|
|
|
|
'numpy; python_version>="3.7"',
|
2020-10-19 19:22:28 +03:00
|
|
|
"regex",
|
|
|
|
"sacrebleu>=1.4.12",
|
|
|
|
"torch",
|
|
|
|
"tqdm",
|
|
|
|
],
|
|
|
|
dependency_links=dependency_links,
|
|
|
|
packages=find_packages(
|
|
|
|
exclude=[
|
|
|
|
"examples",
|
|
|
|
"examples.*",
|
|
|
|
"scripts",
|
|
|
|
"scripts.*",
|
|
|
|
"tests",
|
|
|
|
"tests.*",
|
|
|
|
]
|
2020-11-20 16:59:25 +03:00
|
|
|
)
|
|
|
|
+ extra_packages,
|
2020-10-19 19:22:28 +03:00
|
|
|
package_data=package_data,
|
|
|
|
ext_modules=extensions,
|
|
|
|
test_suite="tests",
|
|
|
|
entry_points={
|
|
|
|
"console_scripts": [
|
|
|
|
"fairseq-eval-lm = fairseq_cli.eval_lm:cli_main",
|
|
|
|
"fairseq-generate = fairseq_cli.generate:cli_main",
|
2020-11-20 16:59:25 +03:00
|
|
|
"fairseq-hydra-train = fairseq_cli.hydra_train:cli_main",
|
2020-10-19 19:22:28 +03:00
|
|
|
"fairseq-interactive = fairseq_cli.interactive:cli_main",
|
|
|
|
"fairseq-preprocess = fairseq_cli.preprocess:cli_main",
|
|
|
|
"fairseq-score = fairseq_cli.score:cli_main",
|
|
|
|
"fairseq-train = fairseq_cli.train:cli_main",
|
|
|
|
"fairseq-validate = fairseq_cli.validate:cli_main",
|
|
|
|
],
|
|
|
|
},
|
|
|
|
cmdclass=cmdclass,
|
|
|
|
zip_safe=False,
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
def get_files(path, relative_to="fairseq"):
|
|
|
|
all_files = []
|
|
|
|
for root, _dirs, files in os.walk(path, followlinks=True):
|
|
|
|
root = os.path.relpath(root, relative_to)
|
|
|
|
for file in files:
|
|
|
|
if file.endswith(".pyc"):
|
|
|
|
continue
|
|
|
|
all_files.append(os.path.join(root, file))
|
|
|
|
return all_files
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
2020-11-07 09:51:29 +03:00
|
|
|
# symlink examples into fairseq package so package_data accepts them
|
2020-11-19 01:30:02 +03:00
|
|
|
fairseq_examples = os.path.join("fairseq", "examples")
|
|
|
|
if "build_ext" not in sys.argv[1:] and not os.path.exists(fairseq_examples):
|
|
|
|
os.symlink(os.path.join("..", "examples"), fairseq_examples)
|
2020-11-20 16:59:25 +03:00
|
|
|
|
2020-10-19 19:22:28 +03:00
|
|
|
package_data = {
|
2020-11-20 16:59:25 +03:00
|
|
|
"fairseq": (
|
|
|
|
get_files(fairseq_examples) + get_files(os.path.join("fairseq", "config"))
|
|
|
|
)
|
2020-10-19 19:22:28 +03:00
|
|
|
}
|
|
|
|
do_setup(package_data)
|
|
|
|
finally:
|
2020-11-19 01:30:02 +03:00
|
|
|
if "build_ext" not in sys.argv[1:] and os.path.exists(fairseq_examples):
|
|
|
|
os.unlink(fairseq_examples)
|