2017-09-25 20:17:43 +03:00
|
|
|
#!/usr/bin/env python3
|
2019-07-30 17:45:13 +03:00
|
|
|
# Copyright (c) Facebook, Inc. and its affiliates.
|
2017-09-15 03:22:43 +03:00
|
|
|
#
|
2019-07-30 17:45:13 +03:00
|
|
|
# This source code is licensed under the MIT license found in the
|
|
|
|
# LICENSE file in the root directory of this source tree.
|
2017-09-15 03:22:43 +03:00
|
|
|
|
2019-11-03 02:51:32 +03:00
|
|
|
import os
|
2020-10-22 22:45:19 +03:00
|
|
|
import subprocess
|
2017-09-15 03:22:43 +03:00
|
|
|
import sys
|
2020-10-22 22:45:19 +03:00
|
|
|
from setuptools import setup, find_packages, Extension
|
2017-09-15 03:22:43 +03:00
|
|
|
|
2020-10-19 04:13:29 +03:00
|
|
|
from setuptools import Extension, find_packages, setup
|
|
|
|
|
2017-09-15 03:22:43 +03:00
|
|
|
|
2019-12-17 06:46:03 +03:00
|
|
|
if sys.version_info < (3, 6):
|
2020-10-19 04:13:29 +03:00
|
|
|
sys.exit("Sorry, Python >= 3.6 is required for fairseq.")
|
2017-09-15 03:22:43 +03:00
|
|
|
|
2019-08-27 20:06:26 +03:00
|
|
|
|
2020-10-22 22:45:19 +03:00
|
|
|
def write_version_py():
|
|
|
|
with open(os.path.join("fairseq", "version.txt")) as f:
|
|
|
|
version = f.read().strip()
|
|
|
|
|
|
|
|
# append latest commit hash to version string
|
|
|
|
try:
|
2020-11-20 16:59:25 +03:00
|
|
|
sha = (
|
|
|
|
subprocess.check_output(["git", "rev-parse", "HEAD"])
|
|
|
|
.decode("ascii")
|
|
|
|
.strip()
|
|
|
|
)
|
2020-10-22 22:45:19 +03:00
|
|
|
version += "+" + sha[:7]
|
|
|
|
except Exception:
|
|
|
|
pass
|
|
|
|
|
|
|
|
# write version info to fairseq/version.py
|
|
|
|
with open(os.path.join("fairseq", "version.py"), "w") as f:
|
2020-11-20 16:59:25 +03:00
|
|
|
f.write('__version__ = "{}"\n'.format(version))
|
2020-10-22 22:45:19 +03:00
|
|
|
return version
|
|
|
|
|
|
|
|
|
|
|
|
version = write_version_py()
|
|
|
|
|
|
|
|
|
2020-10-19 04:13:29 +03:00
|
|
|
with open("README.md") as f:
|
2017-09-15 03:22:43 +03:00
|
|
|
readme = f.read()
|
|
|
|
|
2019-08-27 20:06:26 +03:00
|
|
|
|
2020-10-19 04:13:29 +03:00
|
|
|
if sys.platform == "darwin":
|
|
|
|
extra_compile_args = ["-stdlib=libc++", "-O3"]
|
2019-07-06 22:27:58 +03:00
|
|
|
else:
|
2020-10-19 04:13:29 +03:00
|
|
|
extra_compile_args = ["-std=c++11", "-O3"]
|
2017-09-15 03:22:43 +03:00
|
|
|
|
2019-08-27 17:10:35 +03:00
|
|
|
|
2019-09-01 02:52:03 +03:00
|
|
|
class NumpyExtension(Extension):
|
|
|
|
"""Source: https://stackoverflow.com/a/54128391"""
|
|
|
|
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
|
|
self.__include_dirs = []
|
|
|
|
super().__init__(*args, **kwargs)
|
|
|
|
|
|
|
|
@property
|
|
|
|
def include_dirs(self):
|
|
|
|
import numpy
|
2020-10-19 04:13:29 +03:00
|
|
|
|
2019-09-01 02:52:03 +03:00
|
|
|
return self.__include_dirs + [numpy.get_include()]
|
|
|
|
|
|
|
|
@include_dirs.setter
|
|
|
|
def include_dirs(self, dirs):
|
|
|
|
self.__include_dirs = dirs
|
|
|
|
|
|
|
|
|
2019-08-27 20:06:26 +03:00
|
|
|
extensions = [
|
|
|
|
Extension(
|
2020-10-19 04:13:29 +03:00
|
|
|
"fairseq.libbleu",
|
2019-08-27 20:06:26 +03:00
|
|
|
sources=[
|
2020-10-19 04:13:29 +03:00
|
|
|
"fairseq/clib/libbleu/libbleu.cpp",
|
|
|
|
"fairseq/clib/libbleu/module.cpp",
|
2019-08-27 20:06:26 +03:00
|
|
|
],
|
2019-08-27 17:10:35 +03:00
|
|
|
extra_compile_args=extra_compile_args,
|
2019-08-27 20:06:26 +03:00
|
|
|
),
|
2019-09-01 02:52:03 +03:00
|
|
|
NumpyExtension(
|
2020-10-19 04:13:29 +03:00
|
|
|
"fairseq.data.data_utils_fast",
|
|
|
|
sources=["fairseq/data/data_utils_fast.pyx"],
|
|
|
|
language="c++",
|
2019-08-27 17:10:35 +03:00
|
|
|
extra_compile_args=extra_compile_args,
|
2019-08-27 20:06:26 +03:00
|
|
|
),
|
2019-09-01 02:52:03 +03:00
|
|
|
NumpyExtension(
|
2020-10-19 04:13:29 +03:00
|
|
|
"fairseq.data.token_block_utils_fast",
|
|
|
|
sources=["fairseq/data/token_block_utils_fast.pyx"],
|
|
|
|
language="c++",
|
2019-08-27 20:06:26 +03:00
|
|
|
extra_compile_args=extra_compile_args,
|
|
|
|
),
|
|
|
|
]
|
2019-08-27 17:10:35 +03:00
|
|
|
|
|
|
|
|
2019-11-03 02:51:32 +03:00
|
|
|
cmdclass = {}
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
# torch is not available when generating docs
|
|
|
|
from torch.utils import cpp_extension
|
2019-12-21 13:43:47 +03:00
|
|
|
|
2021-03-30 04:02:07 +03:00
|
|
|
extensions.extend(
|
|
|
|
[
|
|
|
|
cpp_extension.CppExtension(
|
|
|
|
"fairseq.libbase",
|
|
|
|
sources=[
|
|
|
|
"fairseq/clib/libbase/balanced_assignment.cpp",
|
|
|
|
],
|
|
|
|
)
|
|
|
|
]
|
|
|
|
)
|
|
|
|
|
2020-10-19 04:13:29 +03:00
|
|
|
extensions.extend(
|
|
|
|
[
|
2019-12-21 13:43:47 +03:00
|
|
|
cpp_extension.CppExtension(
|
2020-10-19 04:13:29 +03:00
|
|
|
"fairseq.libnat",
|
2019-12-21 13:43:47 +03:00
|
|
|
sources=[
|
2020-10-19 04:13:29 +03:00
|
|
|
"fairseq/clib/libnat/edit_dist.cpp",
|
2019-12-21 13:43:47 +03:00
|
|
|
],
|
2020-10-19 04:13:29 +03:00
|
|
|
)
|
|
|
|
]
|
|
|
|
)
|
|
|
|
if "CUDA_HOME" in os.environ:
|
|
|
|
extensions.extend(
|
|
|
|
[
|
|
|
|
cpp_extension.CppExtension(
|
|
|
|
"fairseq.libnat_cuda",
|
|
|
|
sources=[
|
|
|
|
"fairseq/clib/libnat_cuda/edit_dist.cu",
|
|
|
|
"fairseq/clib/libnat_cuda/binding.cpp",
|
|
|
|
],
|
2020-12-30 23:57:02 +03:00
|
|
|
),
|
|
|
|
cpp_extension.CppExtension(
|
|
|
|
"fairseq.ngram_repeat_block_cuda",
|
|
|
|
sources=[
|
|
|
|
"fairseq/clib/cuda/ngram_repeat_block_cuda.cpp",
|
|
|
|
"fairseq/clib/cuda/ngram_repeat_block_cuda_kernel.cu",
|
|
|
|
],
|
|
|
|
),
|
2020-10-19 04:13:29 +03:00
|
|
|
]
|
|
|
|
)
|
|
|
|
cmdclass["build_ext"] = cpp_extension.BuildExtension
|
2019-12-21 13:43:47 +03:00
|
|
|
|
2019-11-03 02:51:32 +03:00
|
|
|
except ImportError:
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
2020-10-19 04:13:29 +03:00
|
|
|
if "READTHEDOCS" in os.environ:
|
2019-11-03 02:51:32 +03:00
|
|
|
# don't build extensions when generating docs
|
|
|
|
extensions = []
|
2020-10-19 04:13:29 +03:00
|
|
|
if "build_ext" in cmdclass:
|
|
|
|
del cmdclass["build_ext"]
|
2019-11-03 02:51:32 +03:00
|
|
|
|
|
|
|
# use CPU build of PyTorch
|
|
|
|
dependency_links = [
|
2020-11-19 01:30:02 +03:00
|
|
|
"https://download.pytorch.org/whl/cpu/torch-1.7.0%2Bcpu-cp36-cp36m-linux_x86_64.whl"
|
2019-11-03 02:51:32 +03:00
|
|
|
]
|
|
|
|
else:
|
|
|
|
dependency_links = []
|
|
|
|
|
|
|
|
|
2020-10-19 04:13:29 +03:00
|
|
|
if "clean" in sys.argv[1:]:
|
2019-11-13 21:47:05 +03:00
|
|
|
# Source: https://bit.ly/2NLVsgE
|
|
|
|
print("deleting Cython files...")
|
|
|
|
import subprocess
|
2020-10-19 04:13:29 +03:00
|
|
|
|
|
|
|
subprocess.run(
|
|
|
|
["rm -f fairseq/*.so fairseq/**/*.so fairseq/*.pyd fairseq/**/*.pyd"],
|
|
|
|
shell=True,
|
|
|
|
)
|
2019-11-26 00:38:42 +03:00
|
|
|
|
|
|
|
|
2020-11-19 01:30:02 +03:00
|
|
|
extra_packages = []
|
|
|
|
if os.path.exists(os.path.join("fairseq", "model_parallel", "megatron", "mpu")):
|
|
|
|
extra_packages.append("fairseq.model_parallel.megatron.mpu")
|
|
|
|
|
|
|
|
|
2020-10-19 19:22:28 +03:00
|
|
|
def do_setup(package_data):
|
|
|
|
setup(
|
|
|
|
name="fairseq",
|
2020-10-22 22:45:19 +03:00
|
|
|
version=version,
|
2020-10-19 19:22:28 +03:00
|
|
|
description="Facebook AI Research Sequence-to-Sequence Toolkit",
|
|
|
|
url="https://github.com/pytorch/fairseq",
|
|
|
|
classifiers=[
|
|
|
|
"Intended Audience :: Science/Research",
|
|
|
|
"License :: OSI Approved :: MIT License",
|
|
|
|
"Programming Language :: Python :: 3.6",
|
2020-12-16 04:46:37 +03:00
|
|
|
"Programming Language :: Python :: 3.7",
|
|
|
|
"Programming Language :: Python :: 3.8",
|
2020-10-19 19:22:28 +03:00
|
|
|
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
2019-02-05 18:46:44 +03:00
|
|
|
],
|
2020-10-19 19:22:28 +03:00
|
|
|
long_description=readme,
|
|
|
|
long_description_content_type="text/markdown",
|
|
|
|
setup_requires=[
|
|
|
|
"cython",
|
2020-12-04 05:17:09 +03:00
|
|
|
'numpy<1.20.0; python_version<"3.7"',
|
|
|
|
'numpy; python_version>="3.7"',
|
2020-10-19 19:22:28 +03:00
|
|
|
"setuptools>=18.0",
|
|
|
|
],
|
|
|
|
install_requires=[
|
|
|
|
"cffi",
|
|
|
|
"cython",
|
2020-12-04 05:17:09 +03:00
|
|
|
'dataclasses; python_version<"3.7"',
|
2020-12-16 04:46:37 +03:00
|
|
|
"hydra-core<1.1",
|
|
|
|
"omegaconf<2.1",
|
2020-12-04 05:17:09 +03:00
|
|
|
'numpy<1.20.0; python_version<"3.7"',
|
|
|
|
'numpy; python_version>="3.7"',
|
2020-10-19 19:22:28 +03:00
|
|
|
"regex",
|
|
|
|
"sacrebleu>=1.4.12",
|
|
|
|
"torch",
|
|
|
|
"tqdm",
|
|
|
|
],
|
|
|
|
dependency_links=dependency_links,
|
|
|
|
packages=find_packages(
|
|
|
|
exclude=[
|
|
|
|
"examples",
|
|
|
|
"examples.*",
|
|
|
|
"scripts",
|
|
|
|
"scripts.*",
|
|
|
|
"tests",
|
|
|
|
"tests.*",
|
|
|
|
]
|
2020-11-20 16:59:25 +03:00
|
|
|
)
|
|
|
|
+ extra_packages,
|
2020-10-19 19:22:28 +03:00
|
|
|
package_data=package_data,
|
|
|
|
ext_modules=extensions,
|
|
|
|
test_suite="tests",
|
|
|
|
entry_points={
|
|
|
|
"console_scripts": [
|
|
|
|
"fairseq-eval-lm = fairseq_cli.eval_lm:cli_main",
|
|
|
|
"fairseq-generate = fairseq_cli.generate:cli_main",
|
2020-11-20 16:59:25 +03:00
|
|
|
"fairseq-hydra-train = fairseq_cli.hydra_train:cli_main",
|
2020-10-19 19:22:28 +03:00
|
|
|
"fairseq-interactive = fairseq_cli.interactive:cli_main",
|
|
|
|
"fairseq-preprocess = fairseq_cli.preprocess:cli_main",
|
|
|
|
"fairseq-score = fairseq_cli.score:cli_main",
|
|
|
|
"fairseq-train = fairseq_cli.train:cli_main",
|
|
|
|
"fairseq-validate = fairseq_cli.validate:cli_main",
|
|
|
|
],
|
|
|
|
},
|
|
|
|
cmdclass=cmdclass,
|
|
|
|
zip_safe=False,
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
def get_files(path, relative_to="fairseq"):
|
|
|
|
all_files = []
|
|
|
|
for root, _dirs, files in os.walk(path, followlinks=True):
|
|
|
|
root = os.path.relpath(root, relative_to)
|
|
|
|
for file in files:
|
|
|
|
if file.endswith(".pyc"):
|
|
|
|
continue
|
|
|
|
all_files.append(os.path.join(root, file))
|
|
|
|
return all_files
|
|
|
|
|
|
|
|
|
2021-01-21 18:32:08 +03:00
|
|
|
if __name__ == "__main__":
|
|
|
|
try:
|
|
|
|
# symlink examples into fairseq package so package_data accepts them
|
|
|
|
fairseq_examples = os.path.join("fairseq", "examples")
|
|
|
|
if "build_ext" not in sys.argv[1:] and not os.path.exists(fairseq_examples):
|
|
|
|
os.symlink(os.path.join("..", "examples"), fairseq_examples)
|
|
|
|
|
|
|
|
package_data = {
|
|
|
|
"fairseq": (
|
|
|
|
get_files(fairseq_examples) + get_files(os.path.join("fairseq", "config"))
|
|
|
|
)
|
|
|
|
}
|
|
|
|
do_setup(package_data)
|
|
|
|
finally:
|
Fix attempt to unlink directory copied into source package (Python 3.9) (#3235)
Summary:
# Before submitting
- [ ] Was this discussed/approved via a Github issue? (no need for typos, doc improvements)
- [x] Did you read the [contributor guideline](https://github.com/pytorch/fairseq/blob/master/CONTRIBUTING.md)?
- [N/A] Did you make sure to update the docs?
- [N/A] Did you write any new necessary tests?
## What does this PR do?
Currently when installing the newest source package from PyPI I get an error like so:
```
Collecting fairseq
Using cached fairseq-0.10.2.tar.gz (938 kB)
Installing build dependencies ... done
Getting requirements to build wheel ... error
ERROR: Command errored out with exit status 1:
command: /home/frankier/sources/datasets/.venv/bin/python3 /tmp/tmp_ujftsgi_in_process.py get_requires_for_build_wheel /tmp/tmpmn0eumq2
cwd: /tmp/pip-install-dg5d6q9y/fairseq
Complete output (31 lines):
Traceback (most recent call last):
File "setup.py", line 214, in <module>
do_setup(package_data)
File "setup.py", line 136, in do_setup
setup(
File "/tmp/pip-build-env-hag0sxvp/overlay/lib/python3.9/site-packages/setuptools/__init__.py", line 152, in setup
_install_setup_requires(attrs)
File "/tmp/pip-build-env-hag0sxvp/overlay/lib/python3.9/site-packages/setuptools/__init__.py", line 147, in _install_setup_requires
dist.fetch_build_eggs(dist.setup_requires)
File "/tmp/pip-build-env-hag0sxvp/overlay/lib/python3.9/site-packages/setuptools/build_meta.py", line 60, in fetch_build_eggs
raise SetupRequirementsError(specifier_list)
setuptools.build_meta.SetupRequirementsError: ['cython', 'numpy', 'setuptools>=18.0']
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/tmp/tmp_ujftsgi_in_process.py", line 280, in <module>
main()
File "/tmp/tmp_ujftsgi_in_process.py", line 263, in main
json_out['return_val'] = hook(**hook_input['kwargs'])
File "/tmp/tmp_ujftsgi_in_process.py", line 114, in get_requires_for_build_wheel
return hook(config_settings)
File "/tmp/pip-build-env-hag0sxvp/overlay/lib/python3.9/site-packages/setuptools/build_meta.py", line 149, in get_requires_for_build_wheel
return self._get_build_requires(
File "/tmp/pip-build-env-hag0sxvp/overlay/lib/python3.9/site-packages/setuptools/build_meta.py", line 130, in _get_build_requires
self.run_setup()
File "/tmp/pip-build-env-hag0sxvp/overlay/lib/python3.9/site-packages/setuptools/build_meta.py", line 145, in run_setup
exec(compile(code, __file__, 'exec'), locals())
File "setup.py", line 217, in <module>
os.unlink(fairseq_examples)
IsADirectoryError: [Errno 21] Is a directory: 'fairseq/examples'
----------------------------------------
ERROR: Command errored out with exit status 1: /home/frankier/sources/datasets/.venv/bin/python3 /tmp/tmp_ujftsgi_in_process.py get_requires_for_build_wheel /tmp/tmpmn0eumq2 Check the logs for full command output.
```
I believe the reason for this is that the source package contains the examples directory because it was put there during package creation (it seems the symlink because a directory). Now, when setup.py is run again, it seems the setup.py attempts to unlink the directory, which is not possible because only symlinks can be unlinked. This PR therefore only attempts to unlink it if it is a symlink. I have not thoroughly tested whether my proposed cause is the true cause, but this should fix it in any case.
Note that the source package is fetched because there is no wheel for Python 3.9, so most users will not see this because they will use the wheel.
## PR review
Anyone in the community is free to review the PR once the tests have passed.
If we didn't discuss your PR in Github issues there's a high chance it will not be merged.
## Did you have fun?
Make sure you had fun coding �
Pull Request resolved: https://github.com/pytorch/fairseq/pull/3235
Reviewed By: alexeib
Differential Revision: D26513259
Pulled By: myleott
fbshipit-source-id: 775d6c636a5867b9983bb6419829f13ee414e2fd
2021-02-20 17:21:45 +03:00
|
|
|
if "build_ext" not in sys.argv[1:] and os.path.islink(fairseq_examples):
|
2021-01-21 18:32:08 +03:00
|
|
|
os.unlink(fairseq_examples)
|