mirror of
https://github.com/facebookresearch/fairseq.git
synced 2024-08-17 04:20:36 +03:00
Add GitHub Action to build Python wheels (+ minor cleanup in build scripts) (#1447)
Summary: Here's an example run in a forked repo: https://github.com/fairseq/fairseq/runs/1419699104 We can upload the wheels to PyPI to make `pip install fairseq` easier for folks. Pull Request resolved: https://github.com/fairinternal/fairseq-py/pull/1447 Reviewed By: lematt1991 Differential Revision: D25060753 Pulled By: myleott fbshipit-source-id: 9fdc28cc7c8a172daac668dd09684ec43e2ff11a
This commit is contained in:
parent
e931009a91
commit
41a61bd4e2
14
.github/workflows/build.yml
vendored
14
.github/workflows/build.yml
vendored
@ -19,26 +19,32 @@ jobs:
|
||||
runs-on: ${{ matrix.platform }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v1
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Conditionally install pytorch
|
||||
if: matrix.platform == 'windows-latest'
|
||||
run: pip3 install torch -f https://download.pytorch.org/whl/torch_stable.html
|
||||
|
||||
- name: Install locally
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
git submodule update --init --recursive
|
||||
python setup.py build_ext --inplace
|
||||
python -m pip install --editable .
|
||||
|
||||
- name: Lint with flake8
|
||||
run: |
|
||||
pip install flake8
|
||||
# stop the build if there are Python syntax errors or undefined names
|
||||
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-exclude fairseq/model_parallel/megatron
|
||||
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
||||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
||||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --extend-exclude fairseq/model_parallel/megatron
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
python setup.py test
|
||||
|
41
.github/workflows/build_wheels.yml
vendored
Normal file
41
.github/workflows/build_wheels.yml
vendored
Normal file
@ -0,0 +1,41 @@
|
||||
name: build_wheels
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- v[0-9]+.[0-9]+.[x0-9]+
|
||||
tags:
|
||||
- v*
|
||||
|
||||
jobs:
|
||||
build_wheels:
|
||||
name: Build wheels on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Install Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.7'
|
||||
|
||||
- name: Install cibuildwheel
|
||||
run: |
|
||||
python -m pip install cibuildwheel
|
||||
|
||||
- name: Build wheels for CPython
|
||||
run: |
|
||||
python -m cibuildwheel --output-dir dist
|
||||
env:
|
||||
CIBW_BUILD: "cp36-*64 cp37-*64 cp38-*64"
|
||||
CIBW_MANYLINUX_X86_64_IMAGE: manylinux1
|
||||
CIBW_BEFORE_BUILD: git submodule update --init --recursive && pip install .
|
||||
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: wheels
|
||||
path: ./dist/*.whl
|
@ -5,7 +5,6 @@
|
||||
# the root directory of this source tree. An additional grant of patent rights
|
||||
# can be found in the PATENTS file in the same directory.
|
||||
|
||||
import editdistance
|
||||
import os
|
||||
import sys
|
||||
import torch
|
||||
@ -212,6 +211,8 @@ class AudioPretrainingTask(FairseqTask):
|
||||
return model
|
||||
|
||||
def _inference_with_wer(self, generator, sample, model):
|
||||
import editdistance
|
||||
|
||||
def decode(toks):
|
||||
s = self.target_dictionary.string(
|
||||
toks.int().cpu(),
|
||||
|
19
setup.py
19
setup.py
@ -132,7 +132,7 @@ if "READTHEDOCS" in os.environ:
|
||||
|
||||
# use CPU build of PyTorch
|
||||
dependency_links = [
|
||||
"https://download.pytorch.org/whl/cpu/torch-1.3.0%2Bcpu-cp36-cp36m-linux_x86_64.whl"
|
||||
"https://download.pytorch.org/whl/cpu/torch-1.7.0%2Bcpu-cp36-cp36m-linux_x86_64.whl"
|
||||
]
|
||||
else:
|
||||
dependency_links = []
|
||||
@ -149,6 +149,11 @@ if "clean" in sys.argv[1:]:
|
||||
)
|
||||
|
||||
|
||||
extra_packages = []
|
||||
if os.path.exists(os.path.join("fairseq", "model_parallel", "megatron", "mpu")):
|
||||
extra_packages.append("fairseq.model_parallel.megatron.mpu")
|
||||
|
||||
|
||||
def do_setup(package_data):
|
||||
setup(
|
||||
name="fairseq",
|
||||
@ -172,7 +177,6 @@ def do_setup(package_data):
|
||||
"cffi",
|
||||
"cython",
|
||||
"dataclasses",
|
||||
"editdistance",
|
||||
"hydra-core",
|
||||
"numpy",
|
||||
"regex",
|
||||
@ -190,7 +194,7 @@ def do_setup(package_data):
|
||||
"tests",
|
||||
"tests.*",
|
||||
]
|
||||
),
|
||||
) + extra_packages,
|
||||
package_data=package_data,
|
||||
ext_modules=extensions,
|
||||
test_suite="tests",
|
||||
@ -223,12 +227,13 @@ def get_files(path, relative_to="fairseq"):
|
||||
|
||||
try:
|
||||
# symlink examples into fairseq package so package_data accepts them
|
||||
if "build_ext" not in sys.argv[1:]:
|
||||
os.symlink(os.path.join("..", "examples"), "fairseq/examples")
|
||||
fairseq_examples = os.path.join("fairseq", "examples")
|
||||
if "build_ext" not in sys.argv[1:] and not os.path.exists(fairseq_examples):
|
||||
os.symlink(os.path.join("..", "examples"), fairseq_examples)
|
||||
package_data = {
|
||||
"fairseq": get_files("fairseq/examples"),
|
||||
}
|
||||
do_setup(package_data)
|
||||
finally:
|
||||
if "build_ext" not in sys.argv[1:]:
|
||||
os.unlink("fairseq/examples")
|
||||
if "build_ext" not in sys.argv[1:] and os.path.exists(fairseq_examples):
|
||||
os.unlink(fairseq_examples)
|
||||
|
Loading…
Reference in New Issue
Block a user