diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 6ae8093a8..a2d44dd57 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -19,26 +19,32 @@ jobs:
     runs-on: ${{ matrix.platform }}
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
+
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Conditionally install pytorch
       if: matrix.platform == 'windows-latest'
       run: pip3 install torch -f https://download.pytorch.org/whl/torch_stable.html
+
     - name: Install locally
       run: |
         python -m pip install --upgrade pip
+        git submodule update --init --recursive
         python setup.py build_ext --inplace
         python -m pip install --editable .
+
     - name: Lint with flake8
       run: |
         pip install flake8
         # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-exclude fairseq/model_parallel/megatron
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --extend-exclude fairseq/model_parallel/megatron
+
     - name: Run tests
       run: |
           python setup.py test
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
new file mode 100644
index 000000000..726170859
--- /dev/null
+++ b/.github/workflows/build_wheels.yml
@@ -0,0 +1,41 @@
+name: build_wheels
+
+on:
+  push:
+    branches:
+      - v[0-9]+.[0-9]+.[x0-9]+
+    tags:
+      - v*
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+
+      - name: Install cibuildwheel
+        run: |
+          python -m pip install cibuildwheel
+
+      - name: Build wheels for CPython
+        run: |
+          python -m cibuildwheel --output-dir dist
+        env:
+          CIBW_BUILD: "cp36-*64 cp37-*64 cp38-*64"
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux1
+          CIBW_BEFORE_BUILD: git submodule update --init --recursive && pip install .
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: wheels
+          path: ./dist/*.whl
diff --git a/fairseq/tasks/audio_pretraining.py b/fairseq/tasks/audio_pretraining.py
index 90e667c80..a2f7edc34 100644
--- a/fairseq/tasks/audio_pretraining.py
+++ b/fairseq/tasks/audio_pretraining.py
@@ -5,7 +5,6 @@
 # the root directory of this source tree. An additional grant of patent rights
 # can be found in the PATENTS file in the same directory.
 
-import editdistance
 import os
 import sys
 import torch
@@ -212,6 +211,8 @@ class AudioPretrainingTask(FairseqTask):
         return model
 
     def _inference_with_wer(self, generator, sample, model):
+        import editdistance
+
         def decode(toks):
             s = self.target_dictionary.string(
                 toks.int().cpu(),
diff --git a/setup.py b/setup.py
index 572d2b50d..2aae720d7 100644
--- a/setup.py
+++ b/setup.py
@@ -132,7 +132,7 @@ if "READTHEDOCS" in os.environ:
 
     # use CPU build of PyTorch
     dependency_links = [
-        "https://download.pytorch.org/whl/cpu/torch-1.3.0%2Bcpu-cp36-cp36m-linux_x86_64.whl"
+        "https://download.pytorch.org/whl/cpu/torch-1.7.0%2Bcpu-cp36-cp36m-linux_x86_64.whl"
     ]
 else:
     dependency_links = []
@@ -149,6 +149,11 @@ if "clean" in sys.argv[1:]:
     )
 
 
+extra_packages = []
+if os.path.exists(os.path.join("fairseq", "model_parallel", "megatron", "mpu")):
+    extra_packages.append("fairseq.model_parallel.megatron.mpu")
+
+
 def do_setup(package_data):
     setup(
         name="fairseq",
@@ -172,7 +177,6 @@ def do_setup(package_data):
             "cffi",
             "cython",
             "dataclasses",
-            "editdistance",
             "hydra-core",
             "numpy",
             "regex",
@@ -190,7 +194,7 @@ def do_setup(package_data):
                 "tests",
                 "tests.*",
             ]
-        ),
+        ) + extra_packages,
         package_data=package_data,
         ext_modules=extensions,
         test_suite="tests",
@@ -223,12 +227,13 @@ def get_files(path, relative_to="fairseq"):
 
 try:
     # symlink examples into fairseq package so package_data accepts them
-    if "build_ext" not in sys.argv[1:]:
-        os.symlink(os.path.join("..", "examples"), "fairseq/examples")
+    fairseq_examples = os.path.join("fairseq", "examples")
+    if "build_ext" not in sys.argv[1:] and not os.path.exists(fairseq_examples):
+        os.symlink(os.path.join("..", "examples"), fairseq_examples)
     package_data = {
         "fairseq": get_files("fairseq/examples"),
     }
     do_setup(package_data)
 finally:
-    if "build_ext" not in sys.argv[1:]:
-        os.unlink("fairseq/examples")
+    if "build_ext" not in sys.argv[1:] and os.path.exists(fairseq_examples):
+        os.unlink(fairseq_examples)