Build Cython components when loading hub (#1386)

Summary:
Fixes https://github.com/pytorch/fairseq/issues/1376
Pull Request resolved: https://github.com/pytorch/fairseq/pull/1386

Differential Revision: D18566839

Pulled By: myleott

fbshipit-source-id: 71805f58fab90f53f757bf4ef69eb914195af38a

hubconf.py

@@ -18,6 +18,26 @@ dependencies = [
 ]
 
 
+# torch.hub doesn't build Cython components, so if they are not found then try
+# to build them here
+try:
+    import fairseq.data.token_block_utils_fast
+except (ImportError, ModuleNotFoundError):
+    try:
+        import cython
+        import os
+        from setuptools import sandbox
+        sandbox.run_setup(
+            os.path.join(os.path.dirname(__file__), 'setup.py'),
+            ['build_ext', '--inplace'],
+        )
+    except (ImportError, ModuleNotFoundError):
+        print(
+            'Unable to build Cython components. Please make sure Cython is '
+            'installed if the torch.hub model you are loading depends on it.'
+        )
+
+
 for _model_type, _cls in MODEL_REGISTRY.items():
     for model_name in _cls.hub_models().keys():
         globals()[model_name] = functools.partial(