mirror of
https://github.com/facebookresearch/fairseq.git
synced 2024-09-21 22:27:16 +03:00
7d2394b56f
Summary:
Pull Request resolved: https://github.com/fairinternal/fairseq-py/pull/1669
Unit tests for async writes integration done in D26467815 (3100d0b8e5
).
Ongoing performance tests: https://fb.quip.com/kjM7Atb1kKbO
Reviewed By: myleott
Differential Revision: D26732660
fbshipit-source-id: faf8cac67b9167af4195358c1a2592804c13562c
63 lines
1.9 KiB
Python
63 lines
1.9 KiB
Python
# This source code is licensed under the MIT license found in the
|
|
# LICENSE file in the root directory of this source tree.
|
|
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from typing import Optional
|
|
from unittest.mock import MagicMock
|
|
|
|
|
|
class TestFileIO(unittest.TestCase):
|
|
|
|
_tmpdir: Optional[str] = None
|
|
_tmpfile: Optional[str] = None
|
|
_tmpfile_contents = "Hello, World"
|
|
|
|
@classmethod
|
|
def setUpClass(cls) -> None:
|
|
cls._tmpdir = tempfile.mkdtemp()
|
|
with open(os.path.join(cls._tmpdir, "test.txt"), "w") as f:
|
|
cls._tmpfile = f.name
|
|
f.write(cls._tmpfile_contents)
|
|
f.flush()
|
|
|
|
@classmethod
|
|
def tearDownClass(cls) -> None:
|
|
# Cleanup temp working dir.
|
|
if cls._tmpdir is not None:
|
|
shutil.rmtree(cls._tmpdir) # type: ignore
|
|
|
|
def test_file_io(self):
|
|
from fairseq.file_io import PathManager
|
|
|
|
with PathManager.open(os.path.join(self._tmpdir, "test.txt"), "r") as f:
|
|
s = f.read()
|
|
self.assertEqual(s, self._tmpfile_contents)
|
|
|
|
def test_file_io_oss(self):
|
|
# Mock fvcore to simulate oss environment.
|
|
sys.modules["fvcore"] = MagicMock()
|
|
from fairseq.file_io import PathManager
|
|
|
|
with PathManager.open(os.path.join(self._tmpdir, "test.txt"), "r") as f:
|
|
s = f.read()
|
|
self.assertEqual(s, self._tmpfile_contents)
|
|
|
|
def test_file_io_async(self):
|
|
# ioPath `PathManager` is initialized after the first `opena` call.
|
|
try:
|
|
from fairseq.file_io import IOPathPathManager, PathManager
|
|
|
|
self.assertIsNone(IOPathPathManager)
|
|
_asyncfile = os.path.join(self._tmpdir, "async.txt")
|
|
f = PathManager.opena(_asyncfile, "wb")
|
|
f.close()
|
|
|
|
from fairseq.file_io import IOPathPathManager
|
|
self.assertIsNotNone(IOPathPathManager)
|
|
finally:
|
|
self.assertTrue(PathManager.async_close())
|