mirror of
https://github.com/facebookresearch/fairseq.git
synced 2024-09-11 17:25:31 +03:00
Rename data.transforms -> data.encoders
Summary: Pull Request resolved: https://github.com/fairinternal/fairseq-py/pull/747 Differential Revision: D16403464 Pulled By: myleott fbshipit-source-id: ee3b4184f129a02be833c7bdc00685978b4de883
This commit is contained in:
parent
69d0f7f826
commit
f812e52956
@ -24,8 +24,8 @@ build_bpe, register_bpe, BPE_REGISTRY = registry.setup_registry(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# automatically import any Python files in the transforms/ directory
|
# automatically import any Python files in the encoders/ directory
|
||||||
for file in os.listdir(os.path.dirname(__file__)):
|
for file in os.listdir(os.path.dirname(__file__)):
|
||||||
if file.endswith('.py') and not file.startswith('_'):
|
if file.endswith('.py') and not file.startswith('_'):
|
||||||
module = file[:file.find('.py')]
|
module = file[:file.find('.py')]
|
||||||
importlib.import_module('fairseq.data.transforms.' + module)
|
importlib.import_module('fairseq.data.encoders.' + module)
|
@ -6,7 +6,7 @@
|
|||||||
# can be found in the PATENTS file in the same directory.
|
# can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
from fairseq import file_utils
|
from fairseq import file_utils
|
||||||
from fairseq.data.transforms import register_bpe
|
from fairseq.data.encoders import register_bpe
|
||||||
|
|
||||||
|
|
||||||
@register_bpe('gpt2')
|
@register_bpe('gpt2')
|
@ -5,7 +5,7 @@
|
|||||||
# the root directory of this source tree. An additional grant of patent rights
|
# the root directory of this source tree. An additional grant of patent rights
|
||||||
# can be found in the PATENTS file in the same directory.
|
# can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
from fairseq.data.transforms import register_tokenizer
|
from fairseq.data.encoders import register_tokenizer
|
||||||
|
|
||||||
|
|
||||||
@register_tokenizer('moses')
|
@register_tokenizer('moses')
|
@ -5,7 +5,7 @@
|
|||||||
# the root directory of this source tree. An additional grant of patent rights
|
# the root directory of this source tree. An additional grant of patent rights
|
||||||
# can be found in the PATENTS file in the same directory.
|
# can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
from fairseq.data.transforms import register_tokenizer
|
from fairseq.data.encoders import register_tokenizer
|
||||||
|
|
||||||
|
|
||||||
@register_tokenizer('nltk')
|
@register_tokenizer('nltk')
|
@ -6,7 +6,7 @@
|
|||||||
# can be found in the PATENTS file in the same directory.
|
# can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
from fairseq import file_utils
|
from fairseq import file_utils
|
||||||
from fairseq.data.transforms import register_bpe
|
from fairseq.data.encoders import register_bpe
|
||||||
|
|
||||||
|
|
||||||
@register_bpe('sentencepiece')
|
@register_bpe('sentencepiece')
|
@ -7,7 +7,7 @@
|
|||||||
|
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from fairseq.data.transforms import register_tokenizer
|
from fairseq.data.encoders import register_tokenizer
|
||||||
|
|
||||||
|
|
||||||
@register_tokenizer('space')
|
@register_tokenizer('space')
|
@ -6,7 +6,7 @@
|
|||||||
# can be found in the PATENTS file in the same directory.
|
# can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
from fairseq import file_utils
|
from fairseq import file_utils
|
||||||
from fairseq.data.transforms import register_bpe
|
from fairseq.data.encoders import register_bpe
|
||||||
|
|
||||||
|
|
||||||
@register_bpe('subword_nmt')
|
@register_bpe('subword_nmt')
|
@ -9,7 +9,7 @@
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
from fairseq import utils
|
from fairseq import utils
|
||||||
from fairseq.data import transforms
|
from fairseq.data import encoders
|
||||||
|
|
||||||
|
|
||||||
class Generator(object):
|
class Generator(object):
|
||||||
@ -44,8 +44,8 @@ class Generator(object):
|
|||||||
# (None if no unknown word replacement, empty if no path to align dictionary)
|
# (None if no unknown word replacement, empty if no path to align dictionary)
|
||||||
self.align_dict = utils.load_align_dict(getattr(args, 'replace_unk', None))
|
self.align_dict = utils.load_align_dict(getattr(args, 'replace_unk', None))
|
||||||
|
|
||||||
self.tokenizer = transforms.build_tokenizer(args)
|
self.tokenizer = encoders.build_tokenizer(args)
|
||||||
self.bpe = transforms.build_bpe(args)
|
self.bpe = encoders.build_bpe(args)
|
||||||
|
|
||||||
def generate(self, src_str, verbose=False):
|
def generate(self, src_str, verbose=False):
|
||||||
|
|
||||||
|
@ -15,7 +15,7 @@ import fileinput
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
from fairseq import checkpoint_utils, options, tasks, utils
|
from fairseq import checkpoint_utils, options, tasks, utils
|
||||||
from fairseq.data import transforms
|
from fairseq.data import encoders
|
||||||
|
|
||||||
|
|
||||||
Batch = namedtuple('Batch', 'ids src_tokens src_lengths')
|
Batch = namedtuple('Batch', 'ids src_tokens src_lengths')
|
||||||
@ -103,8 +103,8 @@ def main(args):
|
|||||||
generator = task.build_generator(args)
|
generator = task.build_generator(args)
|
||||||
|
|
||||||
# Handle tokenization and BPE
|
# Handle tokenization and BPE
|
||||||
tokenizer = transforms.build_tokenizer(args)
|
tokenizer = encoders.build_tokenizer(args)
|
||||||
bpe = transforms.build_bpe(args)
|
bpe = encoders.build_bpe(args)
|
||||||
|
|
||||||
def encode_fn(x):
|
def encode_fn(x):
|
||||||
if tokenizer is not None:
|
if tokenizer is not None:
|
||||||
|
Loading…
Reference in New Issue
Block a user