hasktorch/tokenizers
1
1
Fork 0
mirror of https://github.com/hasktorch/tokenizers.git synced 2024-10-26 06:00:08 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix imports

Browse Source
This commit is contained in:
thomwolf 2020-01-07 22:55:50 +01:00 committed by Anthony MOI
parent 63063118df
commit 6d0e3ba8f1
No known key found for this signature in database
GPG Key ID: CB646B1164C636A0
3 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -15,3 +15,5 @@ pip-wheel-metadata
*.so
/bindings/python/build
/bindings/python/dist
.vscode

4
bindings/python/tokenizers/implementations/base_tokenizer.py
View File

@ -1,6 +1,6 @@
from .tokenizers import Tokenizer
from ..tokenizers import Tokenizer, Encoding
from typing import List, Union, Tuple
from typing import List, Union, Tuple, Optional
class BaseTokenizer:
_tokenizer: Tokenizer

4
bindings/python/tokenizers/implementations/byte_level_bpe.py
View File

@ -1,7 +1,7 @@
from tokenizers import Tokenizer, pre_tokenizers, decoders
from tokenizers.tokenizers import BaseTokenizer
from tokenizers.models import BPE
from tokenizers.normalizers import NFKC
from .base_tokenizer import BaseTokenizer
from typing import Optional
@ -14,7 +14,7 @@ class ByteLevelBpe(BaseTokenizer):
def __init__(self,
vocab_file: Optional[str]=None,
merges_file: Optional[str]=None,
add_prefix_space: boolean=False):
add_prefix_space: bool=False):
if vocab_file is not None and merges_file is not None:
tokenizer = Tokenizer(BPE.from_files(vocab_file, merges_file))
else: