google/sentencepiece
1
1
Fork 0
mirror of https://github.com/google/sentencepiece.git synced 2025-01-08 18:26:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

Uses property in immutable proto

Browse Source
This commit is contained in:
Taku Kudo 2022-08-04 16:03:31 +09:00
parent 497ee76bd9
commit b738153dd7
5 changed files with 271 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
python/setup.py
View File

@ -96,7 +96,6 @@ class build_ext(_build_ext):
else:
cflags.append('-Wl,-strip-all')
libs.append('-Wl,-strip-all')
cflags.append('-DSWIGPYTHON')
print('## cflags={}'.format(' '.join(cflags)))
print('## libs={}'.format(' '.join(libs)))
ext.extra_compile_args = cflags
@ -116,7 +115,7 @@ if os.name == 'nt':
'..\\build\\root_{}\\lib\\sentencepiece_train.lib'.format(arch)
]
else:
cflags = ['/std:c++17', '/MT', '/I..\\build\\root\\include', '/DSWIGPYTHON']
cflags = ['/std:c++17', '/MT', '/I..\\build\\root\\include']
libs = [
'..\\build\\root\\lib\\sentencepiece.lib',
'..\\build\\root\\lib\\sentencepiece_train.lib'

140
python/src/sentencepiece/__init__.py
View File

@ -69,20 +69,36 @@ class ImmutableSentencePieceText_ImmutableSentencePiece(object):
_sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece_swiginit(self, _sentencepiece.new_ImmutableSentencePieceText_ImmutableSentencePiece())
__swig_destroy__ = _sentencepiece.delete_ImmutableSentencePieceText_ImmutableSentencePiece
def piece(self):
return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece_piece(self)
def _piece(self):
return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__piece(self)
def surface(self):
return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece_surface(self)
def _surface(self):
return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__surface(self)
def id(self):
return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece_id(self)
def _id(self):
return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__id(self)
def begin(self):
return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece_begin(self)
def _begin(self):
return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__begin(self)
def _end(self):
return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__end(self)
piece = property(_piece)
surface = property(_surface)
id = property(_id)
begin = property(_begin)
end = property(_end)
def __str__(self):
return ('piece: \"{}\"\n'
'id: {}\n'
'surface: \"{}\"\n'
'begin: {}\n'
'end: {}\n').format(self.piece, self.id, self.surface,
self.begin, self.end)
__repr__ = __str__
def end(self):
return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece_end(self)
# Register ImmutableSentencePieceText_ImmutableSentencePiece in _sentencepiece:
_sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece_swigregister(ImmutableSentencePieceText_ImmutableSentencePiece)
@ -95,32 +111,45 @@ class ImmutableSentencePieceText(object):
_sentencepiece.ImmutableSentencePieceText_swiginit(self, _sentencepiece.new_ImmutableSentencePieceText())
__swig_destroy__ = _sentencepiece.delete_ImmutableSentencePieceText
def pieces_size(self):
return _sentencepiece.ImmutableSentencePieceText_pieces_size(self)
def pieces(self, index):
return _sentencepiece.ImmutableSentencePieceText_pieces(self, index)
def text(self):
return _sentencepiece.ImmutableSentencePieceText_text(self)
def score(self):
return _sentencepiece.ImmutableSentencePieceText_score(self)
def SerializeAsString(self):
return _sentencepiece.ImmutableSentencePieceText_SerializeAsString(self)
def _pieces_size(self):
return _sentencepiece.ImmutableSentencePieceText__pieces_size(self)
def _pieces(self, index):
return _sentencepiece.ImmutableSentencePieceText__pieces(self, index)
def pieces(self, i):
return self._pieces(i)
def _text(self):
return _sentencepiece.ImmutableSentencePieceText__text(self)
def __len__(self):
return self.pieces_size()
def _score(self):
return _sentencepiece.ImmutableSentencePieceText__score(self)
def __getitem__(self, i):
return self._pieces(i)
def SerializeAsString(self):
return _sentencepiece.ImmutableSentencePieceText_SerializeAsString(self)
text = property(_text)
score = property(_score)
class ImmutableSentencePieceIterator:
def __init__(self, proto):
self.proto = proto
self.len = self.proto._pieces_size()
def __len__(self):
return self.len
def __getitem__(self, index):
if index < 0 or index >= self.len:
raise IndexError('piece index is out of range')
return self.proto._pieces(index)
def __str__(self):
return '\n'.join(['pieces {{\n{}}}'.format(str(x)) for x in self])
__repr__ = __str__
@property
def pieces(self):
return ImmutableSentencePieceText.ImmutableSentencePieceIterator(self)
def __eq__(self, other):
return self.SerializeAsString() == other.SerializeAsString()
@ -128,6 +157,14 @@ class ImmutableSentencePieceText(object):
def __hash__(self):
return hash(self.SerializeAsString())
def __str__(self):
return ('text: \"{}\"\n'
'score: {}\n'
'{}').format(self.text, self.score,
'\n'.join(['pieces {{\n{}}}'.format(str(x)) for x in self.pieces]))
__repr__ = __str__
# Register ImmutableSentencePieceText in _sentencepiece:
_sentencepiece.ImmutableSentencePieceText_swigregister(ImmutableSentencePieceText)
@ -140,26 +177,36 @@ class ImmutableNBestSentencePieceText(object):
_sentencepiece.ImmutableNBestSentencePieceText_swiginit(self, _sentencepiece.new_ImmutableNBestSentencePieceText())
__swig_destroy__ = _sentencepiece.delete_ImmutableNBestSentencePieceText
def nbests_size(self):
return _sentencepiece.ImmutableNBestSentencePieceText_nbests_size(self)
def nbests(self, index):
return _sentencepiece.ImmutableNBestSentencePieceText_nbests(self, index)
def SerializeAsString(self):
return _sentencepiece.ImmutableNBestSentencePieceText_SerializeAsString(self)
def _nbests_size(self):
return _sentencepiece.ImmutableNBestSentencePieceText__nbests_size(self)
def _nbests(self, index):
return _sentencepiece.ImmutableNBestSentencePieceText__nbests(self, index)
def __nbests__(self, i):
return self._nbests(i)
def SerializeAsString(self):
return _sentencepiece.ImmutableNBestSentencePieceText_SerializeAsString(self)
def __len__(self):
return self.nbests_size()
class ImmutableSentencePieceTextIterator:
def __init__(self, proto):
self.proto = proto
self.len = self.proto._nbests_size()
def __getitem__(self, i):
return self._nbests(i)
def __len__(self):
return self.len
def __getitem__(self, index):
if index < 0 or index >= self.len:
raise IndexError('nbests index is out of range')
return self.proto._nbests(index)
def __str__(self):
return '\n'.join(['nbests {{\n{}}}'.format(str(x)) for x in self])
__repr__ = __str__
@property
def nbests(self):
return ImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator(self)
def __eq__(self, other):
return self.SerializeAsString() == other.SerializeAsString()
@ -167,6 +214,11 @@ class ImmutableNBestSentencePieceText(object):
def __hash__(self):
return hash(self.SerializeAsString())
def __str__(self):
return '\n'.join(['nbests {{\n{}}}'.format(str(x)) for x in self.nbests])
__repr__ = __str__
# Register ImmutableNBestSentencePieceText in _sentencepiece:
_sentencepiece.ImmutableNBestSentencePieceText_swigregister(ImmutableNBestSentencePieceText)

143
python/src/sentencepiece/sentencepiece.i
View File

@ -1239,60 +1239,117 @@ inline void InitNumThreads(const std::vector<T> &ins, int *num_threads) {
}
}
%extend sentencepiece::ImmutableSentencePieceText {
ImmutableSentencePieceText_ImmutableSentencePiece _pieces(int index) const {
if (index < 0 || index >= static_cast<int>($self->pieces_size())) {
throw sentencepiece::util::Status(
sentencepiece::util::StatusCode::kOutOfRange,
"piece index is out of range.");
}
return $self->pieces(index);
}
%extend sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece {
%rename(_piece) piece;
%rename(_id) id;
%rename(_surface) surface;
%rename(_begin) begin;
%rename(_end) end;
%pythoncode {
def pieces(self, i):
return self._pieces(i)
%pythoncode %{
piece = property(_piece)
surface = property(_surface)
id = property(_id)
begin = property(_begin)
end = property(_end)
def __len__(self):
return self.pieces_size()
def __getitem__(self, i):
return self._pieces(i)
def __eq__(self, other):
return self.SerializeAsString() == other.SerializeAsString()
def __hash__(self):
return hash(self.SerializeAsString())
def __str__(self):
return ('piece: \"{}\"\n'
'id: {}\n'
'surface: \"{}\"\n'
'begin: {}\n'
'end: {}\n').format(self.piece, self.id, self.surface,
self.begin, self.end)
__repr__ = __str__
%}
}
%extend sentencepiece::ImmutableSentencePieceText {
%rename(_text) text;
%rename(_score) score;
%rename(_pieces) pieces;
%rename(_pieces_size) pieces_size;
%pythoncode %{
text = property(_text)
score = property(_score)
class ImmutableSentencePieceIterator:
def __init__(self, proto):
self.proto = proto
self.len = self.proto._pieces_size()
def __len__(self):
return self.len
def __getitem__(self, index):
if index < 0 or index >= self.len:
raise IndexError('piece index is out of range')
return self.proto._pieces(index)
def __str__(self):
return '\n'.join(['pieces {{\n{}}}'.format(str(x)) for x in self])
__repr__ = __str__
@property
def pieces(self):
return ImmutableSentencePieceText.ImmutableSentencePieceIterator(self)
def __eq__(self, other):
return self.SerializeAsString() == other.SerializeAsString()
def __hash__(self):
return hash(self.SerializeAsString())
def __str__(self):
return ('text: \"{}\"\n'
'score: {}\n'
'{}').format(self.text, self.score,
'\n'.join(['pieces {{\n{}}}'.format(str(x)) for x in self.pieces]))
__repr__ = __str__
%}
}
%extend sentencepiece::ImmutableNBestSentencePieceText {
ImmutableSentencePieceText _nbests(int index) const {
if (index < 0 || index >= static_cast<int>($self->nbests_size())) {
throw sentencepiece::util::Status(
sentencepiece::util::StatusCode::kOutOfRange,
"nbest index is out of range.");
}
return $self->nbests(index);
}
%rename(_nbests) nbests;
%rename(_nbests_size) nbests_size;
%pythoncode {
def __nbests__(self, i):
return self._nbests(i)
%pythoncode %{
class ImmutableSentencePieceTextIterator:
def __init__(self, proto):
self.proto = proto
self.len = self.proto._nbests_size()
def __len__(self):
return self.nbests_size()
def __len__(self):
return self.len
def __getitem__(self, i):
return self._nbests(i)
def __getitem__(self, index):
if index < 0 or index >= self.len:
raise IndexError('nbests index is out of range')
return self.proto._nbests(index)
def __eq__(self, other):
return self.SerializeAsString() == other.SerializeAsString()
def __str__(self):
return '\n'.join(['nbests {{\n{}}}'.format(str(x)) for x in self])
def __hash__(self):
return hash(self.SerializeAsString())
}
__repr__ = __str__
@property
def nbests(self):
return ImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator(self)
def __eq__(self, other):
return self.SerializeAsString() == other.SerializeAsString()
def __hash__(self):
return hash(self.SerializeAsString())
def __str__(self):
return '\n'.join(['nbests {{\n{}}}'.format(str(x)) for x in self.nbests])
__repr__ = __str__
%}
}
%typemap(out) std::vector<int> {

168
python/src/sentencepiece/sentencepiece_wrap.cxx
View File

@ -3299,22 +3299,6 @@ SWIG_From_float (float value)
return SWIG_From_double (value);
}
SWIGINTERN sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece sentencepiece_ImmutableSentencePieceText__pieces(sentencepiece::ImmutableSentencePieceText const *self,int index){
if (index < 0 || index >= static_cast<int>(self->pieces_size())) {
throw sentencepiece::util::Status(
sentencepiece::util::StatusCode::kOutOfRange,
"piece index is out of range.");
}
return self->pieces(index);
}
SWIGINTERN sentencepiece::ImmutableSentencePieceText sentencepiece_ImmutableNBestSentencePieceText__nbests(sentencepiece::ImmutableNBestSentencePieceText const *self,int index){
if (index < 0 || index >= static_cast<int>(self->nbests_size())) {
throw sentencepiece::util::Status(
sentencepiece::util::StatusCode::kOutOfRange,
"nbest index is out of range.");
}
return self->nbests(index);
}
SWIGINTERN swig_type_info*
SWIG_pchar_descriptor(void)
@ -3846,7 +3830,7 @@ fail:
}
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece_piece(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece__piece(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *arg1 = (sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *) 0 ;
void *argp1 = 0 ;
@ -3858,7 +3842,7 @@ SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece_pie
swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, 0 | 0 );
if (!SWIG_IsOK(res1)) {
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece_piece" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'");
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece__piece" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'");
}
arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece * >(argp1);
{
@ -3880,7 +3864,7 @@ fail:
}
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece_surface(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece__surface(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *arg1 = (sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *) 0 ;
void *argp1 = 0 ;
@ -3892,7 +3876,7 @@ SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece_sur
swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, 0 | 0 );
if (!SWIG_IsOK(res1)) {
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece_surface" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'");
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece__surface" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'");
}
arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece * >(argp1);
{
@ -3914,7 +3898,7 @@ fail:
}
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece_id(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece__id(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *arg1 = (sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *) 0 ;
void *argp1 = 0 ;
@ -3926,7 +3910,7 @@ SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece_id(
swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, 0 | 0 );
if (!SWIG_IsOK(res1)) {
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece_id" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'");
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece__id" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'");
}
arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece * >(argp1);
{
@ -3945,7 +3929,7 @@ fail:
}
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece_begin(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece__begin(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *arg1 = (sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *) 0 ;
void *argp1 = 0 ;
@ -3957,7 +3941,7 @@ SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece_beg
swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, 0 | 0 );
if (!SWIG_IsOK(res1)) {
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece_begin" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'");
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece__begin" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'");
}
arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece * >(argp1);
{
@ -3976,7 +3960,7 @@ fail:
}
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece_end(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece__end(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *arg1 = (sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *) 0 ;
void *argp1 = 0 ;
@ -3988,7 +3972,7 @@ SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece_end
swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, 0 | 0 );
if (!SWIG_IsOK(res1)) {
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece_end" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'");
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece__end" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'");
}
arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece * >(argp1);
{
@ -4069,7 +4053,7 @@ fail:
}
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_pieces_size(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText__pieces_size(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::ImmutableSentencePieceText *arg1 = (sentencepiece::ImmutableSentencePieceText *) 0 ;
void *argp1 = 0 ;
@ -4081,7 +4065,7 @@ SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_pieces_size(PyObject *SWIG
swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, 0 | 0 );
if (!SWIG_IsOK(res1)) {
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_pieces_size" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'");
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText__pieces_size" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'");
}
arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText * >(argp1);
{
@ -4100,7 +4084,7 @@ fail:
}
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_pieces(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText__pieces(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::ImmutableSentencePieceText *arg1 = (sentencepiece::ImmutableSentencePieceText *) 0 ;
int arg2 ;
@ -4111,15 +4095,15 @@ SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_pieces(PyObject *SWIGUNUSE
PyObject *swig_obj[2] ;
sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece result;
if (!SWIG_Python_UnpackTuple(args, "ImmutableSentencePieceText_pieces", 2, 2, swig_obj)) SWIG_fail;
if (!SWIG_Python_UnpackTuple(args, "ImmutableSentencePieceText__pieces", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, 0 | 0 );
if (!SWIG_IsOK(res1)) {
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_pieces" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'");
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText__pieces" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'");
}
arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText * >(argp1);
ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
if (!SWIG_IsOK(ecode2)) {
SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ImmutableSentencePieceText_pieces" "', argument " "2"" of type '" "int""'");
SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ImmutableSentencePieceText__pieces" "', argument " "2"" of type '" "int""'");
}
arg2 = static_cast< int >(val2);
{
@ -4138,7 +4122,7 @@ fail:
}
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_text(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText__text(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::ImmutableSentencePieceText *arg1 = (sentencepiece::ImmutableSentencePieceText *) 0 ;
void *argp1 = 0 ;
@ -4150,7 +4134,7 @@ SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_text(PyObject *SWIGUNUSEDP
swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, 0 | 0 );
if (!SWIG_IsOK(res1)) {
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_text" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'");
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText__text" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'");
}
arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText * >(argp1);
{
@ -4172,7 +4156,7 @@ fail:
}
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_score(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText__score(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::ImmutableSentencePieceText *arg1 = (sentencepiece::ImmutableSentencePieceText *) 0 ;
void *argp1 = 0 ;
@ -4184,7 +4168,7 @@ SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_score(PyObject *SWIGUNUSED
swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, 0 | 0 );
if (!SWIG_IsOK(res1)) {
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_score" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'");
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText__score" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'");
}
arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText * >(argp1);
{
@ -4236,44 +4220,6 @@ fail:
}
SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText__pieces(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::ImmutableSentencePieceText *arg1 = (sentencepiece::ImmutableSentencePieceText *) 0 ;
int arg2 ;
void *argp1 = 0 ;
int res1 = 0 ;
int val2 ;
int ecode2 = 0 ;
PyObject *swig_obj[2] ;
sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece result;
if (!SWIG_Python_UnpackTuple(args, "ImmutableSentencePieceText__pieces", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, 0 | 0 );
if (!SWIG_IsOK(res1)) {
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText__pieces" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'");
}
arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText * >(argp1);
ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
if (!SWIG_IsOK(ecode2)) {
SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ImmutableSentencePieceText__pieces" "', argument " "2"" of type '" "int""'");
}
arg2 = static_cast< int >(val2);
{
try {
result = sentencepiece_ImmutableSentencePieceText__pieces((sentencepiece::ImmutableSentencePieceText const *)arg1,arg2);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
resultobj = SWIG_NewPointerObj((new sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece(static_cast< const sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece& >(result))), SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, SWIG_POINTER_OWN | 0 );
return resultobj;
fail:
return NULL;
}
SWIGINTERN PyObject *ImmutableSentencePieceText_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *obj;
if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
@ -4336,7 +4282,7 @@ fail:
}
SWIGINTERN PyObject *_wrap_ImmutableNBestSentencePieceText_nbests_size(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
SWIGINTERN PyObject *_wrap_ImmutableNBestSentencePieceText__nbests_size(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::ImmutableNBestSentencePieceText *arg1 = (sentencepiece::ImmutableNBestSentencePieceText *) 0 ;
void *argp1 = 0 ;
@ -4348,7 +4294,7 @@ SWIGINTERN PyObject *_wrap_ImmutableNBestSentencePieceText_nbests_size(PyObject
swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableNBestSentencePieceText, 0 | 0 );
if (!SWIG_IsOK(res1)) {
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableNBestSentencePieceText_nbests_size" "', argument " "1"" of type '" "sentencepiece::ImmutableNBestSentencePieceText const *""'");
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableNBestSentencePieceText__nbests_size" "', argument " "1"" of type '" "sentencepiece::ImmutableNBestSentencePieceText const *""'");
}
arg1 = reinterpret_cast< sentencepiece::ImmutableNBestSentencePieceText * >(argp1);
{
@ -4367,7 +4313,7 @@ fail:
}
SWIGINTERN PyObject *_wrap_ImmutableNBestSentencePieceText_nbests(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
SWIGINTERN PyObject *_wrap_ImmutableNBestSentencePieceText__nbests(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::ImmutableNBestSentencePieceText *arg1 = (sentencepiece::ImmutableNBestSentencePieceText *) 0 ;
int arg2 ;
@ -4378,15 +4324,15 @@ SWIGINTERN PyObject *_wrap_ImmutableNBestSentencePieceText_nbests(PyObject *SWIG
PyObject *swig_obj[2] ;
sentencepiece::ImmutableSentencePieceText result;
if (!SWIG_Python_UnpackTuple(args, "ImmutableNBestSentencePieceText_nbests", 2, 2, swig_obj)) SWIG_fail;
if (!SWIG_Python_UnpackTuple(args, "ImmutableNBestSentencePieceText__nbests", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableNBestSentencePieceText, 0 | 0 );
if (!SWIG_IsOK(res1)) {
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableNBestSentencePieceText_nbests" "', argument " "1"" of type '" "sentencepiece::ImmutableNBestSentencePieceText const *""'");
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableNBestSentencePieceText__nbests" "', argument " "1"" of type '" "sentencepiece::ImmutableNBestSentencePieceText const *""'");
}
arg1 = reinterpret_cast< sentencepiece::ImmutableNBestSentencePieceText * >(argp1);
ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
if (!SWIG_IsOK(ecode2)) {
SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ImmutableNBestSentencePieceText_nbests" "', argument " "2"" of type '" "int""'");
SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ImmutableNBestSentencePieceText__nbests" "', argument " "2"" of type '" "int""'");
}
arg2 = static_cast< int >(val2);
{
@ -4438,44 +4384,6 @@ fail:
}
SWIGINTERN PyObject *_wrap_ImmutableNBestSentencePieceText__nbests(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::ImmutableNBestSentencePieceText *arg1 = (sentencepiece::ImmutableNBestSentencePieceText *) 0 ;
int arg2 ;
void *argp1 = 0 ;
int res1 = 0 ;
int val2 ;
int ecode2 = 0 ;
PyObject *swig_obj[2] ;
sentencepiece::ImmutableSentencePieceText result;
if (!SWIG_Python_UnpackTuple(args, "ImmutableNBestSentencePieceText__nbests", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableNBestSentencePieceText, 0 | 0 );
if (!SWIG_IsOK(res1)) {
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableNBestSentencePieceText__nbests" "', argument " "1"" of type '" "sentencepiece::ImmutableNBestSentencePieceText const *""'");
}
arg1 = reinterpret_cast< sentencepiece::ImmutableNBestSentencePieceText * >(argp1);
ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
if (!SWIG_IsOK(ecode2)) {
SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ImmutableNBestSentencePieceText__nbests" "', argument " "2"" of type '" "int""'");
}
arg2 = static_cast< int >(val2);
{
try {
result = sentencepiece_ImmutableNBestSentencePieceText__nbests((sentencepiece::ImmutableNBestSentencePieceText const *)arg1,arg2);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
resultobj = SWIG_NewPointerObj((new sentencepiece::ImmutableSentencePieceText(static_cast< const sentencepiece::ImmutableSentencePieceText& >(result))), SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, SWIG_POINTER_OWN | 0 );
return resultobj;
fail:
return NULL;
}
SWIGINTERN PyObject *ImmutableNBestSentencePieceText_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *obj;
if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
@ -8475,29 +8383,27 @@ static PyMethodDef SwigMethods[] = {
{ "SWIG_PyInstanceMethod_New", SWIG_PyInstanceMethod_New, METH_O, NULL},
{ "new_ImmutableSentencePieceText_ImmutableSentencePiece", _wrap_new_ImmutableSentencePieceText_ImmutableSentencePiece, METH_NOARGS, NULL},
{ "delete_ImmutableSentencePieceText_ImmutableSentencePiece", _wrap_delete_ImmutableSentencePieceText_ImmutableSentencePiece, METH_O, NULL},
{ "ImmutableSentencePieceText_ImmutableSentencePiece_piece", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece_piece, METH_O, NULL},
{ "ImmutableSentencePieceText_ImmutableSentencePiece_surface", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece_surface, METH_O, NULL},
{ "ImmutableSentencePieceText_ImmutableSentencePiece_id", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece_id, METH_O, NULL},
{ "ImmutableSentencePieceText_ImmutableSentencePiece_begin", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece_begin, METH_O, NULL},
{ "ImmutableSentencePieceText_ImmutableSentencePiece_end", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece_end, METH_O, NULL},
{ "ImmutableSentencePieceText_ImmutableSentencePiece__piece", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece__piece, METH_O, NULL},
{ "ImmutableSentencePieceText_ImmutableSentencePiece__surface", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece__surface, METH_O, NULL},
{ "ImmutableSentencePieceText_ImmutableSentencePiece__id", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece__id, METH_O, NULL},
{ "ImmutableSentencePieceText_ImmutableSentencePiece__begin", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece__begin, METH_O, NULL},
{ "ImmutableSentencePieceText_ImmutableSentencePiece__end", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece__end, METH_O, NULL},
{ "ImmutableSentencePieceText_ImmutableSentencePiece_swigregister", ImmutableSentencePieceText_ImmutableSentencePiece_swigregister, METH_O, NULL},
{ "ImmutableSentencePieceText_ImmutableSentencePiece_swiginit", ImmutableSentencePieceText_ImmutableSentencePiece_swiginit, METH_VARARGS, NULL},
{ "new_ImmutableSentencePieceText", _wrap_new_ImmutableSentencePieceText, METH_NOARGS, NULL},
{ "delete_ImmutableSentencePieceText", _wrap_delete_ImmutableSentencePieceText, METH_O, NULL},
{ "ImmutableSentencePieceText_pieces_size", _wrap_ImmutableSentencePieceText_pieces_size, METH_O, NULL},
{ "ImmutableSentencePieceText_pieces", _wrap_ImmutableSentencePieceText_pieces, METH_VARARGS, NULL},
{ "ImmutableSentencePieceText_text", _wrap_ImmutableSentencePieceText_text, METH_O, NULL},
{ "ImmutableSentencePieceText_score", _wrap_ImmutableSentencePieceText_score, METH_O, NULL},
{ "ImmutableSentencePieceText_SerializeAsString", _wrap_ImmutableSentencePieceText_SerializeAsString, METH_O, NULL},
{ "ImmutableSentencePieceText__pieces_size", _wrap_ImmutableSentencePieceText__pieces_size, METH_O, NULL},
{ "ImmutableSentencePieceText__pieces", _wrap_ImmutableSentencePieceText__pieces, METH_VARARGS, NULL},
{ "ImmutableSentencePieceText__text", _wrap_ImmutableSentencePieceText__text, METH_O, NULL},
{ "ImmutableSentencePieceText__score", _wrap_ImmutableSentencePieceText__score, METH_O, NULL},
{ "ImmutableSentencePieceText_SerializeAsString", _wrap_ImmutableSentencePieceText_SerializeAsString, METH_O, NULL},
{ "ImmutableSentencePieceText_swigregister", ImmutableSentencePieceText_swigregister, METH_O, NULL},
{ "ImmutableSentencePieceText_swiginit", ImmutableSentencePieceText_swiginit, METH_VARARGS, NULL},
{ "new_ImmutableNBestSentencePieceText", _wrap_new_ImmutableNBestSentencePieceText, METH_NOARGS, NULL},
{ "delete_ImmutableNBestSentencePieceText", _wrap_delete_ImmutableNBestSentencePieceText, METH_O, NULL},
{ "ImmutableNBestSentencePieceText_nbests_size", _wrap_ImmutableNBestSentencePieceText_nbests_size, METH_O, NULL},
{ "ImmutableNBestSentencePieceText_nbests", _wrap_ImmutableNBestSentencePieceText_nbests, METH_VARARGS, NULL},
{ "ImmutableNBestSentencePieceText_SerializeAsString", _wrap_ImmutableNBestSentencePieceText_SerializeAsString, METH_O, NULL},
{ "ImmutableNBestSentencePieceText__nbests_size", _wrap_ImmutableNBestSentencePieceText__nbests_size, METH_O, NULL},
{ "ImmutableNBestSentencePieceText__nbests", _wrap_ImmutableNBestSentencePieceText__nbests, METH_VARARGS, NULL},
{ "ImmutableNBestSentencePieceText_SerializeAsString", _wrap_ImmutableNBestSentencePieceText_SerializeAsString, METH_O, NULL},
{ "ImmutableNBestSentencePieceText_swigregister", ImmutableNBestSentencePieceText_swigregister, METH_O, NULL},
{ "ImmutableNBestSentencePieceText_swiginit", ImmutableNBestSentencePieceText_swiginit, METH_VARARGS, NULL},
{ "new_SentencePieceProcessor", _wrap_new_SentencePieceProcessor, METH_NOARGS, NULL},

68
python/test/sentencepiece_test.py
View File

@ -305,6 +305,12 @@ class TestSentencepieceProcessor(unittest.TestCase):
s4 = self.sp_.DecodePiecesAsImmutableProto(['foo', 'bar'])
s5 = self.sp_.DecodeIdsAsImmutableProto([20, 30])
print(s1)
print(s2)
print(s3)
print(s4)
print(s5)
t1 = self.sp_.encode_as_immutable_proto(text)
t2 = self.sp_.sample_encode_as_immutable_proto(text, 10, 0.2)
t3 = self.sp_.nbest_encode_as_immutable_proto(text, 10)
@ -339,35 +345,35 @@ class TestSentencepieceProcessor(unittest.TestCase):
v1 = self.sp_.EncodeAsIds(text)
v2 = self.sp_.EncodeAsPieces(text)
self.assertEqual([x.id() for x in s1], v1)
self.assertEqual([x.piece() for x in s1], v2)
self.assertEqual(text, s1.text())
self.assertEqual([x.id for x in s1.pieces], v1)
self.assertEqual([x.piece for x in s1.pieces], v2)
self.assertEqual(text, s1.text)
surfaces1 = [s1.text()[x.begin():x.end()] for x in s1]
surfaces2 = [x.surface() for x in s1]
surfaces1 = [s1.text[x.begin:x.end] for x in s1.pieces]
surfaces2 = [x.surface for x in s1.pieces]
self.assertEqual(surfaces1, surfaces2)
ids = []
for i in range(s1.pieces_size()):
ids.append(s1.pieces(i).id())
for i in range(len(s1.pieces)):
ids.append(s1.pieces[i].id)
self.assertEqual(ids, v1)
pieces = []
for i in range(s1.pieces_size()):
pieces.append(s1.pieces(i).piece())
for i in range(len(s1.pieces)):
pieces.append(s1.pieces[i].piece)
self.assertEqual(pieces, v2)
# Japanese offset
s1 = self.jasp_.EncodeAsImmutableProto('吾輩は猫である。Hello world. ABC 123')
surfaces1 = [s1.text()[x.begin():x.end()] for x in s1]
surfaces2 = [x.surface() for x in s1]
surfaces1 = [s1.text[x.begin:x.end] for x in s1.pieces]
surfaces2 = [x.surface for x in s1.pieces]
self.assertEqual(surfaces1, surfaces2)
ids = [x.id() for x in s1]
ids = [x.id for x in s1.pieces]
s2 = self.jasp_.DecodeIdsAsImmutableProto(ids)
self.assertEqual(s2, s1)
pieces = [x.piece() for x in s1]
pieces = [x.piece for x in s1.pieces]
s2 = self.jasp_.DecodePiecesAsImmutableProto(pieces)
self.assertEqual(s2, s1)
@ -395,29 +401,29 @@ class TestSentencepieceProcessor(unittest.TestCase):
self.assertEqual(sp.encode([text], out_type='serialized_proto'), [sprotos])
self.assertEqual(sp.encode([text], out_type='immutable_proto'), [iprotos])
self.assertEqual(len(iprotos), len(pieces))
self.assertEqual(len(iprotos), len(ids))
self.assertEqual(iprotos.text(), text)
self.assertEqual(len(iprotos.pieces), len(pieces))
self.assertEqual(len(iprotos.pieces), len(ids))
self.assertEqual(iprotos.text, text)
self.assertEqual(len(iprotos2), len(pieces2))
self.assertEqual(len(iprotos2), len(ids2))
self.assertEqual(iprotos2.text(), text2)
self.assertEqual(len(iprotos2.pieces), len(pieces2))
self.assertEqual(len(iprotos2.pieces), len(ids2))
self.assertEqual(iprotos2.text, text2)
for i in range(len(iprotos)):
self.assertEqual(ids[i], iprotos.pieces(i).id())
self.assertEqual(pieces[i], iprotos.pieces(i).piece())
for i in range(len(iprotos.pieces)):
self.assertEqual(ids[i], iprotos.pieces[i].id)
self.assertEqual(pieces[i], iprotos.pieces[i].piece)
for i, piece in enumerate(iprotos):
self.assertEqual(ids[i], piece.id())
self.assertEqual(pieces[i], piece.piece())
for i, piece in enumerate(iprotos.pieces):
self.assertEqual(ids[i], piece.id)
self.assertEqual(pieces[i], piece.piece)
for i in range(len(iprotos2)):
self.assertEqual(ids2[i], iprotos2.pieces(i).id())
self.assertEqual(pieces2[i], iprotos2.pieces(i).piece())
for i in range(len(iprotos2.pieces)):
self.assertEqual(ids2[i], iprotos2.pieces[i].id)
self.assertEqual(pieces2[i], iprotos2.pieces[i].piece)
for i, piece in enumerate(iprotos2):
self.assertEqual(ids2[i], piece.id())
self.assertEqual(pieces2[i], piece.piece())
for i, piece in enumerate(iprotos2.pieces):
self.assertEqual(ids2[i], piece.id)
self.assertEqual(pieces2[i], piece.piece)
detok_ids = self.sp_.DecodeIds(ids)
detok_pieces = self.sp_.DecodePieces(pieces)