google/sentencepiece
1
1
Fork 0
mirror of https://github.com/google/sentencepiece.git synced 2025-01-06 09:19:12 +03:00
Code Issues Packages Projects Releases Wiki Activity

change the type of input_sentence_size from int32 to uint64

Browse Source
This commit is contained in:
Taku Kudo 2021-01-08 16:40:03 +09:00
parent 8eaa672a37
commit fe046e1c97
1 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/spec_parser.h
View File

@ -55,6 +55,16 @@ namespace sentencepiece {
return util::OkStatus(); \
}
#define PARSE_UINT64(param_name) \
if (name == #param_name) { \
uint64 v; \
if (!string_util::lexical_cast(value, &v)) \
return util::StatusBuilder(util::StatusCode::kInvalidArgument, GTL_LOC) \
<< "cannot parse \"" << value << "\" as int."; \
message->set_##param_name(v); \
return util::OkStatus(); \
}
#define PARSE_DOUBLE(param_name) \
if (name == #param_name) { \
double v; \
@ -196,7 +206,7 @@ util::Status SentencePieceTrainer::SetProtoField(const std::string &name,
PARSE_REPEATED_STRING(accept_language);
PARSE_INT32(self_test_sample_size);
PARSE_DOUBLE(character_coverage);
PARSE_INT32(input_sentence_size);
PARSE_UINT64(input_sentence_size);
PARSE_BOOL(shuffle_input_sentence);
PARSE_INT32(seed_sentencepiece_size);
PARSE_DOUBLE(shrinking_factor);