google/sentencepiece
1
1
Fork 0
mirror of https://github.com/google/sentencepiece.git synced 2024-10-26 11:38:45 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #867 from vmarkovtsev/patch-1

Browse Source 
Fix nasty bug in BPE position encoding
This commit is contained in:
Taku Kudo 2023-05-25 16:31:30 +09:00 committed by GitHub
commit 7b694e4bdb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/bpe_model_trainer.h
View File

@ -71,7 +71,9 @@ class Trainer : public TrainerInterface {
CHECK_GE(r, 0);
CHECK_LE(l, std::numeric_limits<uint16_t>::max());
CHECK_LE(r, std::numeric_limits<uint16_t>::max());
const uint64_t n = (static_cast<uint64_t>(sid) << 32 | (l << 16 | r));
const uint64_t n = (static_cast<uint64_t>(sid) << 32) |
(static_cast<uint64_t>(l) << 16) |
r;
return n;
}