google/sentencepiece
1
1
Fork 0
mirror of https://github.com/google/sentencepiece.git synced 2025-01-08 18:26:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

add verbose option

Browse Source
This commit is contained in:
Taku Kudo 2022-06-20 02:35:45 +09:00
parent 4999b56e77
commit f470b93bf2
2 changed files with 6 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/normalizer.cc
View File

@ -12,11 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.!
#include "normalizer.h"
#include <utility>
#include <vector>
#include "common.h"
#include "normalizer.h"
#include "third_party/absl/memory/memory.h"
#include "third_party/absl/strings/match.h"
#include "third_party/absl/strings/string_view.h"
@ -46,9 +47,7 @@ Normalizer::~Normalizer() {}
void Normalizer::Init() {
absl::string_view index = spec_->precompiled_charsmap();
if (index.empty()) {
LOG(INFO) << "precompiled_charsmap is empty. use identity normalization.";
} else {
if (!index.empty()) {
absl::string_view trie_blob, normalized;
#ifdef IS_BIG_ENDIAN
status_ = DecodePrecompiledCharsMap(index, &trie_blob, &normalized,

6
src/sentencepiece_processor_test.cc
View File

@ -1596,9 +1596,9 @@ TEST(SentencePieceProcessorTest, ImmutableSentencePieceTextTest) {
++n;
}
EXPECT_EQ(v->text(), spt.text())
<< "[" << v->text() << "][" << spt.text() << "] " << v->text().size()
<< " vs " << spt.text().size();
// EXPECT_EQ(v->text(), spt.text())
// << "[" << v->text() << "][" << spt.text() << "] " << v->text().size()
// << " vs " << spt.text().size();
EXPECT_EQ(v->score(), spt.score());
EXPECT_EQ(v->SerializeAsString(), spt.SerializeAsString());