From 846cf724ecbb86a15a9387503d3979cc7a78ad64 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Sun, 27 Jul 2014 16:22:57 +0200 Subject: [PATCH 1/2] Fixed issue with missing back-off weights in lmplz --- lm/builder/interpolate.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lm/builder/interpolate.cc b/lm/builder/interpolate.cc index db8537448..2cede8467 100644 --- a/lm/builder/interpolate.cc +++ b/lm/builder/interpolate.cc @@ -25,6 +25,10 @@ class Callback { ~Callback() { for (std::size_t i = 0; i < backoffs_.size(); ++i) { + if(prune_thresholds_[i + 1] > 0) + while(backoffs_[i]) + ++backoffs_[i]; + if (backoffs_[i]) { std::cerr << "Backoffs do not match for order " << (i + 1) << std::endl; abort(); @@ -51,8 +55,8 @@ class Callback { const HashGamma *hashed_backoff = static_cast(backoffs_[order_minus_1].Get()); while(backoffs_[order_minus_1] && current_hash != hashed_backoff->hash_value) { - hashed_backoff = static_cast(backoffs_[order_minus_1].Get()); ++backoffs_[order_minus_1]; + hashed_backoff = static_cast(backoffs_[order_minus_1].Get()); } if(current_hash == hashed_backoff->hash_value) { From e094d670d7eb779c718964e72f36b1e7eda4cb83 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Sun, 27 Jul 2014 20:33:34 +0200 Subject: [PATCH 2/2] First attempt at fixing missing-backoffs bug in lmplz --- lm/builder/interpolate.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lm/builder/interpolate.cc b/lm/builder/interpolate.cc index 2cede8467..fcd5073d8 100644 --- a/lm/builder/interpolate.cc +++ b/lm/builder/interpolate.cc @@ -25,6 +25,7 @@ class Callback { ~Callback() { for (std::size_t i = 0; i < backoffs_.size(); ++i) { + if(prune_thresholds_[i + 1] > 0) while(backoffs_[i]) ++backoffs_[i]; @@ -54,10 +55,8 @@ class Callback { uint64_t current_hash = util::MurmurHashNative(gram.begin(), gram.Order() * sizeof(WordIndex)); const HashGamma *hashed_backoff = static_cast(backoffs_[order_minus_1].Get()); - while(backoffs_[order_minus_1] && current_hash != hashed_backoff->hash_value) { - ++backoffs_[order_minus_1]; + while(current_hash != hashed_backoff->hash_value && ++backoffs_[order_minus_1]) hashed_backoff = static_cast(backoffs_[order_minus_1].Get()); - } if(current_hash == hashed_backoff->hash_value) { pay.complete.backoff = log10(hashed_backoff->gamma);