mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
Fix full left/full right definition to be zero length
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/branches/mtm_lm@4289 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
parent
64bfa6bc80
commit
bf9d8dd0df
@ -3,6 +3,7 @@
|
||||
#include "left.hh"
|
||||
#include "../../moses/src/Phrase.h"
|
||||
#include "../../moses/src/ChartHypothesis.h"
|
||||
#include "../../moses/src/ChartManager.h"
|
||||
|
||||
using namespace std;
|
||||
using namespace Moses;
|
||||
@ -19,61 +20,46 @@ ChartState::~ChartState()
|
||||
}
|
||||
|
||||
int ChartState::Compare(const ChartState &other) const {
|
||||
assert(prefix);
|
||||
assert(suffix);
|
||||
assert(prefix->GetSize() > 0);
|
||||
assert(suffix->GetSize() > 0);
|
||||
int comparePre = prefix->Compare(*other.prefix);
|
||||
int compareSuf = suffix->Compare(*other.suffix);
|
||||
|
||||
if (hypo->GetCurrSourceRange().GetStartPos() == 0) comparePre = 0;
|
||||
if (hypo->GetCurrSourceRange().GetEndPos() >= hypo->GetManager().GetSource().GetSize() - 1) compareSuf = 0;
|
||||
|
||||
int lres = left.Compare(other.left);
|
||||
if (lres) return lres;
|
||||
int fres = (int)full - (int)other.full;
|
||||
int rres = right.Compare(other.right);
|
||||
if (rres) return rres;
|
||||
int ret = (int)full - (int)other.full;
|
||||
|
||||
if (lres == 0 && rres != 0)
|
||||
recombCount[4]++;
|
||||
else if (lres == 0 && rres == 0)
|
||||
recombCount[5]++;
|
||||
else if (lres != 0 && rres == 0)
|
||||
recombCount[6]++;
|
||||
else
|
||||
{
|
||||
assert(lres);
|
||||
assert(rres);
|
||||
recombCount[7]++;
|
||||
}
|
||||
if (!comparePre) {
|
||||
assert(!lres);
|
||||
assert(!fres);
|
||||
}
|
||||
if (!compareSuf) assert(!rres);
|
||||
|
||||
if (ret == 0)
|
||||
{
|
||||
int comparePre = prefix->Compare(*other.prefix);
|
||||
int compareSuf = suffix->Compare(*other.suffix);
|
||||
if (lres) return lres;
|
||||
if (fres) return fres;
|
||||
if (rres) return rres;
|
||||
|
||||
if (comparePre)
|
||||
{
|
||||
if (compareSuf)
|
||||
recombCount[1]++; // both. middle
|
||||
else
|
||||
recombCount[0]++; // only prefix. left
|
||||
}
|
||||
else if (compareSuf)
|
||||
recombCount[2]++; // only suffix. right
|
||||
if (comparePre)
|
||||
{
|
||||
if (compareSuf)
|
||||
recombCount[1]++; // both. middle
|
||||
else
|
||||
recombCount[3]++; // nothing. last
|
||||
|
||||
/*
|
||||
cerr << "recomb_stats " << comparePre << " " << compareSuf << endl
|
||||
<< *prefix << " ||| " << *other.prefix << endl
|
||||
<< *suffix << " ||| " << *other.suffix << endl;
|
||||
*/
|
||||
}
|
||||
|
||||
return ret;
|
||||
recombCount[0]++; // only prefix. left
|
||||
}
|
||||
else if (compareSuf)
|
||||
recombCount[2]++; // only suffix. right
|
||||
else
|
||||
recombCount[3]++; // nothing. last
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void ChartState::CreatePreAndSuffices(const ChartHypothesis &hypo) const
|
||||
void ChartState::CreatePreAndSuffices(const ChartHypothesis &hyp)
|
||||
{
|
||||
prefix = &hypo.GetPrefix();
|
||||
suffix = &hypo.GetSuffix();
|
||||
prefix = &hyp.GetPrefix();
|
||||
suffix = &hyp.GetSuffix();
|
||||
hypo = &hyp;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -59,21 +59,21 @@ struct Left {
|
||||
bool operator==(const Left &other) const {
|
||||
return
|
||||
(length == other.length) &&
|
||||
pointers[length - 1] == other.pointers[length - 1];
|
||||
(!length || pointers[length - 1] == other.pointers[length - 1]);
|
||||
}
|
||||
|
||||
int Compare(const Left &other) const {
|
||||
if (length != other.length) {
|
||||
return (int)length - (int)other.length;
|
||||
}
|
||||
if (!length) return 0;
|
||||
if (pointers[length - 1] > other.pointers[length - 1]) return 1;
|
||||
if (pointers[length - 1] < other.pointers[length - 1]) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool operator<(const Left &other) const {
|
||||
if (length != other.length) return length < other.length;
|
||||
return pointers[length - 1] < other.pointers[length - 1];
|
||||
return Compare(other) == -1;
|
||||
}
|
||||
|
||||
void ZeroRemaining() {
|
||||
@ -86,7 +86,7 @@ struct Left {
|
||||
};
|
||||
|
||||
inline size_t hash_value(const Left &left) {
|
||||
return util::MurmurHashNative(&left.length, 1, left.pointers[left.length - 1]);
|
||||
return util::MurmurHashNative(&left.length, 1, left.length ? left.pointers[left.length - 1] : 0);
|
||||
}
|
||||
|
||||
class ChartState {
|
||||
@ -117,12 +117,13 @@ public:
|
||||
State right;
|
||||
bool full;
|
||||
|
||||
void CreatePreAndSuffices(const Moses::ChartHypothesis &hypo) const;
|
||||
void CreatePreAndSuffices(const Moses::ChartHypothesis &hyp);
|
||||
|
||||
static std::vector<int> recombCount;
|
||||
|
||||
protected:
|
||||
mutable const Moses::Phrase *prefix, *suffix;
|
||||
const Moses::ChartHypothesis *hypo;
|
||||
const Moses::Phrase *prefix, *suffix;
|
||||
};
|
||||
|
||||
inline size_t hash_value(const ChartState &state) {
|
||||
@ -197,21 +198,21 @@ template <class M> class RuleScore {
|
||||
ProcessRet(model_.ExtendLeft(out_.right.words, out_.right.words + out_.right.length, out_.right.backoff, in.left.pointers[0], 1, back, next_use));
|
||||
if (next_use != out_.right.length) {
|
||||
left_done_ = true;
|
||||
if (!next_use) {
|
||||
if (!next_use) {
|
||||
out_.right = in.right;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
unsigned char extend_length = 2;
|
||||
for (const uint64_t *i = in.left.pointers + 1; i < in.left.pointers + in.left.length; ++i, ++extend_length) {
|
||||
ProcessRet(model_.ExtendLeft(out_.right.words, out_.right.words + next_use, back, *i, extend_length, back2, next_use));
|
||||
if (next_use != out_.right.length) {
|
||||
left_done_ = true;
|
||||
if (!next_use) {
|
||||
out_.right = in.right;
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (next_use != out_.right.length) {
|
||||
left_done_ = true;
|
||||
if (!next_use) {
|
||||
out_.right = in.right;
|
||||
return;
|
||||
}
|
||||
}
|
||||
std::swap(back, back2);
|
||||
}
|
||||
|
||||
|
@ -163,8 +163,7 @@ FFState *LanguageModelKen<Model>::EvaluateChart(
|
||||
{
|
||||
LanguageModelChartStateKenLM *newState = new LanguageModelChartStateKenLM(hypo);
|
||||
|
||||
const lm::ngram::ChartState &chartState = newState->GetChartState();
|
||||
chartState.CreatePreAndSuffices(hypo);
|
||||
newState->GetChartState().CreatePreAndSuffices(hypo);
|
||||
|
||||
lm::ngram::RuleScore<Model> ruleScore(*m_ngram, newState->GetChartState());
|
||||
const AlignmentInfo::NonTermIndexMap &nonTermIndexMap = hypo.GetCurrTargetPhrase().GetAlignmentInfo().GetNonTermIndexMap();
|
||||
|
Loading…
Reference in New Issue
Block a user