mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
beautify
This commit is contained in:
parent
4d75c889f1
commit
1fe4eb0528
@ -91,7 +91,7 @@ int main(int argc, char** argv)
|
||||
// f.CreateFromString(Input, f_mask, query_f, "|", NULL);
|
||||
// c.CreateFromString(Input, c_mask, query_c,"|", NULL);
|
||||
// Phrase.CreateFromString() calls Word.CreateFromSting(), which gets
|
||||
// the factor delimiter from StaticData, so it should not be hardcoded
|
||||
// the factor delimiter from StaticData, so it should not be hardcoded
|
||||
// here. [UG], thus:
|
||||
e.CreateFromString(Output, e_mask, query_e, NULL);
|
||||
f.CreateFromString(Input, f_mask, query_f, NULL);
|
||||
|
@ -49,7 +49,7 @@ int main(int argc, char **argv)
|
||||
|
||||
Parameter *parameter = new Parameter();
|
||||
// const_cast<std::vector<std::string>&>(parameter->GetParam("factor-delimiter")).resize(1, "||dummy_string||");
|
||||
// UG: I assume "||dummy_string||" means: I'm not using factored data;
|
||||
// UG: I assume "||dummy_string||" means: I'm not using factored data;
|
||||
// This is now expressed by setting the factor delimiter to the empty string
|
||||
const_cast<std::vector<std::string>&>(parameter->GetParam("factor-delimiter")).resize(1, "");
|
||||
const_cast<std::vector<std::string>&>(parameter->GetParam("input-factors")).resize(1, "0");
|
||||
|
@ -437,8 +437,7 @@ void IOWrapper::OutputBestHypo(const Hypothesis *hypo, long /*translationId*/, c
|
||||
if (hypo != NULL) {
|
||||
VERBOSE(1,"BEST TRANSLATION: " << *hypo << endl);
|
||||
VERBOSE(3,"Best path: ");
|
||||
if (StaticData::Instance().IsPassthroughEnabled())
|
||||
{
|
||||
if (StaticData::Instance().IsPassthroughEnabled()) {
|
||||
OutputPassthroughInformation(cout, hypo);
|
||||
}
|
||||
Backtrack(hypo);
|
||||
@ -481,8 +480,7 @@ void OutputNBest(std::ostream& out
|
||||
|
||||
// print the surface factor of the translation
|
||||
out << translationId << " ||| ";
|
||||
if (staticData.IsPassthroughInNBestEnabled())
|
||||
{
|
||||
if (staticData.IsPassthroughInNBestEnabled()) {
|
||||
OutputPassthroughInformation(out, edges[edges.size() - 1]);
|
||||
}
|
||||
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
|
||||
|
@ -162,7 +162,7 @@ ChartKBestExtractor::FindOrCreateVertex(const ChartHypothesis &h)
|
||||
}
|
||||
boost::shared_ptr<Derivation> bestDerivation(new Derivation(bestEdge));
|
||||
#ifndef NDEBUG
|
||||
std::pair<DerivationSet::iterator, bool> q =
|
||||
std::pair<DerivationSet::iterator, bool> q =
|
||||
#endif
|
||||
m_derivations.insert(bestDerivation);
|
||||
assert(q.second);
|
||||
|
@ -11,8 +11,8 @@ ChartTranslationOption::ChartTranslationOption(const TargetPhrase &targetPhrase)
|
||||
}
|
||||
|
||||
void ChartTranslationOption::Evaluate(const InputType &input,
|
||||
const InputPath &inputPath,
|
||||
const StackVec &stackVec)
|
||||
const InputPath &inputPath,
|
||||
const StackVec &stackVec)
|
||||
{
|
||||
const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
|
||||
|
||||
|
@ -45,8 +45,8 @@ public:
|
||||
}
|
||||
|
||||
void Evaluate(const InputType &input,
|
||||
const InputPath &inputPath,
|
||||
const StackVec &stackVec);
|
||||
const InputPath &inputPath,
|
||||
const StackVec &stackVec);
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -106,10 +106,10 @@ void ChartTranslationOptionList::Add(const TargetPhraseCollection &tpc,
|
||||
|
||||
// Prune if bursting
|
||||
if (m_ruleLimit && m_size == m_ruleLimit * 2) {
|
||||
NTH_ELEMENT4(m_collection.begin(),
|
||||
m_collection.begin() + m_ruleLimit - 1,
|
||||
m_collection.begin() + m_size,
|
||||
ChartTranslationOptionOrderer());
|
||||
NTH_ELEMENT4(m_collection.begin(),
|
||||
m_collection.begin() + m_ruleLimit - 1,
|
||||
m_collection.begin() + m_size,
|
||||
ChartTranslationOptionOrderer());
|
||||
m_scoreThreshold = m_collection[m_ruleLimit-1]->GetEstimateOfBestScore();
|
||||
m_size = m_ruleLimit;
|
||||
}
|
||||
@ -182,12 +182,11 @@ void ChartTranslationOptionList::Evaluate(const InputType &input, const InputPat
|
||||
for (size_t i = 0; i < m_size; ++i) {
|
||||
ChartTranslationOptions *transOpts = m_collection[i];
|
||||
if (transOpts->GetSize() == 0) {
|
||||
//delete transOpts;
|
||||
++numDiscard;
|
||||
}
|
||||
else if (numDiscard) {
|
||||
SwapTranslationOptions(i - numDiscard, i);
|
||||
//m_collection[] = transOpts;
|
||||
//delete transOpts;
|
||||
++numDiscard;
|
||||
} else if (numDiscard) {
|
||||
SwapTranslationOptions(i - numDiscard, i);
|
||||
//m_collection[] = transOpts;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -71,10 +71,9 @@ void ChartTranslationOptions::Evaluate(const InputType &input, const InputPath &
|
||||
ChartTranslationOption *transOpt = m_collection[i].get();
|
||||
|
||||
if (transOpt->GetScores().GetWeightedScore() == - std::numeric_limits<float>::infinity()) {
|
||||
++numDiscard;
|
||||
}
|
||||
else if (numDiscard) {
|
||||
m_collection[i - numDiscard] = boost::shared_ptr<ChartTranslationOption>(transOpt);
|
||||
++numDiscard;
|
||||
} else if (numDiscard) {
|
||||
m_collection[i - numDiscard] = boost::shared_ptr<ChartTranslationOption>(transOpt);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -51,7 +51,7 @@ void ConstrainedDecoding::Load()
|
||||
long sentenceID = staticData.GetStartTranslationId() - 1;
|
||||
while (getline(constraintFile, line)) {
|
||||
vector<string> vecStr = Tokenize(line, "\t");
|
||||
|
||||
|
||||
Phrase phrase(0);
|
||||
if (vecStr.size() == 1) {
|
||||
sentenceID++;
|
||||
@ -64,7 +64,7 @@ void ConstrainedDecoding::Load()
|
||||
} else {
|
||||
UTIL_THROW(util::Exception, "Reference file not loaded");
|
||||
}
|
||||
|
||||
|
||||
if (addBeginEndWord) {
|
||||
phrase.InitStartEndWord();
|
||||
}
|
||||
@ -76,11 +76,11 @@ void ConstrainedDecoding::Load()
|
||||
std::vector<float> ConstrainedDecoding::DefaultWeights() const
|
||||
{
|
||||
UTIL_THROW_IF2(m_numScoreComponents != 1,
|
||||
<<<<<<< HEAD
|
||||
<<<<<<< HEAD
|
||||
"ConstrainedDecoding must only have 1 score");
|
||||
=======
|
||||
"ConstrainedDecoding must only have 1 score");
|
||||
>>>>>>> master
|
||||
=======
|
||||
"ConstrainedDecoding must only have 1 score");
|
||||
>>>>>>> master
|
||||
vector<float> ret(1, 1);
|
||||
return ret;
|
||||
}
|
||||
@ -113,7 +113,7 @@ FFState* ConstrainedDecoding::Evaluate(
|
||||
assert(ref);
|
||||
|
||||
ConstrainedDecodingState *ret = new ConstrainedDecodingState(hypo);
|
||||
const Phrase &outputPhrase = ret->GetPhrase();
|
||||
const Phrase &outputPhrase = ret->GetPhrase();
|
||||
|
||||
size_t searchPos = NOT_FOUND;
|
||||
size_t i = 0;
|
||||
@ -127,112 +127,110 @@ FFState* ConstrainedDecoding::Evaluate(
|
||||
float score;
|
||||
if (hypo.IsSourceCompleted()) {
|
||||
// translated entire sentence.
|
||||
<<<<<<< HEAD
|
||||
<<<<<<< HEAD
|
||||
bool match = (searchPos == 0) && (ref->GetSize() == outputPhrase.GetSize());
|
||||
if (!m_negate) {
|
||||
score = match ? 0 : - std::numeric_limits<float>::infinity();
|
||||
} else {
|
||||
score = !match ? 0 : - std::numeric_limits<float>::infinity();
|
||||
=======
|
||||
bool match = (searchPos == 0) && (size == outputPhrase.GetSize());
|
||||
if (!m_negate) {
|
||||
=======
|
||||
bool match = (searchPos == 0) && (size == outputPhrase.GetSize());
|
||||
if (!m_negate) {
|
||||
score = match ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
score = !match ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
|
||||
>>>>>>> master
|
||||
>>>>>>> master
|
||||
}
|
||||
}
|
||||
} else if (m_negate) {
|
||||
// keep all derivations
|
||||
score = 0;
|
||||
<<<<<<< HEAD
|
||||
} else {
|
||||
score = (searchPos != NOT_FOUND) ? 0 : - std::numeric_limits<float>::infinity();
|
||||
=======
|
||||
}
|
||||
else {
|
||||
score = (searchPos != NOT_FOUND) ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
|
||||
>>>>>>> master
|
||||
}
|
||||
|
||||
accumulator->PlusEquals(this, score);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
FFState* ConstrainedDecoding::EvaluateChart(
|
||||
const ChartHypothesis &hypo,
|
||||
int /* featureID - used to index the state in the previous hypotheses */,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{
|
||||
const std::vector<Phrase> *ref = GetConstraint<ChartHypothesis, ChartManager>(m_constraints, hypo);
|
||||
assert(ref);
|
||||
|
||||
const ChartManager &mgr = hypo.GetManager();
|
||||
const Sentence &source = static_cast<const Sentence&>(mgr.GetSource());
|
||||
|
||||
ConstrainedDecodingState *ret = new ConstrainedDecodingState(hypo);
|
||||
const Phrase &outputPhrase = ret->GetPhrase();
|
||||
|
||||
size_t searchPos = NOT_FOUND;
|
||||
size_t i = 0;
|
||||
size_t size = 0;
|
||||
while(searchPos == NOT_FOUND && i < ref->size()) {
|
||||
searchPos = (*ref)[i].Find(outputPhrase, m_maxUnknowns);
|
||||
size = (*ref)[i].GetSize();
|
||||
i++;
|
||||
}
|
||||
|
||||
float score;
|
||||
if (hypo.GetCurrSourceRange().GetStartPos() == 0 &&
|
||||
hypo.GetCurrSourceRange().GetEndPos() == source.GetSize() - 1) {
|
||||
// translated entire sentence.
|
||||
<<<<<<< HEAD
|
||||
bool match = (searchPos == 0) && (ref->GetSize() == outputPhrase.GetSize());
|
||||
|
||||
if (!m_negate) {
|
||||
score = match ? 0 : - std::numeric_limits<float>::infinity();
|
||||
else if (m_negate) {
|
||||
// keep all derivations
|
||||
score = 0;
|
||||
<<<<<<< HEAD
|
||||
} else {
|
||||
score = !match ? 0 : - std::numeric_limits<float>::infinity();
|
||||
=======
|
||||
bool match = (searchPos == 0) && (size == outputPhrase.GetSize());
|
||||
|
||||
if (!m_negate) {
|
||||
score = match ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
|
||||
score = (searchPos != NOT_FOUND) ? 0 : - std::numeric_limits<float>::infinity();
|
||||
=======
|
||||
}
|
||||
else {
|
||||
score = !match ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
|
||||
>>>>>>> master
|
||||
score = (searchPos != NOT_FOUND) ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
|
||||
>>>>>>> master
|
||||
}
|
||||
} else if (m_negate) {
|
||||
// keep all derivations
|
||||
score = 0;
|
||||
} else {
|
||||
score = (searchPos != NOT_FOUND) ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
|
||||
|
||||
accumulator->PlusEquals(this, score);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
accumulator->PlusEquals(this, score);
|
||||
FFState* ConstrainedDecoding::EvaluateChart(
|
||||
const ChartHypothesis &hypo,
|
||||
int /* featureID - used to index the state in the previous hypotheses */,
|
||||
ScoreComponentCollection* accumulator) const {
|
||||
const std::vector<Phrase> *ref = GetConstraint<ChartHypothesis, ChartManager>(m_constraints, hypo);
|
||||
assert(ref);
|
||||
|
||||
return ret;
|
||||
}
|
||||
const ChartManager &mgr = hypo.GetManager();
|
||||
const Sentence &source = static_cast<const Sentence&>(mgr.GetSource());
|
||||
|
||||
ConstrainedDecodingState *ret = new ConstrainedDecodingState(hypo);
|
||||
const Phrase &outputPhrase = ret->GetPhrase();
|
||||
|
||||
size_t searchPos = NOT_FOUND;
|
||||
size_t i = 0;
|
||||
size_t size = 0;
|
||||
while(searchPos == NOT_FOUND && i < ref->size()) {
|
||||
searchPos = (*ref)[i].Find(outputPhrase, m_maxUnknowns);
|
||||
size = (*ref)[i].GetSize();
|
||||
i++;
|
||||
}
|
||||
|
||||
float score;
|
||||
if (hypo.GetCurrSourceRange().GetStartPos() == 0 &&
|
||||
hypo.GetCurrSourceRange().GetEndPos() == source.GetSize() - 1) {
|
||||
// translated entire sentence.
|
||||
<<<<<<< HEAD
|
||||
bool match = (searchPos == 0) && (ref->GetSize() == outputPhrase.GetSize());
|
||||
|
||||
if (!m_negate) {
|
||||
score = match ? 0 : - std::numeric_limits<float>::infinity();
|
||||
} else {
|
||||
score = !match ? 0 : - std::numeric_limits<float>::infinity();
|
||||
=======
|
||||
bool match = (searchPos == 0) && (size == outputPhrase.GetSize());
|
||||
|
||||
if (!m_negate) {
|
||||
score = match ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
|
||||
} else {
|
||||
score = !match ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
|
||||
>>>>>>> master
|
||||
}
|
||||
}
|
||||
else if (m_negate) {
|
||||
// keep all derivations
|
||||
score = 0;
|
||||
} else {
|
||||
score = (searchPos != NOT_FOUND) ? 0 : - ( m_soft ? 1 : std::numeric_limits<float>::infinity());
|
||||
}
|
||||
|
||||
accumulator->PlusEquals(this, score);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void ConstrainedDecoding::SetParameter(const std::string& key, const std::string& value) {
|
||||
if (key == "path") {
|
||||
m_paths = Tokenize(value, ",");
|
||||
} else if (key == "max-unknowns") {
|
||||
m_maxUnknowns = Scan<int>(value);
|
||||
} else if (key == "negate") {
|
||||
m_negate = Scan<bool>(value);
|
||||
<<<<<<< HEAD
|
||||
=======
|
||||
} else if (key == "soft") {
|
||||
m_soft = Scan<bool>(value);
|
||||
>>>>>>> master
|
||||
} else {
|
||||
StatefulFeatureFunction::SetParameter(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
void ConstrainedDecoding::SetParameter(const std::string& key, const std::string& value)
|
||||
{
|
||||
if (key == "path") {
|
||||
m_paths = Tokenize(value, ",");
|
||||
} else if (key == "max-unknowns") {
|
||||
m_maxUnknowns = Scan<int>(value);
|
||||
} else if (key == "negate") {
|
||||
m_negate = Scan<bool>(value);
|
||||
<<<<<<< HEAD
|
||||
=======
|
||||
} else if (key == "soft") {
|
||||
m_soft = Scan<bool>(value);
|
||||
>>>>>>> master
|
||||
} else {
|
||||
StatefulFeatureFunction::SetParameter(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
@ -46,15 +46,15 @@ public:
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
|
||||
void Evaluate(const InputType &input
|
||||
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
||||
{}
|
||||
|
||||
|
||||
FFState* Evaluate(
|
||||
const Hypothesis& cur_hypo,
|
||||
const FFState* prev_state,
|
||||
|
@ -47,17 +47,17 @@ void CoveredReferenceFeature::Evaluate(const Phrase &source
|
||||
{}
|
||||
|
||||
void CoveredReferenceFeature::Evaluate(const InputType &input
|
||||
<<<<<<< HEAD
|
||||
<<<<<<< HEAD
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore) const
|
||||
=======
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore) const
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore) const
|
||||
>>>>>>> master
|
||||
{
|
||||
long id = input.GetTranslationId();
|
||||
|
@ -16,36 +16,35 @@ void InternalStructStatelessFF::Evaluate(const Phrase &source
|
||||
}
|
||||
|
||||
void InternalStructStatelessFF::Evaluate(const InputType &input
|
||||
<<<<<<< HEAD
|
||||
<<<<<<< HEAD
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore) const
|
||||
{
|
||||
=======
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore) const
|
||||
{
|
||||
>>>>>>> master
|
||||
=======
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore) const {
|
||||
>>>>>>> master
|
||||
|
||||
cerr << "HHHHH" << scoreBreakdown << endl;
|
||||
scoreBreakdown.PlusEquals(this, 66);
|
||||
/* FactorList f_mask;
|
||||
f_mask.push_back(0);
|
||||
//if(inputPath.GetPhrase().GetStringRep(f_mask).)
|
||||
int score =50;
|
||||
for(size_t i=0;i<inputPath.GetPhrase().GetSize();i++){
|
||||
if(inputPath.GetPhrase(). GetFactor(i,0)->GetString().as_string()=="ist"){
|
||||
//cout<<inputPath.GetPhrase().GetStringRep(f_mask);
|
||||
score+=1;
|
||||
}
|
||||
}
|
||||
scoreBreakdown.PlusEquals(this, score);
|
||||
*/
|
||||
}
|
||||
cerr << "HHHHH" << scoreBreakdown << endl;
|
||||
scoreBreakdown.PlusEquals(this, 66);
|
||||
/* FactorList f_mask;
|
||||
f_mask.push_back(0);
|
||||
//if(inputPath.GetPhrase().GetStringRep(f_mask).)
|
||||
int score =50;
|
||||
for(size_t i=0;i<inputPath.GetPhrase().GetSize();i++){
|
||||
if(inputPath.GetPhrase(). GetFactor(i,0)->GetString().as_string()=="ist"){
|
||||
//cout<<inputPath.GetPhrase().GetStringRep(f_mask);
|
||||
score+=1;
|
||||
}
|
||||
}
|
||||
scoreBreakdown.PlusEquals(this, score);
|
||||
*/
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
@ -22,7 +22,7 @@ public:
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||
|
||||
<<<<<<< HEAD
|
||||
<<<<<<< HEAD
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
@ -34,20 +34,20 @@ public:
|
||||
void EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
=======
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
||||
virtual void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
void EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
>>>>>>> master
|
||||
=======
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
||||
virtual void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
void EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
>>>>>>> master
|
||||
|
||||
};
|
||||
|
||||
|
@ -14,10 +14,10 @@ using namespace std;
|
||||
namespace Moses
|
||||
{
|
||||
MaxSpanFreeNonTermSource::MaxSpanFreeNonTermSource(const std::string &line)
|
||||
:StatelessFeatureFunction(1, line)
|
||||
,m_maxSpan(2)
|
||||
,m_glueTargetLHSStr("S")
|
||||
,m_glueTargetLHS(true)
|
||||
:StatelessFeatureFunction(1, line)
|
||||
,m_maxSpan(2)
|
||||
,m_glueTargetLHSStr("S")
|
||||
,m_glueTargetLHS(true)
|
||||
{
|
||||
m_tuneable = false;
|
||||
ReadParameters();
|
||||
@ -28,25 +28,25 @@ MaxSpanFreeNonTermSource::MaxSpanFreeNonTermSource(const std::string &line)
|
||||
}
|
||||
|
||||
void MaxSpanFreeNonTermSource::Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{
|
||||
targetPhrase.SetRuleSource(source);
|
||||
}
|
||||
|
||||
void MaxSpanFreeNonTermSource::Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore) const
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore) const
|
||||
{
|
||||
const Word &targetLHS = targetPhrase.GetTargetLHS();
|
||||
|
||||
if (targetLHS == m_glueTargetLHS) {
|
||||
// don't delete glue rules
|
||||
return;
|
||||
// don't delete glue rules
|
||||
return;
|
||||
}
|
||||
|
||||
const Phrase *source = targetPhrase.GetRuleSource();
|
||||
@ -54,17 +54,17 @@ void MaxSpanFreeNonTermSource::Evaluate(const InputType &input
|
||||
float score = 0;
|
||||
|
||||
if (source->Front().IsNonTerminal()) {
|
||||
const ChartCellLabel &cell = *stackVec->front();
|
||||
if (cell.GetCoverage().GetNumWordsCovered() > m_maxSpan) {
|
||||
score = - std::numeric_limits<float>::infinity();
|
||||
}
|
||||
const ChartCellLabel &cell = *stackVec->front();
|
||||
if (cell.GetCoverage().GetNumWordsCovered() > m_maxSpan) {
|
||||
score = - std::numeric_limits<float>::infinity();
|
||||
}
|
||||
}
|
||||
|
||||
if (source->Back().IsNonTerminal()) {
|
||||
const ChartCellLabel &cell = *stackVec->back();
|
||||
if (cell.GetCoverage().GetNumWordsCovered() > m_maxSpan) {
|
||||
score = - std::numeric_limits<float>::infinity();
|
||||
}
|
||||
const ChartCellLabel &cell = *stackVec->back();
|
||||
if (cell.GetCoverage().GetNumWordsCovered() > m_maxSpan) {
|
||||
score = - std::numeric_limits<float>::infinity();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -76,7 +76,7 @@ void MaxSpanFreeNonTermSource::Evaluate(const InputType &input
|
||||
void MaxSpanFreeNonTermSource::SetParameter(const std::string& key, const std::string& value)
|
||||
{
|
||||
if (key == "max-span") {
|
||||
m_maxSpan = Scan<int>(value);
|
||||
m_maxSpan = Scan<int>(value);
|
||||
} else {
|
||||
StatelessFeatureFunction::SetParameter(key, value);
|
||||
}
|
||||
@ -84,8 +84,8 @@ void MaxSpanFreeNonTermSource::SetParameter(const std::string& key, const std::s
|
||||
|
||||
std::vector<float> MaxSpanFreeNonTermSource::DefaultWeights() const
|
||||
{
|
||||
std::vector<float> ret(1, 1);
|
||||
return ret;
|
||||
std::vector<float> ret(1, 1);
|
||||
return ret;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -10,33 +10,34 @@ namespace Moses
|
||||
class MaxSpanFreeNonTermSource : public StatelessFeatureFunction
|
||||
{
|
||||
public:
|
||||
MaxSpanFreeNonTermSource(const std::string &line);
|
||||
MaxSpanFreeNonTermSource(const std::string &line);
|
||||
|
||||
virtual bool IsUseable(const FactorMask &mask) const
|
||||
{ return true; }
|
||||
virtual bool IsUseable(const FactorMask &mask) const {
|
||||
return true;
|
||||
}
|
||||
|
||||
virtual void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||
virtual void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||
|
||||
virtual void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
||||
virtual void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
||||
|
||||
virtual void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
virtual void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
|
||||
virtual void EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
virtual void EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
|
||||
void SetParameter(const std::string& key, const std::string& value);
|
||||
std::vector<float> DefaultWeights() const;
|
||||
void SetParameter(const std::string& key, const std::string& value);
|
||||
std::vector<float> DefaultWeights() const;
|
||||
|
||||
protected:
|
||||
int m_maxSpan;
|
||||
|
@ -18,19 +18,19 @@ NieceTerminal::NieceTerminal(const std::string &line)
|
||||
}
|
||||
|
||||
void NieceTerminal::Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{
|
||||
targetPhrase.SetRuleSource(source);
|
||||
}
|
||||
|
||||
void NieceTerminal::Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore) const
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore) const
|
||||
{
|
||||
assert(stackVec);
|
||||
|
||||
@ -39,59 +39,59 @@ void NieceTerminal::Evaluate(const InputType &input
|
||||
|
||||
std::set<Word> terms;
|
||||
for (size_t i = 0; i < ruleSource->GetSize(); ++i) {
|
||||
const Word &word = ruleSource->GetWord(i);
|
||||
if (!word.IsNonTerminal()) {
|
||||
terms.insert(word);
|
||||
}
|
||||
const Word &word = ruleSource->GetWord(i);
|
||||
if (!word.IsNonTerminal()) {
|
||||
terms.insert(word);
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < stackVec->size(); ++i) {
|
||||
const ChartCellLabel &cell = *stackVec->at(i);
|
||||
const WordsRange &ntRange = cell.GetCoverage();
|
||||
bool containTerm = ContainTerm(input, ntRange, terms);
|
||||
const ChartCellLabel &cell = *stackVec->at(i);
|
||||
const WordsRange &ntRange = cell.GetCoverage();
|
||||
bool containTerm = ContainTerm(input, ntRange, terms);
|
||||
|
||||
if (containTerm) {
|
||||
//cerr << "ruleSource=" << *ruleSource << " ";
|
||||
//cerr << "ntRange=" << ntRange << endl;
|
||||
if (containTerm) {
|
||||
//cerr << "ruleSource=" << *ruleSource << " ";
|
||||
//cerr << "ntRange=" << ntRange << endl;
|
||||
|
||||
// non-term contains 1 of the terms in the rule.
|
||||
float score = m_hardConstraint ? - std::numeric_limits<float>::infinity() : 1;
|
||||
scoreBreakdown.PlusEquals(this, score);
|
||||
return;
|
||||
}
|
||||
// non-term contains 1 of the terms in the rule.
|
||||
float score = m_hardConstraint ? - std::numeric_limits<float>::infinity() : 1;
|
||||
scoreBreakdown.PlusEquals(this, score);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void NieceTerminal::Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
|
||||
void NieceTerminal::EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
|
||||
bool NieceTerminal::ContainTerm(const InputType &input,
|
||||
const WordsRange &ntRange,
|
||||
const std::set<Word> &terms) const
|
||||
const WordsRange &ntRange,
|
||||
const std::set<Word> &terms) const
|
||||
{
|
||||
std::set<Word>::const_iterator iter;
|
||||
std::set<Word>::const_iterator iter;
|
||||
|
||||
for (size_t pos = ntRange.GetStartPos(); pos <= ntRange.GetEndPos(); ++pos) {
|
||||
const Word &word = input.GetWord(pos);
|
||||
iter = terms.find(word);
|
||||
for (size_t pos = ntRange.GetStartPos(); pos <= ntRange.GetEndPos(); ++pos) {
|
||||
const Word &word = input.GetWord(pos);
|
||||
iter = terms.find(word);
|
||||
|
||||
if (iter != terms.end()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
if (iter != terms.end()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void NieceTerminal::SetParameter(const std::string& key, const std::string& value)
|
||||
{
|
||||
if (key == "hard-constraint") {
|
||||
m_hardConstraint = Scan<bool>(value);
|
||||
m_hardConstraint = Scan<bool>(value);
|
||||
} else {
|
||||
StatelessFeatureFunction::SetParameter(key, value);
|
||||
}
|
||||
|
@ -39,8 +39,8 @@ public:
|
||||
protected:
|
||||
bool m_hardConstraint;
|
||||
bool ContainTerm(const InputType &input,
|
||||
const WordsRange &ntRange,
|
||||
const std::set<Word> &terms) const;
|
||||
const WordsRange &ntRange,
|
||||
const std::set<Word> &terms) const;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -22,22 +22,22 @@ public:
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
|
||||
<<<<<<< HEAD
|
||||
<<<<<<< HEAD
|
||||
virtual void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
||||
{}
|
||||
=======
|
||||
virtual void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
||||
{}
|
||||
>>>>>>> master
|
||||
=======
|
||||
virtual void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
||||
{}
|
||||
>>>>>>> master
|
||||
|
||||
virtual void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
|
@ -5,8 +5,8 @@
|
||||
namespace Moses
|
||||
{
|
||||
RuleScope::RuleScope(const std::string &line)
|
||||
:StatelessFeatureFunction(1, line)
|
||||
,m_sourceSyntax(true)
|
||||
:StatelessFeatureFunction(1, line)
|
||||
,m_sourceSyntax(true)
|
||||
{
|
||||
}
|
||||
|
||||
@ -17,9 +17,9 @@ bool IsAmbiguous(const Word &word, bool sourceSyntax)
|
||||
}
|
||||
|
||||
void RuleScope::Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{
|
||||
// adjacent non-term count as 1 ammbiguity, rather than 2 as in rule scope
|
||||
// source can't be empty, right?
|
||||
|
@ -9,7 +9,7 @@ namespace Moses
|
||||
class RuleScope : public StatelessFeatureFunction
|
||||
{
|
||||
public:
|
||||
RuleScope(const std::string &line);
|
||||
RuleScope(const std::string &line);
|
||||
|
||||
virtual bool IsUseable(const FactorMask &mask) const {
|
||||
return true;
|
||||
@ -20,13 +20,13 @@ public:
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||
|
||||
virtual void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
||||
{}
|
||||
virtual void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
||||
{}
|
||||
|
||||
virtual void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
|
@ -4,7 +4,7 @@
|
||||
namespace Moses
|
||||
{
|
||||
SetSourcePhrase::SetSourcePhrase(const std::string &line)
|
||||
:StatelessFeatureFunction(0, line)
|
||||
:StatelessFeatureFunction(0, line)
|
||||
{
|
||||
m_tuneable = false;
|
||||
ReadParameters();
|
||||
|
@ -21,11 +21,11 @@ public:
|
||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||
|
||||
virtual void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhrase &targetPhrase
|
||||
, const StackVec *stackVec
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
||||
{}
|
||||
|
||||
virtual void Evaluate(const Hypothesis& hypo,
|
||||
|
@ -85,12 +85,12 @@ size_t InputPath::GetTotalRuleSize() const
|
||||
size_t ret = 0;
|
||||
std::map<const PhraseDictionary*, std::pair<const TargetPhraseCollection*, const void*> >::const_iterator iter;
|
||||
for (iter = m_targetPhrases.begin(); iter != m_targetPhrases.end(); ++iter) {
|
||||
const PhraseDictionary *pt = iter->first;
|
||||
const TargetPhraseCollection *tpColl = iter->second.first;
|
||||
const PhraseDictionary *pt = iter->first;
|
||||
const TargetPhraseCollection *tpColl = iter->second.first;
|
||||
|
||||
if (tpColl) {
|
||||
ret += tpColl->GetSize();
|
||||
}
|
||||
if (tpColl) {
|
||||
ret += tpColl->GetSize();
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
@ -108,10 +108,9 @@ std::ostream& operator<<(std::ostream& out, const InputPath& obj)
|
||||
|
||||
out << pt << "=";
|
||||
if (tpColl) {
|
||||
cerr << tpColl->GetSize() << " ";
|
||||
}
|
||||
else {
|
||||
cerr << "NULL ";
|
||||
cerr << tpColl->GetSize() << " ";
|
||||
} else {
|
||||
cerr << "NULL ";
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -44,140 +44,140 @@ namespace Moses
|
||||
class DALMState : public FFState
|
||||
{
|
||||
private:
|
||||
DALM::State state;
|
||||
DALM::State state;
|
||||
|
||||
public:
|
||||
DALMState(){
|
||||
}
|
||||
DALMState() {
|
||||
}
|
||||
|
||||
DALMState(const DALMState &from){
|
||||
state = from.state;
|
||||
}
|
||||
DALMState(const DALMState &from) {
|
||||
state = from.state;
|
||||
}
|
||||
|
||||
virtual ~DALMState(){
|
||||
}
|
||||
virtual ~DALMState() {
|
||||
}
|
||||
|
||||
void reset(const DALMState &from){
|
||||
state = from.state;
|
||||
}
|
||||
void reset(const DALMState &from) {
|
||||
state = from.state;
|
||||
}
|
||||
|
||||
virtual int Compare(const FFState& other) const{
|
||||
const DALMState &o = static_cast<const DALMState &>(other);
|
||||
if(state.get_count() < o.state.get_count()) return -1;
|
||||
else if(state.get_count() > o.state.get_count()) return 1;
|
||||
else return state.compare(o.state);
|
||||
}
|
||||
virtual int Compare(const FFState& other) const {
|
||||
const DALMState &o = static_cast<const DALMState &>(other);
|
||||
if(state.get_count() < o.state.get_count()) return -1;
|
||||
else if(state.get_count() > o.state.get_count()) return 1;
|
||||
else return state.compare(o.state);
|
||||
}
|
||||
|
||||
DALM::State &get_state(){
|
||||
return state;
|
||||
}
|
||||
|
||||
void refresh(){
|
||||
state.refresh();
|
||||
}
|
||||
DALM::State &get_state() {
|
||||
return state;
|
||||
}
|
||||
|
||||
void refresh() {
|
||||
state.refresh();
|
||||
}
|
||||
};
|
||||
|
||||
class DALMChartState : public FFState
|
||||
{
|
||||
private:
|
||||
DALM::Fragment prefixFragments[DALM_MAX_ORDER-1];
|
||||
unsigned char prefixLength;
|
||||
DALM::State rightContext;
|
||||
bool isLarge;
|
||||
size_t hypoSize;
|
||||
DALM::Fragment prefixFragments[DALM_MAX_ORDER-1];
|
||||
unsigned char prefixLength;
|
||||
DALM::State rightContext;
|
||||
bool isLarge;
|
||||
size_t hypoSize;
|
||||
|
||||
public:
|
||||
DALMChartState()
|
||||
: prefixLength(0),
|
||||
isLarge(false)
|
||||
{}
|
||||
DALMChartState()
|
||||
: prefixLength(0),
|
||||
isLarge(false)
|
||||
{}
|
||||
|
||||
/*
|
||||
DALMChartState(const DALMChartState &other)
|
||||
: prefixLength(other.prefixLength),
|
||||
rightContext(other.rightContext),
|
||||
isLarge(other.isLarge)
|
||||
{
|
||||
std::copy(
|
||||
other.prefixFragments,
|
||||
other.prefixFragments+other.prefixLength,
|
||||
prefixFragments
|
||||
);
|
||||
}
|
||||
*/
|
||||
/*
|
||||
DALMChartState(const DALMChartState &other)
|
||||
: prefixLength(other.prefixLength),
|
||||
rightContext(other.rightContext),
|
||||
isLarge(other.isLarge)
|
||||
{
|
||||
std::copy(
|
||||
other.prefixFragments,
|
||||
other.prefixFragments+other.prefixLength,
|
||||
prefixFragments
|
||||
);
|
||||
}
|
||||
*/
|
||||
|
||||
virtual ~DALMChartState(){
|
||||
}
|
||||
virtual ~DALMChartState() {
|
||||
}
|
||||
|
||||
/*
|
||||
DALMChartState &operator=(const DALMChartState &other){
|
||||
prefixLength = other.prefixLength;
|
||||
std::copy(
|
||||
other.prefixFragments,
|
||||
other.prefixFragments+other.prefixLength,
|
||||
prefixFragments
|
||||
);
|
||||
rightContext = other.rightContext;
|
||||
isLarge=other.isLarge;
|
||||
/*
|
||||
DALMChartState &operator=(const DALMChartState &other){
|
||||
prefixLength = other.prefixLength;
|
||||
std::copy(
|
||||
other.prefixFragments,
|
||||
other.prefixFragments+other.prefixLength,
|
||||
prefixFragments
|
||||
);
|
||||
rightContext = other.rightContext;
|
||||
isLarge=other.isLarge;
|
||||
|
||||
return *this;
|
||||
}
|
||||
*/
|
||||
return *this;
|
||||
}
|
||||
*/
|
||||
|
||||
inline unsigned char GetPrefixLength() const{
|
||||
return prefixLength;
|
||||
}
|
||||
inline unsigned char GetPrefixLength() const {
|
||||
return prefixLength;
|
||||
}
|
||||
|
||||
inline unsigned char &GetPrefixLength(){
|
||||
return prefixLength;
|
||||
}
|
||||
inline unsigned char &GetPrefixLength() {
|
||||
return prefixLength;
|
||||
}
|
||||
|
||||
inline const DALM::Fragment *GetPrefixFragments() const{
|
||||
return prefixFragments;
|
||||
}
|
||||
inline const DALM::Fragment *GetPrefixFragments() const {
|
||||
return prefixFragments;
|
||||
}
|
||||
|
||||
inline DALM::Fragment *GetPrefixFragments(){
|
||||
return prefixFragments;
|
||||
}
|
||||
inline DALM::Fragment *GetPrefixFragments() {
|
||||
return prefixFragments;
|
||||
}
|
||||
|
||||
inline const DALM::State &GetRightContext() const{
|
||||
return rightContext;
|
||||
}
|
||||
inline const DALM::State &GetRightContext() const {
|
||||
return rightContext;
|
||||
}
|
||||
|
||||
inline DALM::State &GetRightContext() {
|
||||
return rightContext;
|
||||
}
|
||||
inline DALM::State &GetRightContext() {
|
||||
return rightContext;
|
||||
}
|
||||
|
||||
inline bool LargeEnough() const{
|
||||
return isLarge;
|
||||
}
|
||||
inline bool LargeEnough() const {
|
||||
return isLarge;
|
||||
}
|
||||
|
||||
inline void SetAsLarge() {
|
||||
isLarge=true;
|
||||
}
|
||||
|
||||
inline size_t &GetHypoSize() {
|
||||
return hypoSize;
|
||||
}
|
||||
inline size_t GetHypoSize() const {
|
||||
return hypoSize;
|
||||
}
|
||||
inline void SetAsLarge() {
|
||||
isLarge=true;
|
||||
}
|
||||
|
||||
virtual int Compare(const FFState& other) const{
|
||||
const DALMChartState &o = static_cast<const DALMChartState &>(other);
|
||||
if(prefixLength < o.prefixLength) return -1;
|
||||
if(prefixLength > o.prefixLength) return 1;
|
||||
if(prefixLength!=0){
|
||||
const DALM::Fragment &f = prefixFragments[prefixLength-1];
|
||||
const DALM::Fragment &of = o.prefixFragments[prefixLength-1];
|
||||
int ret = DALM::compare_fragments(f,of);
|
||||
if(ret != 0) return ret;
|
||||
}
|
||||
if(isLarge != o.isLarge) return (int)isLarge - (int)o.isLarge;
|
||||
if(rightContext.get_count() < o.rightContext.get_count()) return -1;
|
||||
if(rightContext.get_count() > o.rightContext.get_count()) return 1;
|
||||
return rightContext.compare(o.rightContext);
|
||||
}
|
||||
inline size_t &GetHypoSize() {
|
||||
return hypoSize;
|
||||
}
|
||||
inline size_t GetHypoSize() const {
|
||||
return hypoSize;
|
||||
}
|
||||
|
||||
virtual int Compare(const FFState& other) const {
|
||||
const DALMChartState &o = static_cast<const DALMChartState &>(other);
|
||||
if(prefixLength < o.prefixLength) return -1;
|
||||
if(prefixLength > o.prefixLength) return 1;
|
||||
if(prefixLength!=0) {
|
||||
const DALM::Fragment &f = prefixFragments[prefixLength-1];
|
||||
const DALM::Fragment &of = o.prefixFragments[prefixLength-1];
|
||||
int ret = DALM::compare_fragments(f,of);
|
||||
if(ret != 0) return ret;
|
||||
}
|
||||
if(isLarge != o.isLarge) return (int)isLarge - (int)o.isLarge;
|
||||
if(rightContext.get_count() < o.rightContext.get_count()) return -1;
|
||||
if(rightContext.get_count() > o.rightContext.get_count()) return 1;
|
||||
return rightContext.compare(o.rightContext);
|
||||
}
|
||||
};
|
||||
|
||||
LanguageModelDALM::LanguageModelDALM(const std::string &line)
|
||||
@ -225,20 +225,21 @@ void LanguageModelDALM::Load()
|
||||
wid_start = m_vocab->lookup(BOS_);
|
||||
wid_end = m_vocab->lookup(EOS_);
|
||||
|
||||
// Load the language model.
|
||||
m_lm = new DALM::LM(model, *m_vocab, m_nGramOrder, *m_logger);
|
||||
|
||||
wid_start = m_vocab->lookup(BOS_);
|
||||
wid_end = m_vocab->lookup(EOS_);
|
||||
// Load the language model.
|
||||
m_lm = new DALM::LM(model, *m_vocab, m_nGramOrder, *m_logger);
|
||||
|
||||
wid_start = m_vocab->lookup(BOS_);
|
||||
wid_end = m_vocab->lookup(EOS_);
|
||||
|
||||
FactorCollection &collection = FactorCollection::Instance();
|
||||
m_beginSentenceFactor = collection.AddFactor(BOS_);
|
||||
}
|
||||
|
||||
const FFState *LanguageModelDALM::EmptyHypothesisState(const InputType &/*input*/) const{
|
||||
DALMState *s = new DALMState();
|
||||
m_lm->init_state(s->get_state());
|
||||
return s;
|
||||
const FFState *LanguageModelDALM::EmptyHypothesisState(const InputType &/*input*/) const
|
||||
{
|
||||
DALMState *s = new DALMState();
|
||||
m_lm->init_state(s->get_state());
|
||||
return s;
|
||||
}
|
||||
|
||||
void LanguageModelDALM::CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const
|
||||
@ -252,15 +253,15 @@ void LanguageModelDALM::CalcScore(const Phrase &phrase, float &fullScore, float
|
||||
if (!phraseSize) return;
|
||||
size_t currPos = 0;
|
||||
size_t hist_count = 0;
|
||||
DALM::State state;
|
||||
DALM::State state;
|
||||
|
||||
if(phrase.GetWord(0).GetFactor(m_factorType) == m_beginSentenceFactor){
|
||||
m_lm->init_state(state);
|
||||
currPos++;
|
||||
hist_count++;
|
||||
}
|
||||
|
||||
float score;
|
||||
if(phrase.GetWord(0).GetFactor(m_factorType) == m_beginSentenceFactor) {
|
||||
m_lm->init_state(state);
|
||||
currPos++;
|
||||
hist_count++;
|
||||
}
|
||||
|
||||
float score;
|
||||
while (currPos < phraseSize) {
|
||||
const Word &word = phrase.GetWord(currPos);
|
||||
hist_count++;
|
||||
@ -269,9 +270,9 @@ void LanguageModelDALM::CalcScore(const Phrase &phrase, float &fullScore, float
|
||||
state.refresh();
|
||||
hist_count = 0;
|
||||
} else {
|
||||
DALM::VocabId wid = GetVocabId(word.GetFactor(m_factorType));
|
||||
score = m_lm->query(wid, state);
|
||||
fullScore += score;
|
||||
DALM::VocabId wid = GetVocabId(word.GetFactor(m_factorType));
|
||||
score = m_lm->query(wid, state);
|
||||
fullScore += score;
|
||||
if (hist_count >= m_nGramOrder) ngramScore += score;
|
||||
if (wid==m_vocab->unk()) ++oovCount;
|
||||
}
|
||||
@ -279,8 +280,8 @@ void LanguageModelDALM::CalcScore(const Phrase &phrase, float &fullScore, float
|
||||
currPos++;
|
||||
}
|
||||
|
||||
fullScore = TransformLMScore(fullScore);
|
||||
ngramScore = TransformLMScore(ngramScore);
|
||||
fullScore = TransformLMScore(fullScore);
|
||||
ngramScore = TransformLMScore(ngramScore);
|
||||
}
|
||||
|
||||
FFState *LanguageModelDALM::Evaluate(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const
|
||||
@ -302,10 +303,10 @@ FFState *LanguageModelDALM::Evaluate(const Hypothesis &hypo, const FFState *ps,
|
||||
const std::size_t adjust_end = std::min(end, begin + m_nGramOrder - 1);
|
||||
|
||||
DALMState *dalm_state = new DALMState(*dalm_ps);
|
||||
DALM::State &state = dalm_state->get_state();
|
||||
DALM::State &state = dalm_state->get_state();
|
||||
float score = 0.0;
|
||||
for(std::size_t position=begin; position < adjust_end; position++){
|
||||
score += m_lm->query(GetVocabId(hypo.GetWord(position).GetFactor(m_factorType)), state);
|
||||
for(std::size_t position=begin; position < adjust_end; position++) {
|
||||
score += m_lm->query(GetVocabId(hypo.GetWord(position).GetFactor(m_factorType)), state);
|
||||
}
|
||||
|
||||
if (hypo.IsSourceCompleted()) {
|
||||
@ -313,8 +314,8 @@ FFState *LanguageModelDALM::Evaluate(const Hypothesis &hypo, const FFState *ps,
|
||||
std::vector<DALM::VocabId> indices(m_nGramOrder-1);
|
||||
const DALM::VocabId *last = LastIDs(hypo, &indices.front());
|
||||
m_lm->set_state(&indices.front(), (last-&indices.front()), state);
|
||||
|
||||
score += m_lm->query(wid_end, state);
|
||||
|
||||
score += m_lm->query(wid_end, state);
|
||||
} else if (adjust_end < end) {
|
||||
// Get state after adding a long phrase.
|
||||
std::vector<DALM::VocabId> indices(m_nGramOrder-1);
|
||||
@ -338,50 +339,50 @@ FFState *LanguageModelDALM::Evaluate(const Hypothesis &hypo, const FFState *ps,
|
||||
FFState *LanguageModelDALM::EvaluateChart(const ChartHypothesis& hypo, int featureID, ScoreComponentCollection *out) const
|
||||
{
|
||||
// initialize language model context state
|
||||
DALMChartState *newState = new DALMChartState();
|
||||
DALM::State &state = newState->GetRightContext();
|
||||
DALMChartState *newState = new DALMChartState();
|
||||
DALM::State &state = newState->GetRightContext();
|
||||
|
||||
DALM::Fragment *prefixFragments = newState->GetPrefixFragments();
|
||||
unsigned char &prefixLength = newState->GetPrefixLength();
|
||||
size_t &hypoSizeAll = newState->GetHypoSize();
|
||||
DALM::Fragment *prefixFragments = newState->GetPrefixFragments();
|
||||
unsigned char &prefixLength = newState->GetPrefixLength();
|
||||
size_t &hypoSizeAll = newState->GetHypoSize();
|
||||
|
||||
// initial language model scores
|
||||
float hypoScore = 0.0; // total hypothesis score.
|
||||
|
||||
const TargetPhrase &targetPhrase = hypo.GetCurrTargetPhrase();
|
||||
size_t hypoSize = targetPhrase.GetSize();
|
||||
hypoSizeAll = hypoSize;
|
||||
const TargetPhrase &targetPhrase = hypo.GetCurrTargetPhrase();
|
||||
size_t hypoSize = targetPhrase.GetSize();
|
||||
hypoSizeAll = hypoSize;
|
||||
|
||||
// get index map for underlying hypotheses
|
||||
const AlignmentInfo::NonTermIndexMap &nonTermIndexMap =
|
||||
targetPhrase.GetAlignNonTerm().GetNonTermIndexMap();
|
||||
|
||||
size_t phrasePos = 0;
|
||||
size_t phrasePos = 0;
|
||||
|
||||
// begginig of sentence.
|
||||
if(hypoSize > 0){
|
||||
const Word &word = targetPhrase.GetWord(0);
|
||||
if(word.GetFactor(m_factorType) == m_beginSentenceFactor){
|
||||
m_lm->init_state(state);
|
||||
// state is finalized.
|
||||
newState->SetAsLarge();
|
||||
phrasePos++;
|
||||
}else if(word.IsNonTerminal()){
|
||||
// begginig of sentence.
|
||||
if(hypoSize > 0) {
|
||||
const Word &word = targetPhrase.GetWord(0);
|
||||
if(word.GetFactor(m_factorType) == m_beginSentenceFactor) {
|
||||
m_lm->init_state(state);
|
||||
// state is finalized.
|
||||
newState->SetAsLarge();
|
||||
phrasePos++;
|
||||
} else if(word.IsNonTerminal()) {
|
||||
// special case: rule starts with non-terminal -> copy everything
|
||||
|
||||
const ChartHypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndexMap[0]);
|
||||
const DALMChartState* prevState =
|
||||
static_cast<const DALMChartState*>(prevHypo->GetFFState(featureID));
|
||||
|
||||
// copy chart state
|
||||
(*newState) = (*prevState);
|
||||
hypoSizeAll = hypoSize+prevState->GetHypoSize()-1;
|
||||
// copy chart state
|
||||
(*newState) = (*prevState);
|
||||
hypoSizeAll = hypoSize+prevState->GetHypoSize()-1;
|
||||
|
||||
// get hypoScore
|
||||
hypoScore = UntransformLMScore(prevHypo->GetScoreBreakdown().GetScoresForProducer(this)[0]);
|
||||
hypoScore = UntransformLMScore(prevHypo->GetScoreBreakdown().GetScoresForProducer(this)[0]);
|
||||
|
||||
phrasePos++;
|
||||
}
|
||||
phrasePos++;
|
||||
}
|
||||
}
|
||||
|
||||
// loop over rule
|
||||
@ -392,30 +393,30 @@ FFState *LanguageModelDALM::EvaluateChart(const ChartHypothesis& hypo, int featu
|
||||
|
||||
// regular word
|
||||
if (!word.IsNonTerminal()) {
|
||||
EvaluateTerminal(
|
||||
word, hypoScore,
|
||||
newState, state,
|
||||
prefixFragments, prefixLength
|
||||
);
|
||||
EvaluateTerminal(
|
||||
word, hypoScore,
|
||||
newState, state,
|
||||
prefixFragments, prefixLength
|
||||
);
|
||||
}
|
||||
|
||||
// non-terminal, add phrase from underlying hypothesis
|
||||
// internal non-terminal
|
||||
else {
|
||||
// look up underlying hypothesis
|
||||
const ChartHypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndexMap[phrasePos]);
|
||||
const DALMChartState* prevState =
|
||||
static_cast<const DALMChartState*>(prevHypo->GetFFState(featureID));
|
||||
size_t prevTargetPhraseLength = prevHypo->GetCurrTargetPhrase().GetSize();
|
||||
float prevHypoScore = UntransformLMScore(prevHypo->GetScoreBreakdown().GetScoresForProducer(this)[0]);
|
||||
hypoSizeAll += prevState->GetHypoSize()-1;
|
||||
const ChartHypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndexMap[phrasePos]);
|
||||
const DALMChartState* prevState =
|
||||
static_cast<const DALMChartState*>(prevHypo->GetFFState(featureID));
|
||||
size_t prevTargetPhraseLength = prevHypo->GetCurrTargetPhrase().GetSize();
|
||||
float prevHypoScore = UntransformLMScore(prevHypo->GetScoreBreakdown().GetScoresForProducer(this)[0]);
|
||||
hypoSizeAll += prevState->GetHypoSize()-1;
|
||||
|
||||
EvaluateNonTerminal(
|
||||
word, hypoScore,
|
||||
newState, state,
|
||||
prefixFragments, prefixLength,
|
||||
prevState, prevTargetPhraseLength, prevHypoScore
|
||||
);
|
||||
EvaluateNonTerminal(
|
||||
word, hypoScore,
|
||||
newState, state,
|
||||
prefixFragments, prefixLength,
|
||||
prevState, prevTargetPhraseLength, prevHypoScore
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -477,37 +478,38 @@ void LanguageModelDALM::SetParameter(const std::string& key, const std::string&
|
||||
} else {
|
||||
LanguageModel::SetParameter(key, value);
|
||||
}
|
||||
m_ContextSize = m_nGramOrder-1;
|
||||
m_ContextSize = m_nGramOrder-1;
|
||||
}
|
||||
|
||||
void LanguageModelDALM::EvaluateTerminal(
|
||||
const Word &word,
|
||||
float &hypoScore,
|
||||
DALMChartState *newState,
|
||||
DALM::State &state,
|
||||
DALM::Fragment *prefixFragments,
|
||||
unsigned char &prefixLength) const{
|
||||
const Word &word,
|
||||
float &hypoScore,
|
||||
DALMChartState *newState,
|
||||
DALM::State &state,
|
||||
DALM::Fragment *prefixFragments,
|
||||
unsigned char &prefixLength) const
|
||||
{
|
||||
|
||||
DALM::VocabId wid = GetVocabId(word.GetFactor(m_factorType));
|
||||
if (newState->LargeEnough()) {
|
||||
float score = m_lm->query(wid, state);
|
||||
hypoScore += score;
|
||||
}else{
|
||||
float score = m_lm->query(wid, state, prefixFragments[prefixLength]);
|
||||
DALM::VocabId wid = GetVocabId(word.GetFactor(m_factorType));
|
||||
if (newState->LargeEnough()) {
|
||||
float score = m_lm->query(wid, state);
|
||||
hypoScore += score;
|
||||
} else {
|
||||
float score = m_lm->query(wid, state, prefixFragments[prefixLength]);
|
||||
|
||||
if(score > 0){
|
||||
hypoScore -= score;
|
||||
newState->SetAsLarge();
|
||||
}else if(state.get_count()<=prefixLength){
|
||||
hypoScore += score;
|
||||
prefixLength++;
|
||||
newState->SetAsLarge();
|
||||
}else{
|
||||
hypoScore += score;
|
||||
prefixLength++;
|
||||
if(prefixLength >= m_ContextSize) newState->SetAsLarge();
|
||||
}
|
||||
}
|
||||
if(score > 0) {
|
||||
hypoScore -= score;
|
||||
newState->SetAsLarge();
|
||||
} else if(state.get_count()<=prefixLength) {
|
||||
hypoScore += score;
|
||||
prefixLength++;
|
||||
newState->SetAsLarge();
|
||||
} else {
|
||||
hypoScore += score;
|
||||
prefixLength++;
|
||||
if(prefixLength >= m_ContextSize) newState->SetAsLarge();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void LanguageModelDALM::EvaluateNonTerminal(
|
||||
@ -517,74 +519,75 @@ void LanguageModelDALM::EvaluateNonTerminal(
|
||||
DALM::State &state,
|
||||
DALM::Fragment *prefixFragments,
|
||||
unsigned char &prefixLength,
|
||||
const DALMChartState *prevState,
|
||||
size_t prevTargetPhraseLength,
|
||||
float prevHypoScore
|
||||
) const{
|
||||
const DALMChartState *prevState,
|
||||
size_t prevTargetPhraseLength,
|
||||
float prevHypoScore
|
||||
) const
|
||||
{
|
||||
|
||||
const unsigned char prevPrefixLength = prevState->GetPrefixLength();
|
||||
const DALM::Fragment *prevPrefixFragments = prevState->GetPrefixFragments();
|
||||
hypoScore += prevHypoScore;
|
||||
const DALM::Fragment *prevPrefixFragments = prevState->GetPrefixFragments();
|
||||
hypoScore += prevHypoScore;
|
||||
|
||||
if(prevPrefixLength == 0){
|
||||
newState->SetAsLarge();
|
||||
hypoScore += state.sum_bows(0, state.get_count());
|
||||
state = prevState->GetRightContext();
|
||||
return;
|
||||
}
|
||||
if(!state.has_context()){
|
||||
newState->SetAsLarge();
|
||||
state = prevState->GetRightContext();
|
||||
return;
|
||||
}
|
||||
DALM::Gap gap(state);
|
||||
if(prevPrefixLength == 0) {
|
||||
newState->SetAsLarge();
|
||||
hypoScore += state.sum_bows(0, state.get_count());
|
||||
state = prevState->GetRightContext();
|
||||
return;
|
||||
}
|
||||
if(!state.has_context()) {
|
||||
newState->SetAsLarge();
|
||||
state = prevState->GetRightContext();
|
||||
return;
|
||||
}
|
||||
DALM::Gap gap(state);
|
||||
|
||||
// score its prefix
|
||||
for(size_t prefixPos = 0; prefixPos < prevPrefixLength; prefixPos++) {
|
||||
const DALM::Fragment &f = prevPrefixFragments[prefixPos];
|
||||
if (newState->LargeEnough()) {
|
||||
float score = m_lm->query(f, state, gap);
|
||||
hypoScore += score;
|
||||
const DALM::Fragment &f = prevPrefixFragments[prefixPos];
|
||||
if (newState->LargeEnough()) {
|
||||
float score = m_lm->query(f, state, gap);
|
||||
hypoScore += score;
|
||||
|
||||
if(!gap.is_extended()){
|
||||
state = prevState->GetRightContext();
|
||||
return;
|
||||
}else if(state.get_count() <= prefixPos+1){
|
||||
state = prevState->GetRightContext();
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
DALM::Fragment &fnew = prefixFragments[prefixLength];
|
||||
float score = m_lm->query(f, state, gap, fnew);
|
||||
hypoScore += score;
|
||||
if(!gap.is_extended()) {
|
||||
state = prevState->GetRightContext();
|
||||
return;
|
||||
} else if(state.get_count() <= prefixPos+1) {
|
||||
state = prevState->GetRightContext();
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
DALM::Fragment &fnew = prefixFragments[prefixLength];
|
||||
float score = m_lm->query(f, state, gap, fnew);
|
||||
hypoScore += score;
|
||||
|
||||
if(!gap.is_extended()){
|
||||
newState->SetAsLarge();
|
||||
state = prevState->GetRightContext();
|
||||
return;
|
||||
}else if(state.get_count() <= prefixPos+1){
|
||||
if(!gap.is_finalized()) prefixLength++;
|
||||
newState->SetAsLarge();
|
||||
state = prevState->GetRightContext();
|
||||
return;
|
||||
}else if(gap.is_finalized()){
|
||||
newState->SetAsLarge();
|
||||
}else{
|
||||
prefixLength++;
|
||||
if(prefixLength >= m_ContextSize) newState->SetAsLarge();
|
||||
}
|
||||
}
|
||||
gap.succ();
|
||||
if(!gap.is_extended()) {
|
||||
newState->SetAsLarge();
|
||||
state = prevState->GetRightContext();
|
||||
return;
|
||||
} else if(state.get_count() <= prefixPos+1) {
|
||||
if(!gap.is_finalized()) prefixLength++;
|
||||
newState->SetAsLarge();
|
||||
state = prevState->GetRightContext();
|
||||
return;
|
||||
} else if(gap.is_finalized()) {
|
||||
newState->SetAsLarge();
|
||||
} else {
|
||||
prefixLength++;
|
||||
if(prefixLength >= m_ContextSize) newState->SetAsLarge();
|
||||
}
|
||||
}
|
||||
gap.succ();
|
||||
}
|
||||
|
||||
// check if we are dealing with a large sub-phrase
|
||||
if (prevState->LargeEnough()) {
|
||||
newState->SetAsLarge();
|
||||
if(prevPrefixLength < prevState->GetHypoSize()){
|
||||
hypoScore += state.sum_bows(prevPrefixLength, state.get_count());
|
||||
}
|
||||
// copy language model state
|
||||
state = prevState->GetRightContext();
|
||||
if(prevPrefixLength < prevState->GetHypoSize()) {
|
||||
hypoScore += state.sum_bows(prevPrefixLength, state.get_count());
|
||||
}
|
||||
// copy language model state
|
||||
state = prevState->GetRightContext();
|
||||
} else {
|
||||
m_lm->set_state(state, prevState->GetRightContext(), prevPrefixFragments, gap);
|
||||
}
|
||||
|
@ -49,7 +49,7 @@ protected:
|
||||
|
||||
std::string m_filePath;
|
||||
size_t m_nGramOrder; //! max n-gram length contained in this LM
|
||||
size_t m_ContextSize;
|
||||
size_t m_ContextSize;
|
||||
|
||||
DALM::Logger *m_logger;
|
||||
DALM::Vocabulary *m_vocab;
|
||||
@ -77,26 +77,26 @@ private:
|
||||
}
|
||||
}
|
||||
|
||||
void EvaluateTerminal(
|
||||
const Word &word,
|
||||
float &hypoScore,
|
||||
DALMChartState *newState,
|
||||
DALM::State &state,
|
||||
DALM::Fragment *prefixFragments,
|
||||
unsigned char &prefixLength
|
||||
) const;
|
||||
void EvaluateTerminal(
|
||||
const Word &word,
|
||||
float &hypoScore,
|
||||
DALMChartState *newState,
|
||||
DALM::State &state,
|
||||
DALM::Fragment *prefixFragments,
|
||||
unsigned char &prefixLength
|
||||
) const;
|
||||
|
||||
void EvaluateNonTerminal(
|
||||
const Word &word,
|
||||
float &hypoScore,
|
||||
DALMChartState *newState,
|
||||
DALM::State &state,
|
||||
DALM::Fragment *prefixFragments,
|
||||
unsigned char &prefixLength,
|
||||
const DALMChartState *prevState,
|
||||
size_t prevTargetPhraseLength,
|
||||
float prevHypoScore
|
||||
) const;
|
||||
void EvaluateNonTerminal(
|
||||
const Word &word,
|
||||
float &hypoScore,
|
||||
DALMChartState *newState,
|
||||
DALM::State &state,
|
||||
DALM::Fragment *prefixFragments,
|
||||
unsigned char &prefixLength,
|
||||
const DALMChartState *prevState,
|
||||
size_t prevTargetPhraseLength,
|
||||
float prevHypoScore
|
||||
) const;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -78,13 +78,13 @@ boost::shared_ptr<PhraseProperty> PhrasePropertyFactory::ProduceProperty(const s
|
||||
|
||||
void PhrasePropertyFactory::PrintPP() const
|
||||
{
|
||||
std::cerr << "Registered phrase properties:" << std::endl;
|
||||
Registry::const_iterator iter;
|
||||
for (iter = m_registry.begin(); iter != m_registry.end(); ++iter) {
|
||||
const std::string &ppName = iter->first;
|
||||
std::cerr << ppName << " ";
|
||||
}
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "Registered phrase properties:" << std::endl;
|
||||
Registry::const_iterator iter;
|
||||
for (iter = m_registry.begin(); iter != m_registry.end(); ++iter) {
|
||||
const std::string &ppName = iter->first;
|
||||
std::cerr << ppName << " ";
|
||||
}
|
||||
std::cerr << std::endl;
|
||||
}
|
||||
|
||||
} // namespace Moses
|
||||
|
@ -15,11 +15,13 @@ public:
|
||||
|
||||
virtual void ProcessValue() {};
|
||||
|
||||
const std::string &GetValueString() { return m_value; };
|
||||
const std::string &GetValueString() {
|
||||
return m_value;
|
||||
};
|
||||
|
||||
protected:
|
||||
|
||||
const std::string m_value;
|
||||
const std::string m_value;
|
||||
|
||||
};
|
||||
|
||||
|
@ -920,20 +920,20 @@ void Parameter::CreateWeightsMap()
|
||||
void Parameter::CreateWeightsMap(const PARAM_VEC &vec)
|
||||
{
|
||||
for (size_t i = 0; i < vec.size(); ++i) {
|
||||
const string &line = vec[i];
|
||||
vector<string> toks = Tokenize(line);
|
||||
UTIL_THROW_IF2(toks.size() < 2,
|
||||
"Error in format of weights: " << line);
|
||||
const string &line = vec[i];
|
||||
vector<string> toks = Tokenize(line);
|
||||
UTIL_THROW_IF2(toks.size() < 2,
|
||||
"Error in format of weights: " << line);
|
||||
|
||||
string name = toks[0];
|
||||
name = name.substr(0, name.size() - 1);
|
||||
string name = toks[0];
|
||||
name = name.substr(0, name.size() - 1);
|
||||
|
||||
vector<float> weights(toks.size() - 1);
|
||||
for (size_t i = 1; i < toks.size(); ++i) {
|
||||
float weight = Scan<float>(toks[i]);
|
||||
weights[i - 1] = weight;
|
||||
}
|
||||
m_weights[name] = weights;
|
||||
vector<float> weights(toks.size() - 1);
|
||||
for (size_t i = 1; i < toks.size(); ++i) {
|
||||
float weight = Scan<float>(toks[i]);
|
||||
weights[i - 1] = weight;
|
||||
}
|
||||
m_weights[name] = weights;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -140,8 +140,9 @@ public:
|
||||
}
|
||||
|
||||
size_t GetNumTerminals() const;
|
||||
size_t GetNumNonTerminals() const
|
||||
{ return GetSize() - GetNumTerminals(); }
|
||||
size_t GetNumNonTerminals() const {
|
||||
return GetSize() - GetNumTerminals();
|
||||
}
|
||||
|
||||
//! whether the 2D vector is a substring of this phrase
|
||||
bool Contains(const std::vector< std::vector<std::string> > &subPhraseVector
|
||||
|
@ -158,7 +158,7 @@ int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder)
|
||||
// if sentences is specified as "<passthrough tag1=""/>"
|
||||
if (staticData.IsPassthroughEnabled() || staticData.IsPassthroughInNBestEnabled()) {
|
||||
std::string passthrough = PassthroughSGML(line,"passthrough");
|
||||
this->SetPassthroughInformation(passthrough);
|
||||
this->SetPassthroughInformation(passthrough);
|
||||
}
|
||||
|
||||
|
||||
@ -347,10 +347,10 @@ std::vector <ChartTranslationOptions*> Sentence::GetXmlChartTranslationOptions()
|
||||
return ret;
|
||||
}
|
||||
|
||||
void
|
||||
void
|
||||
Sentence::
|
||||
CreateFromString(const std::vector<FactorType> &factorOrder,
|
||||
const std::string &phraseString)
|
||||
CreateFromString(const std::vector<FactorType> &factorOrder,
|
||||
const std::string &phraseString)
|
||||
// , const std::string &factorDelimiter)
|
||||
{
|
||||
// Phrase::CreateFromString(Input, factorOrder, phraseString, factorDelimiter, NULL);
|
||||
|
@ -758,8 +758,9 @@ public:
|
||||
return m_registry;
|
||||
}
|
||||
|
||||
const PhrasePropertyFactory &GetPhrasePropertyFactory() const
|
||||
{ return m_phrasePropertyFactory; }
|
||||
const PhrasePropertyFactory &GetPhrasePropertyFactory() const {
|
||||
return m_phrasePropertyFactory;
|
||||
}
|
||||
|
||||
/** check whether we should be using the old code to support binary phrase-table.
|
||||
** eventually, we'll stop support the binary phrase-table and delete this legacy code
|
||||
|
@ -50,9 +50,9 @@ TargetPhrase::TargetPhrase( std::string out_string)
|
||||
|
||||
//ACAT
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
CreateFromString(Output, staticData.GetInputFactorOrder(), out_string,
|
||||
// staticData.GetFactorDelimiter(), // eliminated [UG]
|
||||
NULL);
|
||||
CreateFromString(Output, staticData.GetInputFactorOrder(), out_string,
|
||||
// staticData.GetFactorDelimiter(), // eliminated [UG]
|
||||
NULL);
|
||||
}
|
||||
|
||||
TargetPhrase::TargetPhrase()
|
||||
@ -239,7 +239,7 @@ void TargetPhrase::SetProperties(const StringPiece &str)
|
||||
}
|
||||
}
|
||||
|
||||
void TargetPhrase::SetProperty(const std::string &key, const std::string &value)
|
||||
void TargetPhrase::SetProperty(const std::string &key, const std::string &value)
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
const PhrasePropertyFactory& phrasePropertyFactory = staticData.GetPhrasePropertyFactory();
|
||||
|
@ -64,10 +64,10 @@ void CompletedRuleCollection::Add(const TargetPhraseCollection &tpc,
|
||||
|
||||
// Prune if bursting
|
||||
if (m_ruleLimit && m_collection.size() == m_ruleLimit * 2) {
|
||||
NTH_ELEMENT4(m_collection.begin(),
|
||||
m_collection.begin() + m_ruleLimit - 1,
|
||||
m_collection.end(),
|
||||
CompletedRuleOrdered());
|
||||
NTH_ELEMENT4(m_collection.begin(),
|
||||
m_collection.begin() + m_ruleLimit - 1,
|
||||
m_collection.end(),
|
||||
CompletedRuleOrdered());
|
||||
m_scoreThreshold = m_collection[m_ruleLimit-1]->GetScoreEstimate();
|
||||
for (size_t i = 0 + m_ruleLimit; i < m_collection.size(); i++) {
|
||||
delete m_collection[i];
|
||||
|
@ -164,7 +164,7 @@ GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
|
||||
|
||||
// backoff
|
||||
if (!SatisfyBackoff(inputPath)) {
|
||||
continue;
|
||||
continue;
|
||||
}
|
||||
|
||||
const Phrase &phrase = inputPath.GetPhrase();
|
||||
@ -225,25 +225,25 @@ bool PhraseDictionary::SatisfyBackoff(const InputPath &inputPath) const
|
||||
size_t backoff = decodeGraph.GetBackoff();
|
||||
|
||||
if (backoff == 0) {
|
||||
// ie. don't backoff. Collect ALL translations
|
||||
return true;
|
||||
// ie. don't backoff. Collect ALL translations
|
||||
return true;
|
||||
}
|
||||
|
||||
if (sourcePhrase.GetSize() > backoff) {
|
||||
// source phrase too big
|
||||
return false;
|
||||
// source phrase too big
|
||||
return false;
|
||||
}
|
||||
|
||||
// lookup translation only if no other translations
|
||||
InputPath::TargetPhrases::const_iterator iter;
|
||||
for (iter = inputPath.GetTargetPhrases().begin(); iter != inputPath.GetTargetPhrases().end(); ++iter) {
|
||||
const std::pair<const TargetPhraseCollection*, const void*> &temp = iter->second;
|
||||
const TargetPhraseCollection *tpCollPrev = temp.first;
|
||||
const std::pair<const TargetPhraseCollection*, const void*> &temp = iter->second;
|
||||
const TargetPhraseCollection *tpCollPrev = temp.first;
|
||||
|
||||
if (tpCollPrev && tpCollPrev->GetSize()) {
|
||||
// already have translation from another pt. Don't create translations
|
||||
return false;
|
||||
}
|
||||
if (tpCollPrev && tpCollPrev->GetSize()) {
|
||||
// already have translation from another pt. Don't create translations
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
|
@ -161,7 +161,7 @@ GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
|
||||
|
||||
// backoff
|
||||
if (!SatisfyBackoff(inputPath)) {
|
||||
continue;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (prevPtNode) {
|
||||
@ -173,7 +173,7 @@ GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
|
||||
const TargetPhraseCollection &targetPhrases = ptNode->GetTargetPhraseCollection();
|
||||
inputPath.SetTargetPhrases(*this, &targetPhrases, ptNode);
|
||||
} else {
|
||||
inputPath.SetTargetPhrases(*this, NULL, NULL);
|
||||
inputPath.SetTargetPhrases(*this, NULL, NULL);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -139,7 +139,7 @@ void PhraseDictionaryOnDisk::GetTargetPhraseCollectionBatch(InputPath &inputPath
|
||||
|
||||
// backoff
|
||||
if (!SatisfyBackoff(inputPath)) {
|
||||
return;
|
||||
return;
|
||||
}
|
||||
|
||||
if (prevPtNode) {
|
||||
|
@ -395,7 +395,7 @@ void TranslationOptionCollection::CreateTranslationOptions()
|
||||
for (size_t endPos = startPos ; endPos < startPos + maxSize ; endPos++) {
|
||||
// VERBOSE(1,"TranslationOptionCollection::CreateTranslationOptions() endPos:" << endPos << endl);
|
||||
if (graphInd > 0 && // only skip subsequent graphs
|
||||
backoff != 0 && // use of backoff specified
|
||||
backoff != 0 && // use of backoff specified
|
||||
(endPos-startPos+1 <= backoff || // size exceeds backoff limit or ...
|
||||
m_collection[startPos][endPos-startPos].size() > 0)) { // no phrases found so far
|
||||
VERBOSE(3,"No backoff to graph " << graphInd << " for span [" << startPos << ";" << endPos << "]" << endl);
|
||||
@ -623,8 +623,8 @@ void TranslationOptionCollection::Add(TranslationOption *translationOption)
|
||||
const WordsRange &coverage = translationOption->GetSourceWordsRange();
|
||||
|
||||
if (coverage.GetEndPos() - coverage.GetStartPos() >= m_collection[coverage.GetStartPos()].size()) {
|
||||
cerr << "translationOption=" << *translationOption << endl;
|
||||
cerr << "coverage=" << coverage << endl;
|
||||
cerr << "translationOption=" << *translationOption << endl;
|
||||
cerr << "coverage=" << coverage << endl;
|
||||
}
|
||||
|
||||
UTIL_THROW_IF2(coverage.GetEndPos() - coverage.GetStartPos() >= m_collection[coverage.GetStartPos()].size(),
|
||||
|
@ -154,8 +154,9 @@ public:
|
||||
return GetTranslationOptionList(coverage.GetStartPos(), coverage.GetEndPos());
|
||||
}
|
||||
|
||||
const InputPathList &GetInputPaths() const
|
||||
{ return m_inputPathQueue; }
|
||||
const InputPathList &GetInputPaths() const {
|
||||
return m_inputPathQueue;
|
||||
}
|
||||
|
||||
|
||||
TO_STRING();
|
||||
|
@ -48,7 +48,7 @@ TranslationOptionCollectionLattice::TranslationOptionCollectionLattice(
|
||||
WordsRange range(startPos, endPos);
|
||||
|
||||
if (range.GetNumWordsCovered() > maxPhraseLength) {
|
||||
continue;
|
||||
continue;
|
||||
}
|
||||
|
||||
const NonTerminalSet &labels = input.GetLabelSet(startPos, endPos);
|
||||
@ -73,53 +73,53 @@ TranslationOptionCollectionLattice::TranslationOptionCollectionLattice(
|
||||
|
||||
void TranslationOptionCollectionLattice::Extend(const InputPath &prevPath, const WordLattice &input)
|
||||
{
|
||||
size_t nextPos = prevPath.GetWordsRange().GetEndPos() + 1;
|
||||
if (nextPos >= input.GetSize()) {
|
||||
return;
|
||||
}
|
||||
size_t nextPos = prevPath.GetWordsRange().GetEndPos() + 1;
|
||||
if (nextPos >= input.GetSize()) {
|
||||
return;
|
||||
}
|
||||
|
||||
size_t startPos = prevPath.GetWordsRange().GetStartPos();
|
||||
const Phrase &prevPhrase = prevPath.GetPhrase();
|
||||
const ScorePair *prevInputScore = prevPath.GetInputScore();
|
||||
UTIL_THROW_IF2(prevInputScore == NULL,
|
||||
"Null previous score");
|
||||
size_t startPos = prevPath.GetWordsRange().GetStartPos();
|
||||
const Phrase &prevPhrase = prevPath.GetPhrase();
|
||||
const ScorePair *prevInputScore = prevPath.GetInputScore();
|
||||
UTIL_THROW_IF2(prevInputScore == NULL,
|
||||
"Null previous score");
|
||||
|
||||
|
||||
const std::vector<size_t> &nextNodes = input.GetNextNodes(nextPos);
|
||||
const std::vector<size_t> &nextNodes = input.GetNextNodes(nextPos);
|
||||
|
||||
const ConfusionNet::Column &col = input.GetColumn(nextPos);
|
||||
for (size_t i = 0; i < col.size(); ++i) {
|
||||
const Word &word = col[i].first;
|
||||
UTIL_THROW_IF2(word.IsEpsilon(), "Epsilon not supported");
|
||||
const ConfusionNet::Column &col = input.GetColumn(nextPos);
|
||||
for (size_t i = 0; i < col.size(); ++i) {
|
||||
const Word &word = col[i].first;
|
||||
UTIL_THROW_IF2(word.IsEpsilon(), "Epsilon not supported");
|
||||
|
||||
size_t nextNode = nextNodes[i];
|
||||
size_t endPos = nextPos + nextNode - 1;
|
||||
size_t nextNode = nextNodes[i];
|
||||
size_t endPos = nextPos + nextNode - 1;
|
||||
|
||||
WordsRange range(startPos, endPos);
|
||||
|
||||
size_t maxPhraseLength = StaticData::Instance().GetMaxPhraseLength();
|
||||
if (range.GetNumWordsCovered() > maxPhraseLength) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const NonTerminalSet &labels = input.GetLabelSet(startPos, endPos);
|
||||
|
||||
Phrase subphrase(prevPhrase);
|
||||
subphrase.AddWord(word);
|
||||
|
||||
const ScorePair &scores = col[i].second;
|
||||
ScorePair *inputScore = new ScorePair(*prevInputScore);
|
||||
inputScore->PlusEquals(scores);
|
||||
|
||||
InputPath *path = new InputPath(subphrase, labels, range, &prevPath, inputScore);
|
||||
|
||||
path->SetNextNode(nextNode);
|
||||
m_inputPathQueue.push_back(path);
|
||||
|
||||
// recursive
|
||||
Extend(*path, input);
|
||||
WordsRange range(startPos, endPos);
|
||||
|
||||
size_t maxPhraseLength = StaticData::Instance().GetMaxPhraseLength();
|
||||
if (range.GetNumWordsCovered() > maxPhraseLength) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const NonTerminalSet &labels = input.GetLabelSet(startPos, endPos);
|
||||
|
||||
Phrase subphrase(prevPhrase);
|
||||
subphrase.AddWord(word);
|
||||
|
||||
const ScorePair &scores = col[i].second;
|
||||
ScorePair *inputScore = new ScorePair(*prevInputScore);
|
||||
inputScore->PlusEquals(scores);
|
||||
|
||||
InputPath *path = new InputPath(subphrase, labels, range, &prevPath, inputScore);
|
||||
|
||||
path->SetNextNode(nextNode);
|
||||
m_inputPathQueue.push_back(path);
|
||||
|
||||
// recursive
|
||||
Extend(*path, input);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
void TranslationOptionCollectionLattice::CreateTranslationOptions()
|
||||
@ -154,10 +154,9 @@ void TranslationOptionCollectionLattice::CreateTranslationOptions()
|
||||
} else if (path.GetPhrase().GetSize() == 1) {
|
||||
// unknown word processing
|
||||
ProcessOneUnknownWord(path, path.GetWordsRange().GetEndPos(), 1, path.GetInputScore());
|
||||
}
|
||||
else if (path.GetPhrase().GetSize() == 1) {
|
||||
// unknown word processing
|
||||
ProcessOneUnknownWord(path, path.GetWordsRange().GetStartPos(), path.GetWordsRange().GetNumWordsCovered() , path.GetInputScore());
|
||||
} else if (path.GetPhrase().GetSize() == 1) {
|
||||
// unknown word processing
|
||||
ProcessOneUnknownWord(path, path.GetWordsRange().GetStartPos(), path.GetWordsRange().GetNumWordsCovered() , path.GetInputScore());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -34,15 +34,15 @@ using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
// utility function for factorless decoding
|
||||
size_t
|
||||
max_fax()
|
||||
{
|
||||
if (StaticData::Instance().GetFactorDelimiter().size())
|
||||
return MAX_NUM_FACTORS;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// utility function for factorless decoding
|
||||
size_t
|
||||
max_fax()
|
||||
{
|
||||
if (StaticData::Instance().GetFactorDelimiter().size())
|
||||
return MAX_NUM_FACTORS;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// static
|
||||
int Word::Compare(const Word &targetWord, const Word &sourceWord)
|
||||
@ -85,8 +85,8 @@ std::string Word::GetString(const vector<FactorType> factorType,bool endWithBlan
|
||||
unsigned int stop = min(max_fax(),factorType.size());
|
||||
for (unsigned int i = 0 ; i < stop ; i++) {
|
||||
UTIL_THROW_IF2(factorType[i] >= MAX_NUM_FACTORS,
|
||||
"Trying to reference factor " << factorType[i]
|
||||
<< ". Max factor is " << MAX_NUM_FACTORS);
|
||||
"Trying to reference factor " << factorType[i]
|
||||
<< ". Max factor is " << MAX_NUM_FACTORS);
|
||||
|
||||
const Factor *factor = m_factorArray[factorType[i]];
|
||||
if (factor != NULL) {
|
||||
@ -120,39 +120,35 @@ CreateFromString(FactorDirection direction
|
||||
FactorCollection &factorCollection = FactorCollection::Instance();
|
||||
vector<StringPiece> bits(MAX_NUM_FACTORS);
|
||||
string factorDelimiter = StaticData::Instance().GetFactorDelimiter();
|
||||
if (factorDelimiter.size())
|
||||
{
|
||||
util::TokenIter<util::MultiCharacter> fit(str, factorDelimiter);
|
||||
size_t i = 0;
|
||||
for (; i < MAX_NUM_FACTORS && fit; ++i,++fit)
|
||||
bits[i] = *fit;
|
||||
if (i == MAX_NUM_FACTORS)
|
||||
UTIL_THROW_IF(fit, StrayFactorException,
|
||||
"The hard limit for factors is " << MAX_NUM_FACTORS
|
||||
<< ". The word " << str << " contains factor delimiter "
|
||||
<< StaticData::Instance().GetFactorDelimiter()
|
||||
<< " too many times.");
|
||||
if (strict)
|
||||
UTIL_THROW_IF(fit, StrayFactorException,
|
||||
"You have configured " << factorOrder.size()
|
||||
<< " factors but the word " << str
|
||||
<< " contains factor delimiter "
|
||||
<< StaticData::Instance().GetFactorDelimiter()
|
||||
<< " too many times.");
|
||||
|
||||
UTIL_THROW_IF(i < factorOrder.size(),util::Exception,
|
||||
"Too few factors in string '" << str << "'.");
|
||||
}
|
||||
else
|
||||
{
|
||||
bits[0] = str;
|
||||
}
|
||||
for (size_t k = 0; k < factorOrder.size(); ++k)
|
||||
{
|
||||
UTIL_THROW_IF(factorOrder[k] >= MAX_NUM_FACTORS, util::Exception,
|
||||
"Factor order out of bounds.");
|
||||
m_factorArray[factorOrder[k]] = factorCollection.AddFactor(bits[k], isNonTerminal);
|
||||
}
|
||||
if (factorDelimiter.size()) {
|
||||
util::TokenIter<util::MultiCharacter> fit(str, factorDelimiter);
|
||||
size_t i = 0;
|
||||
for (; i < MAX_NUM_FACTORS && fit; ++i,++fit)
|
||||
bits[i] = *fit;
|
||||
if (i == MAX_NUM_FACTORS)
|
||||
UTIL_THROW_IF(fit, StrayFactorException,
|
||||
"The hard limit for factors is " << MAX_NUM_FACTORS
|
||||
<< ". The word " << str << " contains factor delimiter "
|
||||
<< StaticData::Instance().GetFactorDelimiter()
|
||||
<< " too many times.");
|
||||
if (strict)
|
||||
UTIL_THROW_IF(fit, StrayFactorException,
|
||||
"You have configured " << factorOrder.size()
|
||||
<< " factors but the word " << str
|
||||
<< " contains factor delimiter "
|
||||
<< StaticData::Instance().GetFactorDelimiter()
|
||||
<< " too many times.");
|
||||
|
||||
UTIL_THROW_IF(i < factorOrder.size(),util::Exception,
|
||||
"Too few factors in string '" << str << "'.");
|
||||
} else {
|
||||
bits[0] = str;
|
||||
}
|
||||
for (size_t k = 0; k < factorOrder.size(); ++k) {
|
||||
UTIL_THROW_IF(factorOrder[k] >= MAX_NUM_FACTORS, util::Exception,
|
||||
"Factor order out of bounds.");
|
||||
m_factorArray[factorOrder[k]] = factorCollection.AddFactor(bits[k], isNonTerminal);
|
||||
}
|
||||
// assume term/non-term same for all factors
|
||||
m_isNonTerminal = isNonTerminal;
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user