mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
Merge ../mosesdecoder into perf_moses2
This commit is contained in:
commit
6432fefd3b
@ -145,12 +145,8 @@ SetCache(TranslationOption& to) const
|
||||
Phrase const& tphrase = to.GetTargetPhrase();
|
||||
to.CacheLexReorderingScores(*this, this->GetProb(sphrase,tphrase));
|
||||
} else { // e.g. OOV with Mmsapt
|
||||
if (m_haveDefaultScores) {
|
||||
to.CacheLexReorderingScores(*this, m_defaultScores);
|
||||
} else {
|
||||
Scores vals(GetNumScoreComponents(), 0);
|
||||
to.CacheLexReorderingScores(*this, vals);
|
||||
}
|
||||
// Scores vals(GetNumScoreComponents(), 0);
|
||||
// to.CacheLexReorderingScores(*this, vals);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -243,11 +243,12 @@ CopyScores(ScoreComponentCollection* accum,
|
||||
size_t off_remote = m_offset + reoType;
|
||||
size_t off_local = m_configuration.CollapseScores() ? m_offset : off_remote;
|
||||
|
||||
UTIL_THROW_IF2(off_remote >= producer->GetNumScoreComponents(),
|
||||
UTIL_THROW_IF2(off_local >= producer->GetNumScoreComponents(),
|
||||
"offset out of vector bounds!");
|
||||
|
||||
// look up applicable score from vectore of scores
|
||||
if(cached) {
|
||||
UTIL_THROW_IF2(off_remote >= cached->size(), "offset out of vector bounds!");
|
||||
Scores scores(producer->GetNumScoreComponents(),0);
|
||||
scores[off_local ] = (*cached)[off_remote];
|
||||
accum->PlusEquals(producer, scores);
|
||||
|
@ -393,9 +393,7 @@ template <class Model> FFState *LanguageModelKen<Model>::EvaluateWhenApplied(con
|
||||
// Non-terminal is first so we can copy instead of rescoring.
|
||||
const Syntax::SVertex *pred = hyperedge.tail[nonTermIndexMap[phrasePos]];
|
||||
const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(pred->states[featureID])->GetChartState();
|
||||
float prob = UntransformLMScore(
|
||||
pred->best->label.scoreBreakdown.GetScoresForProducer(this)[0]);
|
||||
ruleScore.BeginNonTerminal(prevState, prob);
|
||||
ruleScore.BeginNonTerminal(prevState);
|
||||
phrasePos++;
|
||||
}
|
||||
}
|
||||
@ -405,9 +403,7 @@ template <class Model> FFState *LanguageModelKen<Model>::EvaluateWhenApplied(con
|
||||
if (word.IsNonTerminal()) {
|
||||
const Syntax::SVertex *pred = hyperedge.tail[nonTermIndexMap[phrasePos]];
|
||||
const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(pred->states[featureID])->GetChartState();
|
||||
float prob = UntransformLMScore(
|
||||
pred->best->label.scoreBreakdown.GetScoresForProducer(this)[0]);
|
||||
ruleScore.NonTerminal(prevState, prob);
|
||||
ruleScore.NonTerminal(prevState);
|
||||
} else {
|
||||
ruleScore.Terminal(TranslateID(word));
|
||||
}
|
||||
@ -415,7 +411,16 @@ template <class Model> FFState *LanguageModelKen<Model>::EvaluateWhenApplied(con
|
||||
|
||||
float score = ruleScore.Finish();
|
||||
score = TransformLMScore(score);
|
||||
accumulator->Assign(this, score);
|
||||
score -= target.GetScoreBreakdown().GetScoresForProducer(this)[0];
|
||||
|
||||
if (OOVFeatureEnabled()) {
|
||||
std::vector<float> scores(2);
|
||||
scores[0] = score;
|
||||
scores[1] = 0.0;
|
||||
accumulator->PlusEquals(this, scores);
|
||||
} else {
|
||||
accumulator->PlusEquals(this, score);
|
||||
}
|
||||
return newState;
|
||||
}
|
||||
|
||||
|
@ -457,6 +457,8 @@ void StaticData::LoadDecodeGraphs()
|
||||
params = m_parameter->GetParam("mapping");
|
||||
if (params && params->size()) {
|
||||
mappingVector = *params;
|
||||
} else {
|
||||
mappingVector.assign(1,"0 T 0");
|
||||
}
|
||||
|
||||
params = m_parameter->GetParam("max-chart-span");
|
||||
|
@ -101,10 +101,6 @@ SHyperedge *Cube::CreateHyperedge(const std::vector<int> &coordinates)
|
||||
for (std::size_t i = 0; i < coordinates.size()-1; ++i) {
|
||||
boost::shared_ptr<SVertex> pred = (*m_bundle.stacks[i])[coordinates[i]];
|
||||
hyperedge->tail[i] = pred.get();
|
||||
if (pred->best) {
|
||||
hyperedge->label.scoreBreakdown.PlusEquals(
|
||||
pred->best->label.scoreBreakdown);
|
||||
}
|
||||
}
|
||||
|
||||
hyperedge->label.inputWeight = m_bundle.inputWeight;
|
||||
@ -112,8 +108,7 @@ SHyperedge *Cube::CreateHyperedge(const std::vector<int> &coordinates)
|
||||
hyperedge->label.translation =
|
||||
*(m_bundle.translations->begin()+coordinates.back());
|
||||
|
||||
hyperedge->label.scoreBreakdown.PlusEquals(
|
||||
hyperedge->label.translation->GetScoreBreakdown());
|
||||
// Calculate feature deltas.
|
||||
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
|
||||
@ -123,7 +118,7 @@ SHyperedge *Cube::CreateHyperedge(const std::vector<int> &coordinates)
|
||||
StatelessFeatureFunction::GetStatelessFeatureFunctions();
|
||||
for (unsigned i = 0; i < sfs.size(); ++i) {
|
||||
if (!staticData.IsFeatureFunctionIgnored(*sfs[i])) {
|
||||
sfs[i]->EvaluateWhenApplied(*hyperedge, &hyperedge->label.scoreBreakdown);
|
||||
sfs[i]->EvaluateWhenApplied(*hyperedge, &hyperedge->label.deltas);
|
||||
}
|
||||
}
|
||||
|
||||
@ -132,12 +127,24 @@ SHyperedge *Cube::CreateHyperedge(const std::vector<int> &coordinates)
|
||||
for (unsigned i = 0; i < ffs.size(); ++i) {
|
||||
if (!staticData.IsFeatureFunctionIgnored(*ffs[i])) {
|
||||
head->states[i] =
|
||||
ffs[i]->EvaluateWhenApplied(*hyperedge, i,
|
||||
&hyperedge->label.scoreBreakdown);
|
||||
ffs[i]->EvaluateWhenApplied(*hyperedge, i, &hyperedge->label.deltas);
|
||||
}
|
||||
}
|
||||
|
||||
hyperedge->label.score = hyperedge->label.scoreBreakdown.GetWeightedScore();
|
||||
// Calculate future score.
|
||||
|
||||
hyperedge->label.futureScore =
|
||||
hyperedge->label.translation->GetScoreBreakdown().GetWeightedScore();
|
||||
|
||||
hyperedge->label.futureScore += hyperedge->label.deltas.GetWeightedScore();
|
||||
|
||||
for (std::vector<SVertex*>::const_iterator p = hyperedge->tail.begin();
|
||||
p != hyperedge->tail.end(); ++p) {
|
||||
const SVertex *pred = *p;
|
||||
if (pred->best) {
|
||||
hyperedge->label.futureScore += pred->best->label.futureScore;
|
||||
}
|
||||
}
|
||||
|
||||
return hyperedge;
|
||||
}
|
||||
|
@ -42,7 +42,7 @@ private:
|
||||
{
|
||||
public:
|
||||
bool operator()(const QueueItem &p, const QueueItem &q) const {
|
||||
return p.first->label.score < q.first->label.score;
|
||||
return p.first->label.futureScore < q.first->label.futureScore;
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -31,7 +31,7 @@ private:
|
||||
{
|
||||
public:
|
||||
bool operator()(const Cube *p, const Cube *q) const {
|
||||
return p->Top()->label.score < q->Top()->label.score;
|
||||
return p->Top()->label.futureScore < q->Top()->label.futureScore;
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -304,7 +304,7 @@ void Manager<RuleMatcher>::RecombineAndSort(
|
||||
// Compare the score of h against the score of the best incoming hyperedge
|
||||
// for the stored vertex.
|
||||
SVertex *storedVertex = result.first->second;
|
||||
if (h->label.score > storedVertex->best->label.score) {
|
||||
if (h->label.futureScore > storedVertex->best->label.futureScore) {
|
||||
// h's score is better.
|
||||
storedVertex->recombined.push_back(storedVertex->best);
|
||||
storedVertex->best = h;
|
||||
|
@ -32,24 +32,25 @@ void KBestExtractor::Extract(
|
||||
supremeVertex->best = new SHyperedge();
|
||||
supremeVertex->best->head = supremeVertex.get();
|
||||
supremeVertex->best->tail.push_back(&bestTopLevelVertex);
|
||||
supremeVertex->best->label.score = bestTopLevelVertex.best->label.score;
|
||||
supremeVertex->best->label.scoreBreakdown =
|
||||
bestTopLevelVertex.best->label.scoreBreakdown;
|
||||
supremeVertex->best->label.futureScore =
|
||||
bestTopLevelVertex.best->label.futureScore;
|
||||
supremeVertex->best->label.deltas = bestTopLevelVertex.best->label.deltas;
|
||||
supremeVertex->best->label.translation = 0;
|
||||
|
||||
// For each alternative top-level SVertex, add a new incoming hyperedge to
|
||||
// supremeVertex.
|
||||
for (++p; p != topLevelVertices.end(); ++p) {
|
||||
// Check that the first item in topLevelVertices really was the best.
|
||||
UTIL_THROW_IF2((*p)->best->label.score > bestTopLevelVertex.best->label.score,
|
||||
UTIL_THROW_IF2((*p)->best->label.futureScore >
|
||||
bestTopLevelVertex.best->label.futureScore,
|
||||
"top-level SVertices are not correctly sorted");
|
||||
// Note: there's no need for a smart pointer here: supremeVertex will take
|
||||
// ownership of altEdge.
|
||||
SHyperedge *altEdge = new SHyperedge();
|
||||
altEdge->head = supremeVertex.get();
|
||||
altEdge->tail.push_back((*p).get());
|
||||
altEdge->label.score = (*p)->best->label.score;
|
||||
altEdge->label.scoreBreakdown = (*p)->best->label.scoreBreakdown;
|
||||
altEdge->label.futureScore = (*p)->best->label.futureScore;
|
||||
altEdge->label.deltas = (*p)->best->label.deltas;
|
||||
altEdge->label.translation = 0;
|
||||
supremeVertex->recombined.push_back(altEdge);
|
||||
}
|
||||
@ -282,7 +283,13 @@ void KBestExtractor::LazyNext(KVertex &v, const Derivation &d,
|
||||
KBestExtractor::Derivation::Derivation(const boost::shared_ptr<KHyperedge> &e)
|
||||
{
|
||||
edge = e;
|
||||
std::size_t arity = edge->tail.size();
|
||||
const TargetPhrase *translation = edge->shyperedge.label.translation;
|
||||
// Every hyperedge should have an associated target phrase, except for
|
||||
// incoming hyperedges of the 'supreme' vertex.
|
||||
if (translation) {
|
||||
scoreBreakdown = translation->GetScoreBreakdown();
|
||||
}
|
||||
const std::size_t arity = edge->tail.size();
|
||||
backPointers.resize(arity, 0);
|
||||
subderivations.reserve(arity);
|
||||
for (std::size_t i = 0; i < arity; ++i) {
|
||||
@ -290,9 +297,10 @@ KBestExtractor::Derivation::Derivation(const boost::shared_ptr<KHyperedge> &e)
|
||||
assert(pred.kBestList.size() >= 1);
|
||||
boost::shared_ptr<Derivation> sub(pred.kBestList[0]);
|
||||
subderivations.push_back(sub);
|
||||
scoreBreakdown.PlusEquals(sub->scoreBreakdown);
|
||||
}
|
||||
score = edge->shyperedge.label.score;
|
||||
scoreBreakdown = edge->shyperedge.label.scoreBreakdown;
|
||||
scoreBreakdown.PlusEquals(edge->shyperedge.label.deltas);
|
||||
score = scoreBreakdown.GetWeightedScore();
|
||||
}
|
||||
|
||||
// Construct a Derivation that neighbours an existing Derivation.
|
||||
|
@ -32,7 +32,7 @@ void Manager::OutputBest(OutputCollector *collector) const
|
||||
out << '\n';
|
||||
} else {
|
||||
if (options().output.ReportHypoScore) {
|
||||
out << best->label.score << " ";
|
||||
out << best->label.futureScore << " ";
|
||||
}
|
||||
Phrase yield = GetOneBestTargetYield(*best);
|
||||
// delete 1st & last
|
||||
|
@ -368,7 +368,7 @@ void Manager<Parser>::RecombineAndSort(const std::vector<SHyperedge*> &buffer,
|
||||
// Compare the score of h against the score of the best incoming hyperedge
|
||||
// for the stored vertex.
|
||||
SVertex *storedVertex = result.first->second;
|
||||
if (h->label.score > storedVertex->best->label.score) {
|
||||
if (h->label.futureScore > storedVertex->best->label.futureScore) {
|
||||
// h's score is better.
|
||||
storedVertex->recombined.push_back(storedVertex->best);
|
||||
storedVertex->best = h;
|
||||
|
@ -16,7 +16,7 @@ public:
|
||||
bundle.stacks.begin(); p != bundle.stacks.end(); ++p) {
|
||||
const SVertexStack *stack = *p;
|
||||
if (stack->front()->best) {
|
||||
score += stack->front()->best->label.score;
|
||||
score += stack->front()->best->label.futureScore;
|
||||
}
|
||||
}
|
||||
return score;
|
||||
|
@ -8,11 +8,24 @@ namespace Moses
|
||||
namespace Syntax
|
||||
{
|
||||
|
||||
// A SHyperedge label.
|
||||
//
|
||||
struct SLabel {
|
||||
float inputWeight;
|
||||
float score;
|
||||
ScoreComponentCollection scoreBreakdown;
|
||||
// Deltas for individual feature scores. i.e. this object records the change
|
||||
// in each feature score that results from applying the rule associated with
|
||||
// this hyperedge.
|
||||
ScoreComponentCollection deltas;
|
||||
|
||||
// Total derivation score to be used for comparison in beam search (i.e.
|
||||
// including future cost estimates). This is the sum of the 1-best
|
||||
// subderivations' future scores + deltas.
|
||||
float futureScore;
|
||||
|
||||
// Target-side of the grammar rule.
|
||||
const TargetPhrase *translation;
|
||||
|
||||
// Input weight of this hyperedge (e.g. from weighted input forest).
|
||||
float inputWeight;
|
||||
};
|
||||
|
||||
} // Syntax
|
||||
|
@ -18,7 +18,7 @@ struct SVertexStackContentOrderer {
|
||||
public:
|
||||
bool operator()(const boost::shared_ptr<SVertex> &x,
|
||||
const boost::shared_ptr<SVertex> &y) {
|
||||
return x->best->label.score > y->best->label.score;
|
||||
return x->best->label.futureScore > y->best->label.futureScore;
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -264,7 +264,7 @@ void Manager<RuleMatcher>::RecombineAndSort(
|
||||
// Compare the score of h against the score of the best incoming hyperedge
|
||||
// for the stored vertex.
|
||||
SVertex *storedVertex = result.first->second;
|
||||
if (h->label.score > storedVertex->best->label.score) {
|
||||
if (h->label.futureScore > storedVertex->best->label.futureScore) {
|
||||
// h's score is better.
|
||||
storedVertex->recombined.push_back(storedVertex->best);
|
||||
storedVertex->best = h;
|
||||
|
@ -33,8 +33,15 @@ namespace Moses
|
||||
PhraseDictionaryGroup::PhraseDictionaryGroup(const string &line)
|
||||
: PhraseDictionary(line, true),
|
||||
m_numModels(0),
|
||||
m_totalModelScores(0),
|
||||
m_phraseCounts(false),
|
||||
m_wordCounts(false),
|
||||
m_modelBitmapCounts(false),
|
||||
m_restrict(false),
|
||||
m_haveDefaultScores(false)
|
||||
m_haveDefaultScores(false),
|
||||
m_defaultAverageOthers(false),
|
||||
m_scoresPerModel(0),
|
||||
m_haveMmsaptLrFunc(false)
|
||||
{
|
||||
ReadParameters();
|
||||
}
|
||||
@ -44,11 +51,23 @@ void PhraseDictionaryGroup::SetParameter(const string& key, const string& value)
|
||||
if (key == "members") {
|
||||
m_memberPDStrs = Tokenize(value, ",");
|
||||
m_numModels = m_memberPDStrs.size();
|
||||
m_seenByAll = dynamic_bitset<>(m_numModels);
|
||||
m_seenByAll.set();
|
||||
} else if (key == "restrict") {
|
||||
m_restrict = Scan<bool>(value);
|
||||
} else if (key == "phrase-counts") {
|
||||
m_phraseCounts = Scan<bool>(value);
|
||||
} else if (key == "word-counts") {
|
||||
m_wordCounts = Scan<bool>(value);
|
||||
} else if (key == "model-bitmap-counts") {
|
||||
m_modelBitmapCounts = Scan<bool>(value);
|
||||
} else if (key =="default-scores") {
|
||||
m_haveDefaultScores = true;
|
||||
m_defaultScores = Scan<float>(Tokenize(value, ","));
|
||||
} else if (key =="default-average-others") {
|
||||
m_defaultAverageOthers = Scan<bool>(value);
|
||||
} else if (key =="mmsapt-lr-func") {
|
||||
m_haveMmsaptLrFunc = true;
|
||||
} else {
|
||||
PhraseDictionary::SetParameter(key, value);
|
||||
}
|
||||
@ -58,28 +77,58 @@ void PhraseDictionaryGroup::Load()
|
||||
{
|
||||
SetFeaturesToApply();
|
||||
m_pdFeature.push_back(const_cast<PhraseDictionaryGroup*>(this));
|
||||
size_t numScoreComponents = 0;
|
||||
|
||||
// Locate/check component phrase tables
|
||||
size_t componentWeights = 0;
|
||||
BOOST_FOREACH(const string& pdName, m_memberPDStrs) {
|
||||
bool pdFound = false;
|
||||
BOOST_FOREACH(PhraseDictionary* pd, PhraseDictionary::GetColl()) {
|
||||
if (pd->GetScoreProducerDescription() == pdName) {
|
||||
pdFound = true;
|
||||
m_memberPDs.push_back(pd);
|
||||
componentWeights += pd->GetNumScoreComponents();
|
||||
size_t nScores = pd->GetNumScoreComponents();
|
||||
numScoreComponents += nScores;
|
||||
if (m_scoresPerModel == 0) {
|
||||
m_scoresPerModel = nScores;
|
||||
} else if (m_defaultAverageOthers) {
|
||||
UTIL_THROW_IF2(nScores != m_scoresPerModel,
|
||||
m_description << ": member models must have the same number of scores when using default-average-others");
|
||||
}
|
||||
}
|
||||
}
|
||||
UTIL_THROW_IF2(!pdFound,
|
||||
"Could not find member phrase table " << pdName);
|
||||
m_description << ": could not find member phrase table " << pdName);
|
||||
}
|
||||
UTIL_THROW_IF2(componentWeights != m_numScoreComponents,
|
||||
"Total number of member model scores is unequal to specified number of scores");
|
||||
m_totalModelScores = numScoreComponents;
|
||||
|
||||
// Check feature total
|
||||
if (m_phraseCounts) {
|
||||
numScoreComponents += m_numModels;
|
||||
}
|
||||
if (m_wordCounts) {
|
||||
numScoreComponents += m_numModels;
|
||||
}
|
||||
if (m_modelBitmapCounts) {
|
||||
numScoreComponents += (pow(2, m_numModels) - 1);
|
||||
}
|
||||
UTIL_THROW_IF2(numScoreComponents != m_numScoreComponents,
|
||||
m_description << ": feature count mismatch: specify \"num-features=" << numScoreComponents << "\" and supply " << numScoreComponents << " weights");
|
||||
|
||||
#ifdef PT_UG
|
||||
// Locate mmsapt lexical reordering functions if specified
|
||||
if (m_haveMmsaptLrFunc) {
|
||||
BOOST_FOREACH(PhraseDictionary* pd, m_memberPDs) {
|
||||
// pointer to pointer, all start as NULL and some may be populated prior
|
||||
// to translation
|
||||
m_mmsaptLrFuncs.push_back(&(static_cast<Mmsapt*>(pd)->m_lr_func));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
// Determine "zero" scores for features
|
||||
if (m_haveDefaultScores) {
|
||||
UTIL_THROW_IF2(m_defaultScores.size() != m_numScoreComponents,
|
||||
"Number of specified default scores is unequal to number of member model scores");
|
||||
m_description << ": number of specified default scores is unequal to number of member model scores");
|
||||
} else {
|
||||
// Default is all 0 (as opposed to e.g. -99 or similar to approximate log(0)
|
||||
// or a smoothed "not in model" score)
|
||||
@ -132,11 +181,10 @@ TargetPhraseCollection::shared_ptr
|
||||
PhraseDictionaryGroup::
|
||||
CreateTargetPhraseCollection(const ttasksptr& ttask, const Phrase& src) const
|
||||
{
|
||||
// Aggregation of phrases and the scores that will be applied to them
|
||||
vector<TargetPhrase*> allPhrases;
|
||||
// Maps phrase from member model to <phrase copy, scores>
|
||||
typedef unordered_map<const TargetPhrase*, pair<TargetPhrase*, vector<float> >, UnorderedComparer<Phrase>, UnorderedComparer<Phrase> > PhraseMap;
|
||||
PhraseMap allScores;
|
||||
// Aggregation of phrases and corresponding statistics (scores, models seen by)
|
||||
vector<TargetPhrase*> phraseList;
|
||||
typedef unordered_map<const TargetPhrase*, PDGroupPhrase, UnorderedComparer<Phrase>, UnorderedComparer<Phrase> > PhraseMap;
|
||||
PhraseMap phraseMap;
|
||||
|
||||
// For each model
|
||||
size_t offset = 0;
|
||||
@ -154,8 +202,8 @@ CreateTargetPhraseCollection(const ttasksptr& ttask, const Phrase& src) const
|
||||
targetPhrase->GetScoreBreakdown().GetScoresForProducer(&pd);
|
||||
|
||||
// Phrase not in collection -> add if unrestricted or first model
|
||||
PhraseMap::iterator iter = allScores.find(targetPhrase);
|
||||
if (iter == allScores.end()) {
|
||||
PhraseMap::iterator iter = phraseMap.find(targetPhrase);
|
||||
if (iter == phraseMap.end()) {
|
||||
if (m_restrict && i > 0) {
|
||||
continue;
|
||||
}
|
||||
@ -171,31 +219,144 @@ CreateTargetPhraseCollection(const ttasksptr& ttask, const Phrase& src) const
|
||||
// Zero out scores from original phrase table
|
||||
phrase->GetScoreBreakdown().ZeroDenseFeatures(&pd);
|
||||
// Add phrase entry
|
||||
allPhrases.push_back(phrase);
|
||||
allScores[targetPhrase] = make_pair(phrase, vector<float>(m_defaultScores));
|
||||
phraseList.push_back(phrase);
|
||||
phraseMap[targetPhrase] = PDGroupPhrase(phrase, m_defaultScores, m_numModels);
|
||||
} else {
|
||||
// For existing phrases: merge extra scores (such as lr-func scores for mmsapt)
|
||||
TargetPhrase* phrase = iter->second.first;
|
||||
TargetPhrase* phrase = iter->second.m_targetPhrase;
|
||||
BOOST_FOREACH(const TargetPhrase::ScoreCache_t::value_type pair, targetPhrase->GetExtraScores()) {
|
||||
phrase->SetExtraScores(pair.first, pair.second);
|
||||
}
|
||||
}
|
||||
vector<float>& scores = allScores.find(targetPhrase)->second.second;
|
||||
// Don't repeat lookup if phrase already found
|
||||
PDGroupPhrase& pdgPhrase = (iter == phraseMap.end()) ? phraseMap.find(targetPhrase)->second : iter->second;
|
||||
|
||||
// Copy scores from this model
|
||||
for (size_t j = 0; j < pd.GetNumScoreComponents(); ++j) {
|
||||
scores[offset + j] = raw_scores[j];
|
||||
pdgPhrase.m_scores[offset + j] = raw_scores[j];
|
||||
}
|
||||
|
||||
// Phrase seen by this model
|
||||
pdgPhrase.m_seenBy[i] = true;
|
||||
}
|
||||
}
|
||||
offset += pd.GetNumScoreComponents();
|
||||
}
|
||||
|
||||
// Apply scores to phrases and add them to return collection
|
||||
// Compute additional scores as phrases are added to return collection
|
||||
TargetPhraseCollection::shared_ptr ret(new TargetPhraseCollection);
|
||||
const vector<FeatureFunction*> pd_feature_const(m_pdFeature);
|
||||
BOOST_FOREACH(TargetPhrase* phrase, allPhrases) {
|
||||
phrase->GetScoreBreakdown().Assign(this, allScores.find(phrase)->second.second);
|
||||
BOOST_FOREACH(TargetPhrase* phrase, phraseList) {
|
||||
PDGroupPhrase& pdgPhrase = phraseMap.find(phrase)->second;
|
||||
|
||||
// Score order (example with 2 models)
|
||||
// member1_scores member2_scores [m1_pc m2_pc] [m1_wc m2_wc]
|
||||
|
||||
// Extra scores added after member model scores
|
||||
size_t offset = m_totalModelScores;
|
||||
// Phrase count (per member model)
|
||||
if (m_phraseCounts) {
|
||||
for (size_t i = 0; i < m_numModels; ++i) {
|
||||
if (pdgPhrase.m_seenBy[i]) {
|
||||
pdgPhrase.m_scores[offset + i] = 1;
|
||||
}
|
||||
}
|
||||
offset += m_numModels;
|
||||
}
|
||||
// Word count (per member model)
|
||||
if (m_wordCounts) {
|
||||
size_t wc = pdgPhrase.m_targetPhrase->GetSize();
|
||||
for (size_t i = 0; i < m_numModels; ++i) {
|
||||
if (pdgPhrase.m_seenBy[i]) {
|
||||
pdgPhrase.m_scores[offset + i] = wc;
|
||||
}
|
||||
}
|
||||
offset += m_numModels;
|
||||
}
|
||||
|
||||
// Model bitmap features (one feature per possible bitmap)
|
||||
// e.g. seen by models 1 and 3 but not 2 -> "101" fires
|
||||
if (m_modelBitmapCounts) {
|
||||
// Throws exception if someone tries to combine more than 64 models
|
||||
pdgPhrase.m_scores[offset + (pdgPhrase.m_seenBy.to_ulong() - 1)] = 1;
|
||||
offset += m_seenByAll.to_ulong();
|
||||
}
|
||||
|
||||
// Average other-model scores to fill in defaults when models have not seen
|
||||
// this phrase
|
||||
if (m_defaultAverageOthers) {
|
||||
// Average seen scores
|
||||
if (pdgPhrase.m_seenBy != m_seenByAll) {
|
||||
vector<float> avgScores(m_scoresPerModel, 0);
|
||||
size_t seenBy = 0;
|
||||
offset = 0;
|
||||
// sum
|
||||
for (size_t i = 0; i < m_numModels; ++i) {
|
||||
if (pdgPhrase.m_seenBy[i]) {
|
||||
for (size_t j = 0; j < m_scoresPerModel; ++j) {
|
||||
avgScores[j] += pdgPhrase.m_scores[offset + j];
|
||||
}
|
||||
seenBy += 1;
|
||||
}
|
||||
offset += m_scoresPerModel;
|
||||
}
|
||||
// divide
|
||||
for (size_t j = 0; j < m_scoresPerModel; ++j) {
|
||||
avgScores[j] /= seenBy;
|
||||
}
|
||||
// copy
|
||||
offset = 0;
|
||||
for (size_t i = 0; i < m_numModels; ++i) {
|
||||
if (!pdgPhrase.m_seenBy[i]) {
|
||||
for (size_t j = 0; j < m_scoresPerModel; ++j) {
|
||||
pdgPhrase.m_scores[offset + j] = avgScores[j];
|
||||
}
|
||||
}
|
||||
offset += m_scoresPerModel;
|
||||
}
|
||||
#ifdef PT_UG
|
||||
// Also average LexicalReordering scores if specified
|
||||
// We don't necessarily have a lr-func for each model
|
||||
if (m_haveMmsaptLrFunc) {
|
||||
SPTR<Scores> avgLRScores;
|
||||
size_t seenBy = 0;
|
||||
// For each model
|
||||
for (size_t i = 0; i < m_numModels; ++i) {
|
||||
const LexicalReordering* lrFunc = *m_mmsaptLrFuncs[i];
|
||||
// Add if phrase seen and model has lr-func
|
||||
if (pdgPhrase.m_seenBy[i] && lrFunc != NULL) {
|
||||
const Scores* scores = pdgPhrase.m_targetPhrase->GetExtraScores(lrFunc);
|
||||
if (!avgLRScores) {
|
||||
avgLRScores.reset(new Scores(*scores));
|
||||
} else {
|
||||
for (size_t j = 0; j < scores->size(); ++j) {
|
||||
(*avgLRScores)[j] += (*scores)[j];
|
||||
}
|
||||
}
|
||||
seenBy += 1;
|
||||
}
|
||||
}
|
||||
// Make sure we have at least one lr-func
|
||||
if (avgLRScores) {
|
||||
// divide
|
||||
for (size_t j = 0; j < avgLRScores->size(); ++j) {
|
||||
(*avgLRScores)[j] /= seenBy;
|
||||
}
|
||||
// set
|
||||
for (size_t i = 0; i < m_numModels; ++i) {
|
||||
const LexicalReordering* lrFunc = *m_mmsaptLrFuncs[i];
|
||||
if (!pdgPhrase.m_seenBy[i] && lrFunc != NULL) {
|
||||
pdgPhrase.m_targetPhrase->SetExtraScores(lrFunc, avgLRScores);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
// Assign scores
|
||||
phrase->GetScoreBreakdown().Assign(this, pdgPhrase.m_scores);
|
||||
// Correct future cost estimates and total score
|
||||
phrase->EvaluateInIsolation(src, pd_feature_const);
|
||||
ret->Add(phrase);
|
||||
@ -224,14 +385,6 @@ PhraseDictionaryGroup::
|
||||
CleanUpAfterSentenceProcessing(const InputType &source)
|
||||
{
|
||||
GetPhraseCache().clear();
|
||||
// PhraseCache &ref = GetPhraseCache();
|
||||
// for (PhraseCache::iterator it = ref.begin(); it != ref.end(); it++) {
|
||||
// delete *it;
|
||||
// }
|
||||
|
||||
// PhraseCache temp;
|
||||
// temp.swap(ref);
|
||||
|
||||
CleanUpComponentModels(source);
|
||||
}
|
||||
|
||||
|
@ -20,22 +20,43 @@
|
||||
#ifndef moses_PhraseDictionaryGroup_h
|
||||
#define moses_PhraseDictionaryGroup_h
|
||||
|
||||
#include "moses/TranslationModel/PhraseDictionary.h"
|
||||
|
||||
#include <boost/dynamic_bitset.hpp>
|
||||
#include <boost/unordered_map.hpp>
|
||||
#include <boost/thread/shared_mutex.hpp>
|
||||
|
||||
#include "moses/StaticData.h"
|
||||
#include "moses/TargetPhrase.h"
|
||||
#include "moses/Util.h"
|
||||
|
||||
#include "moses/FF/LexicalReordering/LexicalReordering.h"
|
||||
|
||||
#include "moses/TranslationModel/PhraseDictionary.h"
|
||||
|
||||
#ifdef PT_UG
|
||||
#include "moses/TranslationModel/UG/mmsapt.h"
|
||||
#endif
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
struct PDGroupPhrase {
|
||||
TargetPhrase* m_targetPhrase;
|
||||
std::vector<float> m_scores;
|
||||
boost::dynamic_bitset<> m_seenBy;
|
||||
|
||||
PDGroupPhrase() : m_targetPhrase(NULL) { }
|
||||
|
||||
PDGroupPhrase(TargetPhrase* targetPhrase, const std::vector<float>& scores, const size_t nModels)
|
||||
: m_targetPhrase(targetPhrase),
|
||||
m_scores(scores),
|
||||
m_seenBy(nModels) { }
|
||||
};
|
||||
|
||||
/** Combines multiple phrase tables into a single interface. Each member phrase
|
||||
* table scores each phrase and a single set of translations/scores is returned.
|
||||
* If a phrase is not in one of the tables, its scores are zero-filled. Use the
|
||||
* "restrict" option to restrict phrases to those in the table-limit of the
|
||||
* first member table, intended to be a "union" table built on all data.
|
||||
* If a phrase is not in one of the tables, its scores are zero-filled unless
|
||||
* otherwise specified. See model combination section of Moses advanced feature
|
||||
* documentation.
|
||||
*/
|
||||
class PhraseDictionaryGroup: public PhraseDictionary
|
||||
{
|
||||
@ -66,11 +87,28 @@ public:
|
||||
protected:
|
||||
std::vector<std::string> m_memberPDStrs;
|
||||
std::vector<PhraseDictionary*> m_memberPDs;
|
||||
std::vector<FeatureFunction*> m_pdFeature;
|
||||
size_t m_numModels;
|
||||
size_t m_totalModelScores;
|
||||
boost::dynamic_bitset<> m_seenByAll;
|
||||
// phrase-counts option
|
||||
bool m_phraseCounts;
|
||||
// word-counts option
|
||||
bool m_wordCounts;
|
||||
// model-bitmap-counts option
|
||||
bool m_modelBitmapCounts;
|
||||
// restrict option
|
||||
bool m_restrict;
|
||||
// default-scores option
|
||||
bool m_haveDefaultScores;
|
||||
std::vector<float> m_defaultScores;
|
||||
std::vector<FeatureFunction*> m_pdFeature;
|
||||
// default-average-others option
|
||||
bool m_defaultAverageOthers;
|
||||
size_t m_scoresPerModel;
|
||||
// mmsapt-lr-func options
|
||||
bool m_haveMmsaptLrFunc;
|
||||
// pointers to pointers since member mmsapts may not load these until later
|
||||
std::vector<LexicalReordering**> m_mmsaptLrFuncs;
|
||||
|
||||
typedef std::vector<TargetPhraseCollection::shared_ptr > PhraseCache;
|
||||
#ifdef WITH_THREADS
|
||||
|
@ -20,8 +20,15 @@
|
||||
namespace sapt
|
||||
{
|
||||
|
||||
enum sampling_method { full_coverage, random_sampling, ranked_sampling, ranked_sampling2 };
|
||||
|
||||
enum
|
||||
sampling_method
|
||||
{
|
||||
full_coverage,
|
||||
random_sampling,
|
||||
ranked_sampling,
|
||||
ranked_sampling2
|
||||
};
|
||||
|
||||
typedef ttrack::Position TokenPosition;
|
||||
class CandidateSorter
|
||||
{
|
||||
|
@ -89,9 +89,9 @@ namespace sapt
|
||||
p2 = 0;
|
||||
if (ps)
|
||||
{
|
||||
raw1 = ps->raw_cnt;
|
||||
sample1 = ps->sample_cnt;
|
||||
good1 = ps->good;
|
||||
raw1 = ps->raw_cnt;
|
||||
sample1 = ps->sample_cnt;
|
||||
good1 = ps->good;
|
||||
}
|
||||
else raw1 = sample1 = good1 = 0;
|
||||
joint = 0;
|
||||
@ -127,11 +127,11 @@ namespace sapt
|
||||
// should we do that here or leave the raw counts?
|
||||
for (int i = 0; i <= LRModel::NONE; i++)
|
||||
{
|
||||
PhraseOrientation po = static_cast<PhraseOrientation>(i);
|
||||
dfwd[i] = js.dcnt_fwd(po);
|
||||
dbwd[i] = js.dcnt_bwd(po);
|
||||
PhraseOrientation po = static_cast<PhraseOrientation>(i);
|
||||
dfwd[i] = js.dcnt_fwd(po);
|
||||
dbwd[i] = js.dcnt_bwd(po);
|
||||
}
|
||||
|
||||
|
||||
indoc = js.indoc;
|
||||
return *this;
|
||||
}
|
||||
@ -282,8 +282,8 @@ namespace sapt
|
||||
size_t offset;
|
||||
if (dir == LRModel::Bidirectional)
|
||||
{
|
||||
offset = num_scores;
|
||||
num_scores *= 2;
|
||||
offset = num_scores;
|
||||
num_scores *= 2;
|
||||
}
|
||||
else offset = 0;
|
||||
|
||||
|
@ -864,7 +864,7 @@ namespace Moses
|
||||
SPTR<ContextForQuery> context = scope->get<ContextForQuery>(btfix.get(), true);
|
||||
boost::unique_lock<boost::shared_mutex> ctxlock(context->lock);
|
||||
|
||||
if (localcache) std::cerr << "have local cache " << std::endl;
|
||||
// if (localcache) std::cerr << "have local cache " << std::endl;
|
||||
// std::cerr << "BOO at " << HERE << std::endl;
|
||||
if (!localcache)
|
||||
{
|
||||
|
@ -58,6 +58,10 @@ namespace Moses
|
||||
friend class Alignment;
|
||||
std::map<std::string,std::string> param;
|
||||
std::string m_name;
|
||||
#ifndef NO_MOSES
|
||||
// Allows PhraseDictionaryGroup to get &m_lr_func
|
||||
friend class PhraseDictionaryGroup;
|
||||
#endif
|
||||
public:
|
||||
typedef sapt::L2R_Token<sapt::SimpleWordId> Token;
|
||||
typedef sapt::mmBitext<Token> mmbitext;
|
||||
|
@ -168,7 +168,7 @@ void
|
||||
TranslationRequest::
|
||||
output_phrase(ostream& out, Phrase const& phrase) const
|
||||
{
|
||||
if (!m_reportAllFactors) {
|
||||
if (!m_options.output.ReportAllFactors) {
|
||||
for (size_t i = 0 ; i < phrase.GetSize(); ++i)
|
||||
out << *phrase.GetFactor(i, 0) << " ";
|
||||
} else out << phrase;
|
||||
@ -294,8 +294,6 @@ parse_request(std::map<std::string, xmlrpc_c::value> const& params)
|
||||
m_withWordAlignInfo = check(params, "word-align");
|
||||
m_withGraphInfo = check(params, "sg");
|
||||
m_withTopts = check(params, "topt");
|
||||
// m_reportAllFactors = check(params, "report-all-factors");
|
||||
// m_nbestDistinct = check(params, "nbest-distinct");
|
||||
m_withScoreBreakdown = check(params, "add-score-breakdown");
|
||||
si = params.find("lambda");
|
||||
if (si != params.end())
|
||||
|
@ -42,11 +42,7 @@ TranslationRequest : public virtual Moses::TranslationTask
|
||||
bool m_withWordAlignInfo;
|
||||
bool m_withGraphInfo;
|
||||
bool m_withTopts;
|
||||
bool m_reportAllFactors;
|
||||
// bool m_nbestDistinct;
|
||||
bool m_withScoreBreakdown;
|
||||
// size_t m_nbestSize;
|
||||
|
||||
uint64_t m_session_id; // 0 means none, 1 means new
|
||||
|
||||
void
|
||||
|
@ -31,10 +31,22 @@ while(-e "$stem$ref") {
|
||||
&add_to_ref($stem,\@REF) if -e $stem;
|
||||
die("ERROR: could not find reference file $stem") unless scalar @REF;
|
||||
|
||||
# add additional references explicitly specified on the command line
|
||||
shift;
|
||||
foreach my $stem (@ARGV) {
|
||||
&add_to_ref($stem,\@REF) if -e $stem;
|
||||
}
|
||||
|
||||
|
||||
|
||||
sub add_to_ref {
|
||||
my ($file,$REF) = @_;
|
||||
my $s=0;
|
||||
open(REF,$file) or die "Can't read $file";
|
||||
if ($file =~ /.gz$/) {
|
||||
open(REF,"gzip -dc $file|") or die "Can't read $file";
|
||||
} else {
|
||||
open(REF,$file) or die "Can't read $file";
|
||||
}
|
||||
while(<REF>) {
|
||||
chop;
|
||||
push @{$$REF[$s++]}, $_;
|
||||
|
Loading…
Reference in New Issue
Block a user