moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 19:27:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge ../mosesdecoder into perf_moses2

Browse Source
This commit is contained in:
Hieu Hoang 2015-11-17 15:51:29 +00:00
commit 6432fefd3b
24 changed files with 340 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
moses/FF/LexicalReordering/LexicalReordering.cpp
View File

@ -145,12 +145,8 @@ SetCache(TranslationOption& to) const
Phrase const& tphrase = to.GetTargetPhrase();
to.CacheLexReorderingScores(*this, this->GetProb(sphrase,tphrase));
} else { // e.g. OOV with Mmsapt
if (m_haveDefaultScores) {
to.CacheLexReorderingScores(*this, m_defaultScores);
} else {
Scores vals(GetNumScoreComponents(), 0);
to.CacheLexReorderingScores(*this, vals);
}
// Scores vals(GetNumScoreComponents(), 0);
// to.CacheLexReorderingScores(*this, vals);
}
}

3
moses/FF/LexicalReordering/LexicalReorderingState.cpp
View File

@ -243,11 +243,12 @@ CopyScores(ScoreComponentCollection* accum,
size_t off_remote = m_offset + reoType;
size_t off_local = m_configuration.CollapseScores() ? m_offset : off_remote;
UTIL_THROW_IF2(off_remote >= producer->GetNumScoreComponents(),
UTIL_THROW_IF2(off_local >= producer->GetNumScoreComponents(),
"offset out of vector bounds!");
// look up applicable score from vectore of scores
if(cached) {
UTIL_THROW_IF2(off_remote >= cached->size(), "offset out of vector bounds!");
Scores scores(producer->GetNumScoreComponents(),0);
scores[off_local ] = (*cached)[off_remote];
accum->PlusEquals(producer, scores);

19
moses/LM/Ken.cpp
View File

@ -393,9 +393,7 @@ template <class Model> FFState *LanguageModelKen<Model>::EvaluateWhenApplied(con
// Non-terminal is first so we can copy instead of rescoring.
const Syntax::SVertex *pred = hyperedge.tail[nonTermIndexMap[phrasePos]];
const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(pred->states[featureID])->GetChartState();
float prob = UntransformLMScore(
pred->best->label.scoreBreakdown.GetScoresForProducer(this)[0]);
ruleScore.BeginNonTerminal(prevState, prob);
ruleScore.BeginNonTerminal(prevState);
phrasePos++;
}
}
@ -405,9 +403,7 @@ template <class Model> FFState *LanguageModelKen<Model>::EvaluateWhenApplied(con
if (word.IsNonTerminal()) {
const Syntax::SVertex *pred = hyperedge.tail[nonTermIndexMap[phrasePos]];
const lm::ngram::ChartState &prevState = static_cast<const LanguageModelChartStateKenLM*>(pred->states[featureID])->GetChartState();
float prob = UntransformLMScore(
pred->best->label.scoreBreakdown.GetScoresForProducer(this)[0]);
ruleScore.NonTerminal(prevState, prob);
ruleScore.NonTerminal(prevState);
} else {
ruleScore.Terminal(TranslateID(word));
}
@ -415,7 +411,16 @@ template <class Model> FFState *LanguageModelKen<Model>::EvaluateWhenApplied(con
float score = ruleScore.Finish();
score = TransformLMScore(score);
accumulator->Assign(this, score);
score -= target.GetScoreBreakdown().GetScoresForProducer(this)[0];
if (OOVFeatureEnabled()) {
std::vector<float> scores(2);
scores[0] = score;
scores[1] = 0.0;
accumulator->PlusEquals(this, scores);
} else {
accumulator->PlusEquals(this, score);
}
return newState;
}

2
moses/StaticData.cpp
View File

@ -457,6 +457,8 @@ void StaticData::LoadDecodeGraphs()
params = m_parameter->GetParam("mapping");
if (params && params->size()) {
mappingVector = *params;
} else {
mappingVector.assign(1,"0 T 0");
}
params = m_parameter->GetParam("max-chart-span");

27
moses/Syntax/Cube.cpp
View File

@ -101,10 +101,6 @@ SHyperedge *Cube::CreateHyperedge(const std::vector<int> &coordinates)
for (std::size_t i = 0; i < coordinates.size()-1; ++i) {
boost::shared_ptr<SVertex> pred = (*m_bundle.stacks[i])[coordinates[i]];
hyperedge->tail[i] = pred.get();
if (pred->best) {
hyperedge->label.scoreBreakdown.PlusEquals(
pred->best->label.scoreBreakdown);
}
}
hyperedge->label.inputWeight = m_bundle.inputWeight;
@ -112,8 +108,7 @@ SHyperedge *Cube::CreateHyperedge(const std::vector<int> &coordinates)
hyperedge->label.translation =
*(m_bundle.translations->begin()+coordinates.back());
hyperedge->label.scoreBreakdown.PlusEquals(
hyperedge->label.translation->GetScoreBreakdown());
// Calculate feature deltas.
const StaticData &staticData = StaticData::Instance();
@ -123,7 +118,7 @@ SHyperedge *Cube::CreateHyperedge(const std::vector<int> &coordinates)
StatelessFeatureFunction::GetStatelessFeatureFunctions();
for (unsigned i = 0; i < sfs.size(); ++i) {
if (!staticData.IsFeatureFunctionIgnored(*sfs[i])) {
sfs[i]->EvaluateWhenApplied(*hyperedge, &hyperedge->label.scoreBreakdown);
sfs[i]->EvaluateWhenApplied(*hyperedge, &hyperedge->label.deltas);
}
}
@ -132,12 +127,24 @@ SHyperedge *Cube::CreateHyperedge(const std::vector<int> &coordinates)
for (unsigned i = 0; i < ffs.size(); ++i) {
if (!staticData.IsFeatureFunctionIgnored(*ffs[i])) {
head->states[i] =
ffs[i]->EvaluateWhenApplied(*hyperedge, i,
&hyperedge->label.scoreBreakdown);
ffs[i]->EvaluateWhenApplied(*hyperedge, i, &hyperedge->label.deltas);
}
}
hyperedge->label.score = hyperedge->label.scoreBreakdown.GetWeightedScore();
// Calculate future score.
hyperedge->label.futureScore =
hyperedge->label.translation->GetScoreBreakdown().GetWeightedScore();
hyperedge->label.futureScore += hyperedge->label.deltas.GetWeightedScore();
for (std::vector<SVertex*>::const_iterator p = hyperedge->tail.begin();
p != hyperedge->tail.end(); ++p) {
const SVertex *pred = *p;
if (pred->best) {
hyperedge->label.futureScore += pred->best->label.futureScore;
}
}
return hyperedge;
}

2
moses/Syntax/Cube.h
View File

@ -42,7 +42,7 @@ private:
{
public:
bool operator()(const QueueItem &p, const QueueItem &q) const {
return p.first->label.score < q.first->label.score;
return p.first->label.futureScore < q.first->label.futureScore;
}
};

2
moses/Syntax/CubeQueue.h
View File

@ -31,7 +31,7 @@ private:
{
public:
bool operator()(const Cube *p, const Cube *q) const {
return p->Top()->label.score < q->Top()->label.score;
return p->Top()->label.futureScore < q->Top()->label.futureScore;
}
};

2
moses/Syntax/F2S/Manager-inl.h
View File

@ -304,7 +304,7 @@ void Manager<RuleMatcher>::RecombineAndSort(
// Compare the score of h against the score of the best incoming hyperedge
// for the stored vertex.
SVertex *storedVertex = result.first->second;
if (h->label.score > storedVertex->best->label.score) {
if (h->label.futureScore > storedVertex->best->label.futureScore) {
// h's score is better.
storedVertex->recombined.push_back(storedVertex->best);
storedVertex->best = h;

26
moses/Syntax/KBestExtractor.cpp
View File

@ -32,24 +32,25 @@ void KBestExtractor::Extract(
supremeVertex->best = new SHyperedge();
supremeVertex->best->head = supremeVertex.get();
supremeVertex->best->tail.push_back(&bestTopLevelVertex);
supremeVertex->best->label.score = bestTopLevelVertex.best->label.score;
supremeVertex->best->label.scoreBreakdown =
bestTopLevelVertex.best->label.scoreBreakdown;
supremeVertex->best->label.futureScore =
bestTopLevelVertex.best->label.futureScore;
supremeVertex->best->label.deltas = bestTopLevelVertex.best->label.deltas;
supremeVertex->best->label.translation = 0;
// For each alternative top-level SVertex, add a new incoming hyperedge to
// supremeVertex.
for (++p; p != topLevelVertices.end(); ++p) {
// Check that the first item in topLevelVertices really was the best.
UTIL_THROW_IF2((*p)->best->label.score > bestTopLevelVertex.best->label.score,
UTIL_THROW_IF2((*p)->best->label.futureScore >
bestTopLevelVertex.best->label.futureScore,
"top-level SVertices are not correctly sorted");
// Note: there's no need for a smart pointer here: supremeVertex will take
// ownership of altEdge.
SHyperedge *altEdge = new SHyperedge();
altEdge->head = supremeVertex.get();
altEdge->tail.push_back((*p).get());
altEdge->label.score = (*p)->best->label.score;
altEdge->label.scoreBreakdown = (*p)->best->label.scoreBreakdown;
altEdge->label.futureScore = (*p)->best->label.futureScore;
altEdge->label.deltas = (*p)->best->label.deltas;
altEdge->label.translation = 0;
supremeVertex->recombined.push_back(altEdge);
}
@ -282,7 +283,13 @@ void KBestExtractor::LazyNext(KVertex &v, const Derivation &d,
KBestExtractor::Derivation::Derivation(const boost::shared_ptr<KHyperedge> &e)
{
edge = e;
std::size_t arity = edge->tail.size();
const TargetPhrase *translation = edge->shyperedge.label.translation;
// Every hyperedge should have an associated target phrase, except for
// incoming hyperedges of the 'supreme' vertex.
if (translation) {
scoreBreakdown = translation->GetScoreBreakdown();
}
const std::size_t arity = edge->tail.size();
backPointers.resize(arity, 0);
subderivations.reserve(arity);
for (std::size_t i = 0; i < arity; ++i) {
@ -290,9 +297,10 @@ KBestExtractor::Derivation::Derivation(const boost::shared_ptr<KHyperedge> &e)
assert(pred.kBestList.size() >= 1);
boost::shared_ptr<Derivation> sub(pred.kBestList[0]);
subderivations.push_back(sub);
scoreBreakdown.PlusEquals(sub->scoreBreakdown);
}
score = edge->shyperedge.label.score;
scoreBreakdown = edge->shyperedge.label.scoreBreakdown;
scoreBreakdown.PlusEquals(edge->shyperedge.label.deltas);
score = scoreBreakdown.GetWeightedScore();
}
// Construct a Derivation that neighbours an existing Derivation.

2
moses/Syntax/Manager.cpp
View File

@ -32,7 +32,7 @@ void Manager::OutputBest(OutputCollector *collector) const
out << '\n';
} else {
if (options().output.ReportHypoScore) {
out << best->label.score << " ";
out << best->label.futureScore << " ";
}
Phrase yield = GetOneBestTargetYield(*best);
// delete 1st & last

2
moses/Syntax/S2T/Manager-inl.h
View File

@ -368,7 +368,7 @@ void Manager<Parser>::RecombineAndSort(const std::vector<SHyperedge*> &buffer,
// Compare the score of h against the score of the best incoming hyperedge
// for the stored vertex.
SVertex *storedVertex = result.first->second;
if (h->label.score > storedVertex->best->label.score) {
if (h->label.futureScore > storedVertex->best->label.futureScore) {
// h's score is better.
storedVertex->recombined.push_back(storedVertex->best);
storedVertex->best = h;

2
moses/Syntax/SHyperedgeBundleScorer.h
View File

@ -16,7 +16,7 @@ public:
bundle.stacks.begin(); p != bundle.stacks.end(); ++p) {
const SVertexStack *stack = *p;
if (stack->front()->best) {
score += stack->front()->best->label.score;
score += stack->front()->best->label.futureScore;
}
}
return score;

19
moses/Syntax/SLabel.h
View File

@ -8,11 +8,24 @@ namespace Moses
namespace Syntax
{
// A SHyperedge label.
//
struct SLabel {
float inputWeight;
float score;
ScoreComponentCollection scoreBreakdown;
// Deltas for individual feature scores. i.e. this object records the change
// in each feature score that results from applying the rule associated with
// this hyperedge.
ScoreComponentCollection deltas;
// Total derivation score to be used for comparison in beam search (i.e.
// including future cost estimates). This is the sum of the 1-best
// subderivations' future scores + deltas.
float futureScore;
// Target-side of the grammar rule.
const TargetPhrase *translation;
// Input weight of this hyperedge (e.g. from weighted input forest).
float inputWeight;
};
} // Syntax

2
moses/Syntax/SVertexStack.h
View File

@ -18,7 +18,7 @@ struct SVertexStackContentOrderer {
public:
bool operator()(const boost::shared_ptr<SVertex> &x,
const boost::shared_ptr<SVertex> &y) {
return x->best->label.score > y->best->label.score;
return x->best->label.futureScore > y->best->label.futureScore;
}
};

2
moses/Syntax/T2S/Manager-inl.h
View File

@ -264,7 +264,7 @@ void Manager<RuleMatcher>::RecombineAndSort(
// Compare the score of h against the score of the best incoming hyperedge
// for the stored vertex.
SVertex *storedVertex = result.first->second;
if (h->label.score > storedVertex->best->label.score) {
if (h->label.futureScore > storedVertex->best->label.futureScore) {
// h's score is better.
storedVertex->recombined.push_back(storedVertex->best);
storedVertex->best = h;

213
moses/TranslationModel/PhraseDictionaryGroup.cpp
View File

@ -33,8 +33,15 @@ namespace Moses
PhraseDictionaryGroup::PhraseDictionaryGroup(const string &line)
: PhraseDictionary(line, true),
m_numModels(0),
m_totalModelScores(0),
m_phraseCounts(false),
m_wordCounts(false),
m_modelBitmapCounts(false),
m_restrict(false),
m_haveDefaultScores(false)
m_haveDefaultScores(false),
m_defaultAverageOthers(false),
m_scoresPerModel(0),
m_haveMmsaptLrFunc(false)
{
ReadParameters();
}
@ -44,11 +51,23 @@ void PhraseDictionaryGroup::SetParameter(const string& key, const string& value)
if (key == "members") {
m_memberPDStrs = Tokenize(value, ",");
m_numModels = m_memberPDStrs.size();
m_seenByAll = dynamic_bitset<>(m_numModels);
m_seenByAll.set();
} else if (key == "restrict") {
m_restrict = Scan<bool>(value);
} else if (key == "phrase-counts") {
m_phraseCounts = Scan<bool>(value);
} else if (key == "word-counts") {
m_wordCounts = Scan<bool>(value);
} else if (key == "model-bitmap-counts") {
m_modelBitmapCounts = Scan<bool>(value);
} else if (key =="default-scores") {
m_haveDefaultScores = true;
m_defaultScores = Scan<float>(Tokenize(value, ","));
} else if (key =="default-average-others") {
m_defaultAverageOthers = Scan<bool>(value);
} else if (key =="mmsapt-lr-func") {
m_haveMmsaptLrFunc = true;
} else {
PhraseDictionary::SetParameter(key, value);
}
@ -58,28 +77,58 @@ void PhraseDictionaryGroup::Load()
{
SetFeaturesToApply();
m_pdFeature.push_back(const_cast<PhraseDictionaryGroup*>(this));
size_t numScoreComponents = 0;
// Locate/check component phrase tables
size_t componentWeights = 0;
BOOST_FOREACH(const string& pdName, m_memberPDStrs) {
bool pdFound = false;
BOOST_FOREACH(PhraseDictionary* pd, PhraseDictionary::GetColl()) {
if (pd->GetScoreProducerDescription() == pdName) {
pdFound = true;
m_memberPDs.push_back(pd);
componentWeights += pd->GetNumScoreComponents();
size_t nScores = pd->GetNumScoreComponents();
numScoreComponents += nScores;
if (m_scoresPerModel == 0) {
m_scoresPerModel = nScores;
} else if (m_defaultAverageOthers) {
UTIL_THROW_IF2(nScores != m_scoresPerModel,
m_description << ": member models must have the same number of scores when using default-average-others");
}
}
}
UTIL_THROW_IF2(!pdFound,
"Could not find member phrase table " << pdName);
m_description << ": could not find member phrase table " << pdName);
}
UTIL_THROW_IF2(componentWeights != m_numScoreComponents,
"Total number of member model scores is unequal to specified number of scores");
m_totalModelScores = numScoreComponents;
// Check feature total
if (m_phraseCounts) {
numScoreComponents += m_numModels;
}
if (m_wordCounts) {
numScoreComponents += m_numModels;
}
if (m_modelBitmapCounts) {
numScoreComponents += (pow(2, m_numModels) - 1);
}
UTIL_THROW_IF2(numScoreComponents != m_numScoreComponents,
m_description << ": feature count mismatch: specify \"num-features=" << numScoreComponents << "\" and supply " << numScoreComponents << " weights");
#ifdef PT_UG
// Locate mmsapt lexical reordering functions if specified
if (m_haveMmsaptLrFunc) {
BOOST_FOREACH(PhraseDictionary* pd, m_memberPDs) {
// pointer to pointer, all start as NULL and some may be populated prior
// to translation
m_mmsaptLrFuncs.push_back(&(static_cast<Mmsapt*>(pd)->m_lr_func));
}
}
#endif
// Determine "zero" scores for features
if (m_haveDefaultScores) {
UTIL_THROW_IF2(m_defaultScores.size() != m_numScoreComponents,
"Number of specified default scores is unequal to number of member model scores");
m_description << ": number of specified default scores is unequal to number of member model scores");
} else {
// Default is all 0 (as opposed to e.g. -99 or similar to approximate log(0)
// or a smoothed "not in model" score)
@ -132,11 +181,10 @@ TargetPhraseCollection::shared_ptr
PhraseDictionaryGroup::
CreateTargetPhraseCollection(const ttasksptr& ttask, const Phrase& src) const
{
// Aggregation of phrases and the scores that will be applied to them
vector<TargetPhrase*> allPhrases;
// Maps phrase from member model to <phrase copy, scores>
typedef unordered_map<const TargetPhrase*, pair<TargetPhrase*, vector<float> >, UnorderedComparer<Phrase>, UnorderedComparer<Phrase> > PhraseMap;
PhraseMap allScores;
// Aggregation of phrases and corresponding statistics (scores, models seen by)
vector<TargetPhrase*> phraseList;
typedef unordered_map<const TargetPhrase*, PDGroupPhrase, UnorderedComparer<Phrase>, UnorderedComparer<Phrase> > PhraseMap;
PhraseMap phraseMap;
// For each model
size_t offset = 0;
@ -154,8 +202,8 @@ CreateTargetPhraseCollection(const ttasksptr& ttask, const Phrase& src) const
targetPhrase->GetScoreBreakdown().GetScoresForProducer(&pd);
// Phrase not in collection -> add if unrestricted or first model
PhraseMap::iterator iter = allScores.find(targetPhrase);
if (iter == allScores.end()) {
PhraseMap::iterator iter = phraseMap.find(targetPhrase);
if (iter == phraseMap.end()) {
if (m_restrict && i > 0) {
continue;
}
@ -171,31 +219,144 @@ CreateTargetPhraseCollection(const ttasksptr& ttask, const Phrase& src) const
// Zero out scores from original phrase table
phrase->GetScoreBreakdown().ZeroDenseFeatures(&pd);
// Add phrase entry
allPhrases.push_back(phrase);
allScores[targetPhrase] = make_pair(phrase, vector<float>(m_defaultScores));
phraseList.push_back(phrase);
phraseMap[targetPhrase] = PDGroupPhrase(phrase, m_defaultScores, m_numModels);
} else {
// For existing phrases: merge extra scores (such as lr-func scores for mmsapt)
TargetPhrase* phrase = iter->second.first;
TargetPhrase* phrase = iter->second.m_targetPhrase;
BOOST_FOREACH(const TargetPhrase::ScoreCache_t::value_type pair, targetPhrase->GetExtraScores()) {
phrase->SetExtraScores(pair.first, pair.second);
}
}
vector<float>& scores = allScores.find(targetPhrase)->second.second;
// Don't repeat lookup if phrase already found
PDGroupPhrase& pdgPhrase = (iter == phraseMap.end()) ? phraseMap.find(targetPhrase)->second : iter->second;
// Copy scores from this model
for (size_t j = 0; j < pd.GetNumScoreComponents(); ++j) {
scores[offset + j] = raw_scores[j];
pdgPhrase.m_scores[offset + j] = raw_scores[j];
}
// Phrase seen by this model
pdgPhrase.m_seenBy[i] = true;
}
}
offset += pd.GetNumScoreComponents();
}
// Apply scores to phrases and add them to return collection
// Compute additional scores as phrases are added to return collection
TargetPhraseCollection::shared_ptr ret(new TargetPhraseCollection);
const vector<FeatureFunction*> pd_feature_const(m_pdFeature);
BOOST_FOREACH(TargetPhrase* phrase, allPhrases) {
phrase->GetScoreBreakdown().Assign(this, allScores.find(phrase)->second.second);
BOOST_FOREACH(TargetPhrase* phrase, phraseList) {
PDGroupPhrase& pdgPhrase = phraseMap.find(phrase)->second;
// Score order (example with 2 models)
// member1_scores member2_scores [m1_pc m2_pc] [m1_wc m2_wc]
// Extra scores added after member model scores
size_t offset = m_totalModelScores;
// Phrase count (per member model)
if (m_phraseCounts) {
for (size_t i = 0; i < m_numModels; ++i) {
if (pdgPhrase.m_seenBy[i]) {
pdgPhrase.m_scores[offset + i] = 1;
}
}
offset += m_numModels;
}
// Word count (per member model)
if (m_wordCounts) {
size_t wc = pdgPhrase.m_targetPhrase->GetSize();
for (size_t i = 0; i < m_numModels; ++i) {
if (pdgPhrase.m_seenBy[i]) {
pdgPhrase.m_scores[offset + i] = wc;
}
}
offset += m_numModels;
}
// Model bitmap features (one feature per possible bitmap)
// e.g. seen by models 1 and 3 but not 2 -> "101" fires
if (m_modelBitmapCounts) {
// Throws exception if someone tries to combine more than 64 models
pdgPhrase.m_scores[offset + (pdgPhrase.m_seenBy.to_ulong() - 1)] = 1;
offset += m_seenByAll.to_ulong();
}
// Average other-model scores to fill in defaults when models have not seen
// this phrase
if (m_defaultAverageOthers) {
// Average seen scores
if (pdgPhrase.m_seenBy != m_seenByAll) {
vector<float> avgScores(m_scoresPerModel, 0);
size_t seenBy = 0;
offset = 0;
// sum
for (size_t i = 0; i < m_numModels; ++i) {
if (pdgPhrase.m_seenBy[i]) {
for (size_t j = 0; j < m_scoresPerModel; ++j) {
avgScores[j] += pdgPhrase.m_scores[offset + j];
}
seenBy += 1;
}
offset += m_scoresPerModel;
}
// divide
for (size_t j = 0; j < m_scoresPerModel; ++j) {
avgScores[j] /= seenBy;
}
// copy
offset = 0;
for (size_t i = 0; i < m_numModels; ++i) {
if (!pdgPhrase.m_seenBy[i]) {
for (size_t j = 0; j < m_scoresPerModel; ++j) {
pdgPhrase.m_scores[offset + j] = avgScores[j];
}
}
offset += m_scoresPerModel;
}
#ifdef PT_UG
// Also average LexicalReordering scores if specified
// We don't necessarily have a lr-func for each model
if (m_haveMmsaptLrFunc) {
SPTR<Scores> avgLRScores;
size_t seenBy = 0;
// For each model
for (size_t i = 0; i < m_numModels; ++i) {
const LexicalReordering* lrFunc = *m_mmsaptLrFuncs[i];
// Add if phrase seen and model has lr-func
if (pdgPhrase.m_seenBy[i] && lrFunc != NULL) {
const Scores* scores = pdgPhrase.m_targetPhrase->GetExtraScores(lrFunc);
if (!avgLRScores) {
avgLRScores.reset(new Scores(*scores));
} else {
for (size_t j = 0; j < scores->size(); ++j) {
(*avgLRScores)[j] += (*scores)[j];
}
}
seenBy += 1;
}
}
// Make sure we have at least one lr-func
if (avgLRScores) {
// divide
for (size_t j = 0; j < avgLRScores->size(); ++j) {
(*avgLRScores)[j] /= seenBy;
}
// set
for (size_t i = 0; i < m_numModels; ++i) {
const LexicalReordering* lrFunc = *m_mmsaptLrFuncs[i];
if (!pdgPhrase.m_seenBy[i] && lrFunc != NULL) {
pdgPhrase.m_targetPhrase->SetExtraScores(lrFunc, avgLRScores);
}
}
}
}
#endif
}
}
// Assign scores
phrase->GetScoreBreakdown().Assign(this, pdgPhrase.m_scores);
// Correct future cost estimates and total score
phrase->EvaluateInIsolation(src, pd_feature_const);
ret->Add(phrase);
@ -224,14 +385,6 @@ PhraseDictionaryGroup::
CleanUpAfterSentenceProcessing(const InputType &source)
{
GetPhraseCache().clear();
// PhraseCache &ref = GetPhraseCache();
// for (PhraseCache::iterator it = ref.begin(); it != ref.end(); it++) {
// delete *it;
// }
// PhraseCache temp;
// temp.swap(ref);
CleanUpComponentModels(source);
}

50
moses/TranslationModel/PhraseDictionaryGroup.h
View File

@ -20,22 +20,43 @@
#ifndef moses_PhraseDictionaryGroup_h
#define moses_PhraseDictionaryGroup_h
#include "moses/TranslationModel/PhraseDictionary.h"
#include <boost/dynamic_bitset.hpp>
#include <boost/unordered_map.hpp>
#include <boost/thread/shared_mutex.hpp>
#include "moses/StaticData.h"
#include "moses/TargetPhrase.h"
#include "moses/Util.h"
#include "moses/FF/LexicalReordering/LexicalReordering.h"
#include "moses/TranslationModel/PhraseDictionary.h"
#ifdef PT_UG
#include "moses/TranslationModel/UG/mmsapt.h"
#endif
namespace Moses
{
struct PDGroupPhrase {
TargetPhrase* m_targetPhrase;
std::vector<float> m_scores;
boost::dynamic_bitset<> m_seenBy;
PDGroupPhrase() : m_targetPhrase(NULL) { }
PDGroupPhrase(TargetPhrase* targetPhrase, const std::vector<float>& scores, const size_t nModels)
: m_targetPhrase(targetPhrase),
m_scores(scores),
m_seenBy(nModels) { }
};
/** Combines multiple phrase tables into a single interface. Each member phrase
* table scores each phrase and a single set of translations/scores is returned.
* If a phrase is not in one of the tables, its scores are zero-filled. Use the
* "restrict" option to restrict phrases to those in the table-limit of the
* first member table, intended to be a "union" table built on all data.
* If a phrase is not in one of the tables, its scores are zero-filled unless
* otherwise specified. See model combination section of Moses advanced feature
* documentation.
*/
class PhraseDictionaryGroup: public PhraseDictionary
{
@ -66,11 +87,28 @@ public:
protected:
std::vector<std::string> m_memberPDStrs;
std::vector<PhraseDictionary*> m_memberPDs;
std::vector<FeatureFunction*> m_pdFeature;
size_t m_numModels;
size_t m_totalModelScores;
boost::dynamic_bitset<> m_seenByAll;
// phrase-counts option
bool m_phraseCounts;
// word-counts option
bool m_wordCounts;
// model-bitmap-counts option
bool m_modelBitmapCounts;
// restrict option
bool m_restrict;
// default-scores option
bool m_haveDefaultScores;
std::vector<float> m_defaultScores;
std::vector<FeatureFunction*> m_pdFeature;
// default-average-others option
bool m_defaultAverageOthers;
size_t m_scoresPerModel;
// mmsapt-lr-func options
bool m_haveMmsaptLrFunc;
// pointers to pointers since member mmsapts may not load these until later
std::vector<LexicalReordering**> m_mmsaptLrFuncs;
typedef std::vector<TargetPhraseCollection::shared_ptr > PhraseCache;
#ifdef WITH_THREADS

11
moses/TranslationModel/UG/mm/ug_bitext_sampler.h
View File

@ -20,8 +20,15 @@
namespace sapt
{
enum sampling_method { full_coverage, random_sampling, ranked_sampling, ranked_sampling2 };
enum
sampling_method
{
full_coverage,
random_sampling,
ranked_sampling,
ranked_sampling2
};
typedef ttrack::Position TokenPosition;
class CandidateSorter
{

18
moses/TranslationModel/UG/mm/ug_phrasepair.h
View File

@ -89,9 +89,9 @@ namespace sapt
p2 = 0;
if (ps)
{
raw1 = ps->raw_cnt;
sample1 = ps->sample_cnt;
good1 = ps->good;
raw1 = ps->raw_cnt;
sample1 = ps->sample_cnt;
good1 = ps->good;
}
else raw1 = sample1 = good1 = 0;
joint = 0;
@ -127,11 +127,11 @@ namespace sapt
// should we do that here or leave the raw counts?
for (int i = 0; i <= LRModel::NONE; i++)
{
PhraseOrientation po = static_cast<PhraseOrientation>(i);
dfwd[i] = js.dcnt_fwd(po);
dbwd[i] = js.dcnt_bwd(po);
PhraseOrientation po = static_cast<PhraseOrientation>(i);
dfwd[i] = js.dcnt_fwd(po);
dbwd[i] = js.dcnt_bwd(po);
}
indoc = js.indoc;
return *this;
}
@ -282,8 +282,8 @@ namespace sapt
size_t offset;
if (dir == LRModel::Bidirectional)
{
offset = num_scores;
num_scores *= 2;
offset = num_scores;
num_scores *= 2;
}
else offset = 0;

2
moses/TranslationModel/UG/mmsapt.cpp
View File

@ -864,7 +864,7 @@ namespace Moses
SPTR<ContextForQuery> context = scope->get<ContextForQuery>(btfix.get(), true);
boost::unique_lock<boost::shared_mutex> ctxlock(context->lock);
if (localcache) std::cerr << "have local cache " << std::endl;
// if (localcache) std::cerr << "have local cache " << std::endl;
// std::cerr << "BOO at " << HERE << std::endl;
if (!localcache)
{

4
moses/TranslationModel/UG/mmsapt.h
View File

@ -58,6 +58,10 @@ namespace Moses
friend class Alignment;
std::map<std::string,std::string> param;
std::string m_name;
#ifndef NO_MOSES
// Allows PhraseDictionaryGroup to get &m_lr_func
friend class PhraseDictionaryGroup;
#endif
public:
typedef sapt::L2R_Token<sapt::SimpleWordId> Token;
typedef sapt::mmBitext<Token> mmbitext;

4
moses/server/TranslationRequest.cpp
View File

@ -168,7 +168,7 @@ void
TranslationRequest::
output_phrase(ostream& out, Phrase const& phrase) const
{
if (!m_reportAllFactors) {
if (!m_options.output.ReportAllFactors) {
for (size_t i = 0 ; i < phrase.GetSize(); ++i)
out << *phrase.GetFactor(i, 0) << " ";
} else out << phrase;
@ -294,8 +294,6 @@ parse_request(std::map<std::string, xmlrpc_c::value> const& params)
m_withWordAlignInfo = check(params, "word-align");
m_withGraphInfo = check(params, "sg");
m_withTopts = check(params, "topt");
// m_reportAllFactors = check(params, "report-all-factors");
// m_nbestDistinct = check(params, "nbest-distinct");
m_withScoreBreakdown = check(params, "add-score-breakdown");
si = params.find("lambda");
if (si != params.end())

4
moses/server/TranslationRequest.h
View File

@ -42,11 +42,7 @@ TranslationRequest : public virtual Moses::TranslationTask
bool m_withWordAlignInfo;
bool m_withGraphInfo;
bool m_withTopts;
bool m_reportAllFactors;
// bool m_nbestDistinct;
bool m_withScoreBreakdown;
// size_t m_nbestSize;
uint64_t m_session_id; // 0 means none, 1 means new
void

14
scripts/generic/multi-bleu.perl
View File

@ -31,10 +31,22 @@ while(-e "$stem$ref") {
&add_to_ref($stem,\@REF) if -e $stem;
die("ERROR: could not find reference file $stem") unless scalar @REF;
# add additional references explicitly specified on the command line
shift;
foreach my $stem (@ARGV) {
&add_to_ref($stem,\@REF) if -e $stem;
}
sub add_to_ref {
my ($file,$REF) = @_;
my $s=0;
open(REF,$file) or die "Can't read $file";
if ($file =~ /.gz$/) {
open(REF,"gzip -dc $file|") or die "Can't read $file";
} else {
open(REF,$file) or die "Can't read $file";
}
while(<REF>) {
chop;
push @{$$REF[$s++]}, $_;