moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-10-27 03:49:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

generalized n-best list reporting for feature functions, added experimental version of global lexical model

Browse Source 
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@2343 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
phkoehn 2009-05-26 19:30:35 +00:00
parent 17c3cfffac
commit 8833098925
22 changed files with 265 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
moses-cmd/src/IOWrapper.cpp
View File

@ -320,9 +320,10 @@ void IOWrapper::OutputBestHypo(const Hypothesis *hypo, long /*translationId*/, b
void IOWrapper::OutputNBestList(const TrellisPathList &nBestList, long translationId)
{
bool labeledOutput = StaticData::Instance().IsLabeledNBestList();
bool includeAlignment = StaticData::Instance().NBestIncludesAlignment();
bool includeWordAlignment = StaticData::Instance().PrintAlignmentInfoInNbest();
const StaticData &staticData = StaticData::Instance();
bool labeledOutput = staticData.IsLabeledNBestList();
bool includeAlignment = staticData.NBestIncludesAlignment();
bool includeWordAlignment = staticData.PrintAlignmentInfoInNbest();
TrellisPathList::const_iterator iter;
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter)
@ -337,55 +338,88 @@ void IOWrapper::OutputNBestList(const TrellisPathList &nBestList, long translati
const Hypothesis &edge = *edges[currEdge];
OutputSurface(*m_nBestStream, edge.GetCurrTargetPhrase(), m_outputFactorOrder, false); // false for not reporting all factors
}
*m_nBestStream << " ||| ";
*m_nBestStream << " |||";
std::string lastName = "";
const vector<const StatefulFeatureFunction*>& sff =
staticData.GetScoreIndexManager().GetStatefulFeatureFunctions();
for( size_t i=0; i<sff.size(); i++ )
{
if( labeledOutput && lastName != sff[i]->GetScoreProducerWeightShortName() )
{
lastName = sff[i]->GetScoreProducerWeightShortName();
*m_nBestStream << " " << lastName << ":";
}
vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( sff[i] );
for (size_t j = 0; j<scores.size(); ++j)
{
*m_nBestStream << " " << scores[j];
}
}
const vector<const StatelessFeatureFunction*>& slf =
staticData.GetScoreIndexManager().GetStatelessFeatureFunctions();
for( size_t i=0; i<slf.size(); i++ )
{
if( labeledOutput && lastName != slf[i]->GetScoreProducerWeightShortName() )
{
lastName = slf[i]->GetScoreProducerWeightShortName();
*m_nBestStream << " " << lastName << ":";
}
vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( slf[i] );
for (size_t j = 0; j<scores.size(); ++j)
{
*m_nBestStream << " " << scores[j];
}
}
// print the scores in a hardwired order
// before each model type, the corresponding command-line-like name must be emitted
// MERT script relies on this
// basic distortion
if (labeledOutput)
*m_nBestStream << "d: ";
*m_nBestStream << path.GetScoreBreakdown().GetScoreForProducer(StaticData::Instance().GetDistortionScoreProducer()) << " ";
// if (labeledOutput)
// *m_nBestStream << "d: ";
// *m_nBestStream << path.GetScoreBreakdown().GetScoreForProducer(StaticData::Instance().GetDistortionScoreProducer()) << " ";
// reordering
vector<LexicalReordering*> rms = StaticData::Instance().GetReorderModels();
if(rms.size() > 0)
{
vector<LexicalReordering*>::iterator iter;
for(iter = rms.begin(); iter != rms.end(); ++iter)
{
vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer(*iter);
for (size_t j = 0; j<scores.size(); ++j)
{
*m_nBestStream << scores[j] << " ";
}
}
}
// lm
const LMList& lml = StaticData::Instance().GetAllLM();
if (lml.size() > 0) {
if (labeledOutput)
*m_nBestStream << "lm: ";
LMList::const_iterator lmi = lml.begin();
for (; lmi != lml.end(); ++lmi) {
*m_nBestStream << path.GetScoreBreakdown().GetScoreForProducer(*lmi) << " ";
}
}
// vector<LexicalReordering*> rms = StaticData::Instance().GetReorderModels();
// if(rms.size() > 0)
// {
// vector<LexicalReordering*>::iterator iter;
// for(iter = rms.begin(); iter != rms.end(); ++iter)
// {
// vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer(*iter);
// for (size_t j = 0; j<scores.size(); ++j)
// {
// *m_nBestStream << scores[j] << " ";
// }
// }
// }
//
// // lm
// const LMList& lml = StaticData::Instance().GetAllLM();
// if (lml.size() > 0) {
// if (labeledOutput)
// *m_nBestStream << "lm: ";
// LMList::const_iterator lmi = lml.begin();
// for (; lmi != lml.end(); ++lmi) {
// *m_nBestStream << path.GetScoreBreakdown().GetScoreForProducer(*lmi) << " ";
// }
// }
//
// translation components
if (StaticData::Instance().GetInputType()==SentenceInput){
// translation components for text input
vector<PhraseDictionary*> pds = StaticData::Instance().GetPhraseDictionaries();
if (pds.size() > 0) {
if (labeledOutput)
*m_nBestStream << "tm: ";
*m_nBestStream << " tm:";
vector<PhraseDictionary*>::iterator iter;
for (iter = pds.begin(); iter != pds.end(); ++iter) {
vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer(*iter);
for (size_t j = 0; j<scores.size(); ++j)
*m_nBestStream << scores[j] << " ";
*m_nBestStream << " " << scores[j];
}
}
}
@ -405,10 +439,10 @@ void IOWrapper::OutputNBestList(const TrellisPathList &nBestList, long translati
if (pd_numinputscore){
if (labeledOutput)
*m_nBestStream << "I: ";
*m_nBestStream << " I:";
for (size_t j = 0; j < pd_numinputscore; ++j)
*m_nBestStream << scores[j] << " ";
*m_nBestStream << " " << scores[j];
}
@ -418,39 +452,32 @@ void IOWrapper::OutputNBestList(const TrellisPathList &nBestList, long translati
size_t pd_numinputscore = (*iter)->GetNumInputScores();
if (iter == pds.begin() && labeledOutput)
*m_nBestStream << "tm: ";
*m_nBestStream << " tm:";
for (size_t j = pd_numinputscore; j < scores.size() ; ++j)
*m_nBestStream << scores[j] << " ";
*m_nBestStream << " " << scores[j];
}
}
}
// word penalty
if (labeledOutput)
*m_nBestStream << "w: ";
*m_nBestStream << path.GetScoreBreakdown().GetScoreForProducer(StaticData::Instance().GetWordPenaltyProducer()) << " ";
// generation
vector<GenerationDictionary*> gds = StaticData::Instance().GetGenerationDictionaries();
if (gds.size() > 0) {
if (gds.size() > 0) {
if (labeledOutput)
*m_nBestStream << "g: ";
vector<GenerationDictionary*>::iterator iter;
for (iter = gds.begin(); iter != gds.end(); ++iter) {
vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer(*iter);
for (size_t j = 0; j<scores.size(); j++) {
*m_nBestStream << scores[j] << " ";
}
}
}
*m_nBestStream << " g: ";
vector<GenerationDictionary*>::iterator iter;
for (iter = gds.begin(); iter != gds.end(); ++iter) {
vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer(*iter);
for (size_t j = 0; j<scores.size(); j++) {
*m_nBestStream << scores[j] << " ";
}
}
}
// total
*m_nBestStream << "||| " << path.GetTotalScore();
*m_nBestStream << " ||| " << path.GetTotalScore();
//phrase-to-phrase alignment
if (includeAlignment) {
if (includeAlignment) {
*m_nBestStream << " |||";
for (int currEdge = (int)edges.size() - 2 ; currEdge >= 0 ; currEdge--)
{

15
moses/src/DummyScoreProducers.cpp
View File

@ -46,6 +46,11 @@ std::string DistortionScoreProducer::GetScoreProducerDescription() const
return "Distortion";
}
std::string DistortionScoreProducer::GetScoreProducerWeightShortName() const
{
return "d";
}
float DistortionScoreProducer::CalculateDistortionScore(const WordsRange &prev, const WordsRange &curr, const int FirstGap) const
{
const int USE_OLD = 1;
@ -106,6 +111,11 @@ std::string WordPenaltyProducer::GetScoreProducerDescription() const
return "WordPenalty";
}
std::string WordPenaltyProducer::GetScoreProducerWeightShortName() const
{
return "w";
}
size_t WordPenaltyProducer::GetNumInputScores() const { return 0;}
void WordPenaltyProducer::Evaluate(const TargetPhrase& tp, ScoreComponentCollection* out) const
@ -128,6 +138,11 @@ std::string UnknownWordPenaltyProducer::GetScoreProducerDescription() const
return "!UnknownWordPenalty";
}
std::string UnknownWordPenaltyProducer::GetScoreProducerWeightShortName() const
{
return "u";
}
size_t UnknownWordPenaltyProducer::GetNumInputScores() const { return 0;}
bool UnknownWordPenaltyProducer::ComputeValueInTranslationOption() const {

3
moses/src/DummyScoreProducers.h
View File

@ -20,6 +20,7 @@ public:
size_t GetNumScoreComponents() const;
std::string GetScoreProducerDescription() const;
std::string GetScoreProducerWeightShortName() const;
size_t GetNumInputScores() const;
virtual const FFState* EmptyHypothesisState() const;
@ -40,6 +41,7 @@ public:
size_t GetNumScoreComponents() const;
std::string GetScoreProducerDescription() const;
std::string GetScoreProducerWeightShortName() const;
size_t GetNumInputScores() const;
virtual void Evaluate(
@ -55,6 +57,7 @@ public:
size_t GetNumScoreComponents() const;
std::string GetScoreProducerDescription() const;
std::string GetScoreProducerWeightShortName() const;
size_t GetNumInputScores() const;
virtual bool ComputeValueInTranslationOption() const;

4
moses/src/GenerationDictionary.h
View File

@ -71,6 +71,10 @@ public:
size_t GetNumScoreComponents() const;
std::string GetScoreProducerDescription() const;
std::string GetScoreProducerWeightShortName() const
{
return "g";
}
/** number of unique input entries in the generation table.
* NOT the number of lines in the generation table

5
moses/src/LanguageModel.h
View File

@ -122,6 +122,11 @@ public:
virtual std::string GetScoreProducerDescription() const = 0;
std::string GetScoreProducerWeightShortName() const
{
return "lm";
}
//! overrideable funtions for IRST LM to cleanup. Maybe something to do with on demand/cache loading/unloading
virtual void InitializeBeforeSentenceProcessing(){};
virtual void CleanUpAfterSentenceProcessing() {};

5
moses/src/LexicalReordering.h
View File

@ -54,6 +54,11 @@ class LexicalReordering : public StatefulFeatureFunction {
virtual std::string GetScoreProducerDescription() const {
return "Generic Lexical Reordering Model... overwrite in subclass.";
};
std::string GetScoreProducerWeightShortName() const {
return "d";
};
//new
virtual int GetNumOrientationTypes() const = 0;
virtual OrientationType GetOrientationType(Hypothesis*) const = 0;

1
moses/src/Makefile.am
View File

@ -19,6 +19,7 @@ libmoses_a_SOURCES = \
FFState.cpp \
FloydWarshall.cpp \
GenerationDictionary.cpp \
GlobalLexicalModel.cpp \
hash.cpp \
Hypothesis.cpp \
HypothesisStack.cpp \

2
moses/src/Manager.cpp
View File

@ -62,7 +62,7 @@ Manager::Manager(InputType const& source, SearchAlgorithm searchAlgorithm)
Manager::~Manager()
{
delete m_transOptColl;
delete m_transOptColl;
delete m_search;
StaticData::Instance().CleanUpAfterSentenceProcessing();

2
moses/src/Parameter.cpp
View File

@ -44,6 +44,7 @@ Parameter::Parameter()
AddParam("drop-unknown", "du", "drop unknown words instead of copying them");
AddParam("factor-delimiter", "fd", "specify a different factor delimiter than the default");
AddParam("generation-file", "location and properties of the generation table");
AddParam("global-lexical-file", "gl", "discriminatively trained global lexical translation model file");
AddParam("input-factors", "list of factors in the input");
AddParam("input-file", "i", "location of the input file to be translated");
AddParam("inputtype", "text (0), confusion network (1), word lattice (2) (default = 0)");
@ -74,6 +75,7 @@ Parameter::Parameter()
AddParam("weight-generation", "g", "weight(s) for generation components");
AddParam("weight-i", "I", "weight(s) for word insertion - used for parameters from confusion network and lattice input links");
AddParam("weight-l", "lm", "weight(s) for language models");
AddParam("weight-lex", "lex", "weight for global lexical model");
AddParam("weight-t", "tm", "weights for translation model components");
AddParam("weight-w", "w", "weight for word penalty");
AddParam("weight-u", "u", "weight for unknown word penalty");

4
moses/src/PhraseDictionary.h
View File

@ -58,6 +58,10 @@ class PhraseDictionary : public Dictionary, public StatelessFeatureFunction
//! Overriden by load on demand phrase tables classes to load data for each input
virtual void InitializeForInput(InputType const &/*source*/) {}
std::string GetScoreProducerDescription() const;
std::string GetScoreProducerWeightShortName() const
{
return "tm";
}
size_t GetNumScoreComponents() const;
size_t GetNumInputScores() const;

4
moses/src/PhraseDictionaryTree.h
View File

@ -116,6 +116,10 @@ public:
// for debugging
void PrintTargetCandidates(PrefixPtr p,std::ostream& out) const;
std::string GetScoreProducerDescription() const;
std::string GetScoreProducerWeightShortName() const
{
return "tm";
}
};

4
moses/src/PhraseDictionaryTreeAdaptor.h
View File

@ -67,6 +67,10 @@ class PhraseDictionaryTreeAdaptor : public PhraseDictionary {
void AddEquivPhrase(const Phrase &source, const TargetPhrase &targetPhrase);
std::string GetScoreProducerDescription() const;
std::string GetScoreProducerWeightShortName() const
{
return "tm";
}
size_t GetNumInputScores() const;

3
moses/src/ScoreProducer.h
View File

@ -48,6 +48,9 @@ public:
//! returns a string description of this producer
virtual std::string GetScoreProducerDescription() const = 0;
//! returns the weight parameter name of this producer (used in n-best list)
virtual std::string GetScoreProducerWeightShortName() const = 0;
//! returns the number of scores gathered from the input (0 by default)
virtual size_t GetNumInputScores() const { return 0; };

2
moses/src/Sentence.cpp
View File

@ -148,7 +148,7 @@ bool Sentence::XmlOverlap(size_t startPos, size_t endPos) const {
{
if (pos < m_xmlCoverageMap.size() && m_xmlCoverageMap[pos]) {
return true;
}
}
}
return false;
}

121
moses/src/StaticData.cpp
View File

@ -35,6 +35,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "LanguageModelMultiFactor.h"
#include "LanguageModelFactory.h"
#include "LexicalReordering.h"
#include "GlobalLexicalModel.h"
#include "SentenceStats.h"
#include "PhraseDictionaryTreeAdaptor.h"
#include "UserMessage.h"
@ -263,7 +264,7 @@ bool StaticData::LoadData(Parameter *parameter)
m_unknownWordPenaltyProducer = new UnknownWordPenaltyProducer(m_scoreIndexManager);
m_allWeights.push_back(m_weightUnknownWord);
// reordering constraints
// reordering constraints
m_maxDistortion = (m_parameter->GetParam("distortion-limit").size() > 0) ?
Scan<int>(m_parameter->GetParam("distortion-limit")[0])
: -1;
@ -272,28 +273,28 @@ bool StaticData::LoadData(Parameter *parameter)
// settings for pruning
m_maxHypoStackSize = (m_parameter->GetParam("stack").size() > 0)
? Scan<size_t>(m_parameter->GetParam("stack")[0]) : DEFAULT_MAX_HYPOSTACK_SIZE;
m_minHypoStackDiversity = 0;
if (m_parameter->GetParam("stack-diversity").size() > 0) {
if (m_maxDistortion > 15) {
UserMessage::Add("stack diversity > 0 is not allowed for distortion limits larger than 15");
return false;
}
if (m_inputType == WordLatticeInput) {
UserMessage::Add("stack diversity > 0 is not allowed for lattice input");
return false;
}
m_minHypoStackDiversity = Scan<size_t>(m_parameter->GetParam("stack-diversity")[0]);
}
m_minHypoStackDiversity = 0;
if (m_parameter->GetParam("stack-diversity").size() > 0) {
if (m_maxDistortion > 15) {
UserMessage::Add("stack diversity > 0 is not allowed for distortion limits larger than 15");
return false;
}
if (m_inputType == WordLatticeInput) {
UserMessage::Add("stack diversity > 0 is not allowed for lattice input");
return false;
}
m_minHypoStackDiversity = Scan<size_t>(m_parameter->GetParam("stack-diversity")[0]);
}
m_beamWidth = (m_parameter->GetParam("beam-threshold").size() > 0) ?
TransformScore(Scan<float>(m_parameter->GetParam("beam-threshold")[0]))
: TransformScore(DEFAULT_BEAM_WIDTH);
m_earlyDiscardingThreshold = (m_parameter->GetParam("early-discarding-threshold").size() > 0) ?
TransformScore(Scan<float>(m_parameter->GetParam("early-discarding-threshold")[0]))
: TransformScore(DEFAULT_EARLY_DISCARDING_THRESHOLD);
m_translationOptionThreshold = (m_parameter->GetParam("translation-option-threshold").size() > 0) ?
TransformScore(Scan<float>(m_parameter->GetParam("translation-option-threshold")[0]))
: TransformScore(DEFAULT_TRANSLATION_OPTION_THRESHOLD);
m_earlyDiscardingThreshold = (m_parameter->GetParam("early-discarding-threshold").size() > 0) ?
TransformScore(Scan<float>(m_parameter->GetParam("early-discarding-threshold")[0]))
: TransformScore(DEFAULT_EARLY_DISCARDING_THRESHOLD);
m_translationOptionThreshold = (m_parameter->GetParam("translation-option-threshold").size() > 0) ?
TransformScore(Scan<float>(m_parameter->GetParam("translation-option-threshold")[0]))
: TransformScore(DEFAULT_TRANSLATION_OPTION_THRESHOLD);
m_maxNoTransOptPerCoverage = (m_parameter->GetParam("max-trans-opt-per-coverage").size() > 0)
? Scan<size_t>(m_parameter->GetParam("max-trans-opt-per-coverage")[0]) : DEFAULT_MAX_TRANS_OPT_SIZE;
@ -310,8 +311,7 @@ bool StaticData::LoadData(Parameter *parameter)
m_cubePruningDiversity = (m_parameter->GetParam("cube-pruning-diversity").size() > 0)
? Scan<size_t>(m_parameter->GetParam("cube-pruning-diversity")[0]) : DEFAULT_CUBE_PRUNING_DIVERSITY;
// Unknown Word Processing -- wade
//TODO replace this w/general word dropping -- EVH
// unknown word processing
SetBooleanParameter( &m_dropUnknown, "drop-unknown", false );
// minimum Bayes risk decoding
@ -321,7 +321,7 @@ bool StaticData::LoadData(Parameter *parameter)
m_mbrScale = (m_parameter->GetParam("mbr-scale").size() > 0) ?
Scan<float>(m_parameter->GetParam("mbr-scale")[0]) : 1.0f;
m_timeout_threshold = (m_parameter->GetParam("time-out").size() > 0) ?
m_timeout_threshold = (m_parameter->GetParam("time-out").size() > 0) ?
Scan<size_t>(m_parameter->GetParam("time-out")[0]) : -1;
m_timeout = (GetTimeoutThreshold() == -1) ? false : true;
@ -374,13 +374,14 @@ bool StaticData::LoadData(Parameter *parameter)
if (!LoadGenerationTables()) return false;
if (!LoadPhraseTables()) return false;
if (!LoadMapping()) return false;
if (!LoadGlobalLexicalModel()) return false;
m_scoreIndexManager.InitFeatureNames();
m_scoreIndexManager.InitFeatureNames();
if (m_parameter->GetParam("weight-file").size() > 0) {
if (m_parameter->GetParam("weight-file").size() != 1) {
UserMessage::Add(string("ERROR: weight-file takes a single parameter"));
return false;
}
if (m_parameter->GetParam("weight-file").size() != 1) {
UserMessage::Add(string("ERROR: weight-file takes a single parameter"));
return false;
}
string fnam = m_parameter->GetParam("weight-file")[0];
m_scoreIndexManager.InitWeightVectorFromFile(fnam, &m_allWeights);
}
@ -419,6 +420,7 @@ StaticData::~StaticData()
RemoveAllInColl(m_languageModel);
RemoveAllInColl(m_decodeStepVL);
RemoveAllInColl(m_reorderModels);
RemoveAllInColl(m_globalLexicalModels);
// delete trans opt
map<std::pair<const DecodeGraph*, Phrase>, std::pair< TranslationOptionList*, clock_t > >::iterator iterCache;
@ -443,11 +445,7 @@ bool StaticData::LoadLexicalReorderingModel()
std::cerr << "Loading lexical distortion models...\n";
const vector<string> fileStr = m_parameter->GetParam("distortion-file");
const vector<string> weightsStr = m_parameter->GetParam("weight-d");
/*old code
const vector<string> modelStr = m_parameter.GetParam("distortion-type"); //TODO check name?
const vector<string> fileStr = m_parameter.GetParam("distortion-file");
const vector<string> weightsStr = m_parameter.GetParam("weight-d");
*/
std::vector<float> weights;
size_t w = 1; //cur weight
size_t f = 0; //cur file
@ -459,8 +457,6 @@ bool StaticData::LoadLexicalReorderingModel()
//load all models
for(size_t i = 0; i < fileStr.size(); ++i)
{
//std::cerr << "Model " << i << ":";
//Todo: 'else' should be 'else if(...)' to check it is a lexical model...
vector<string> spec = Tokenize<string>(fileStr[f], " ");
++f; //mark file as consumed
if(4 != spec.size()){
@ -594,6 +590,39 @@ bool StaticData::LoadLexicalReorderingModel()
return true;
}
bool StaticData::LoadGlobalLexicalModel()
{
const vector<float> &weight = Scan<float>(m_parameter->GetParam("weight-lex"));
const vector<string> &file = m_parameter->GetParam("global-lexical-file");
if (weight.size() != file.size())
{
std::cerr << "number of weights and models for the global lexical model does not match ("
<< weight.size() << " != " << file.size() << ")" << std::endl;
return false;
}
for (size_t i = 0; i < weight.size(); i++ )
{
vector<string> spec = Tokenize<string>(file[i], " ");
if ( spec.size() != 2 )
{
std::cerr << "wrong global lexical model specification: " << file[i] << endl;
return false;
}
vector< string > factors = Tokenize(spec[0],"-");
if ( factors.size() != 2 )
{
std::cerr << "wrong factor definition for global lexical model: " << spec[0] << endl;
return false;
}
vector<FactorType> inputFactors = Tokenize<FactorType>(factors[0],",");
vector<FactorType> outputFactors = Tokenize<FactorType>(factors[1],",");
m_globalLexicalModels.push_back( new GlobalLexicalModel( spec[1], weight[i], inputFactors, outputFactors ) );
}
return true;
}
bool StaticData::LoadLanguageModels()
{
if (m_parameter->GetParam("lmodel-file").size() > 0)
@ -968,21 +997,23 @@ void StaticData::CleanUpAfterSentenceProcessing() const
binary format is used) */
void StaticData::InitializeBeforeSentenceProcessing(InputType const& in) const
{
m_input = &in;
for(size_t i=0;i<m_phraseDictionary.size();++i) {
m_phraseDictionary[i]->InitializeForInput(in);
}
for(size_t j=0;j<m_reorderModels.size();++j){
m_reorderModels[j]->InitializeForInput(in);
}
//something LMs could do before translating a sentence
LMList::const_iterator iterLM;
m_input = &in;
for(size_t i=0;i<m_phraseDictionary.size();++i) {
m_phraseDictionary[i]->InitializeForInput(in);
}
for(size_t i=0;i<m_reorderModels.size();++i) {
m_reorderModels[i]->InitializeForInput(in);
}
for(size_t i=0;i<m_globalLexicalModels.size();++i) {
m_globalLexicalModels[i]->InitializeForInput((Sentence const&)in);
}
//something LMs could do before translating a sentence
LMList::const_iterator iterLM;
for (iterLM = m_languageModel.begin() ; iterLM != m_languageModel.end() ; ++iterLM)
{
LanguageModel &languageModel = **iterLM;
languageModel.InitializeBeforeSentenceProcessing();
languageModel.InitializeBeforeSentenceProcessing();
}
}
void StaticData::SetWeightsForScoreProducer(const ScoreProducer* sp, const std::vector<float>& weights)

3
moses/src/StaticData.h
View File

@ -45,6 +45,7 @@ namespace Moses
class InputType;
class LexicalReordering;
class GlobalLexicalModel;
class PhraseDictionary;
class GenerationDictionary;
class DistortionScoreProducer;
@ -69,6 +70,7 @@ protected:
ScoreIndexManager m_scoreIndexManager;
std::vector<float> m_allWeights;
std::vector<LexicalReordering*> m_reorderModels;
std::vector<GlobalLexicalModel*> m_globalLexicalModels;
// Initial = 0 = can be used when creating poss trans
// Other = 1 = used to calculate LM score once all steps have been processed
float
@ -176,6 +178,7 @@ protected:
//! load decoding steps
bool LoadMapping();
bool LoadLexicalReorderingModel();
bool LoadGlobalLexicalModel();
public:

3
moses/src/TargetPhraseCollection.h
View File

@ -28,7 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
namespace Moses
{
//! a list of target phrases that is trsnalated from the same source phrase
//! a list of target phrases that is translated from the same source phrase
class TargetPhraseCollection
{
protected:
@ -72,4 +72,3 @@ public:
}

1
moses/src/TranslationOptionCollectionText.cpp
View File

@ -51,7 +51,6 @@ void TranslationOptionCollectionText::ProcessUnknownWord(size_t sourcePos)
bool TranslationOptionCollectionText::HasXmlOptionsOverlappingRange(size_t startPosition, size_t endPosition) const {
Sentence const& source=dynamic_cast<Sentence const&>(m_source);
return source.XmlOverlap(startPosition,endPosition);
}
/**

72
regression-testing/tests/lattice-distortion/truth/results.txt
View File

@ -14,40 +14,40 @@ SCORE_4 = 2.004
SCORE_5 = -0.233
SCORE_6 = -1.851
SCORE_7 = -196.549
TRANSLATION_0_NBEST_1=1 2 ||| d: 0 lm: -4.30484 I: 0 tm: -4.60517 -4.60517 w: -2
TRANSLATION_0_NBEST_2=2 1 ||| d: -3 lm: -9.70915 I: 0 tm: -4.60517 -4.60517 w: -2
TRANSLATION_1_NBEST_1=1 2 3 ||| d: 0 lm: -6.08823 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_1_NBEST_2=12 3 ||| d: 0 lm: -7.98575 I: 0 tm: -4.60517 -4.60517 w: -2
TRANSLATION_1_NBEST_3=2 3 1 ||| d: -2 lm: -9.26988 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_1_NBEST_4=1 3 2 ||| d: -2 lm: -10.7767 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_1_NBEST_5=3 1 2 ||| d: -3 lm: -10.9714 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_1_NBEST_6=2 1 3 ||| d: -1 lm: -13.645 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_1_NBEST_7=3 12 ||| d: -3 lm: -10.0318 I: 0 tm: -4.60517 -4.60517 w: -2
TRANSLATION_1_NBEST_8=3 2 1 ||| d: -4 lm: -12.57 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_2_NBEST_1=1 2 3 ||| d: 0 lm: -6.08823 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_2_NBEST_2=2 3 1 ||| d: -2 lm: -9.26988 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_2_NBEST_3=1 3 2 ||| d: -2 lm: -10.7767 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_2_NBEST_4=3 1 2 ||| d: -3 lm: -10.9714 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_2_NBEST_5=2 1 3 ||| d: -1 lm: -13.645 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_2_NBEST_6=3 2 1 ||| d: -4 lm: -12.57 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_3_NBEST_1=1 2 3 ||| d: 0 lm: -6.08823 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_3_NBEST_2=1 3 2 ||| d: -1 lm: -10.7767 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_3_NBEST_3=1 12 ||| d: 0 lm: -9.28642 I: 0 tm: -4.60517 -4.60517 w: -2
TRANSLATION_3_NBEST_4=2 3 1 ||| d: -3 lm: -9.26988 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_3_NBEST_5=3 1 2 ||| d: -3 lm: -10.9714 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_3_NBEST_6=12 1 ||| d: -3 lm: -8.59503 I: 0 tm: -4.60517 -4.60517 w: -2
TRANSLATION_3_NBEST_7=3 2 1 ||| d: -4 lm: -12.57 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_3_NBEST_8=2 1 3 ||| d: -3 lm: -13.645 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_4_NBEST_1=1234 3 4 5 ||| d: -3 lm: -8.02249 I: 0 tm: -9.21034 -9.21034 w: -4
TRANSLATION_4_NBEST_2=4 1234 5 ||| d: -2 lm: -7.07413 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_4_NBEST_3=5 4 1234 ||| d: -1 lm: -8.17776 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_4_NBEST_4=3 4 5 1234 ||| d: 0 lm: -12.8014 I: 0 tm: -9.21034 -9.21034 w: -4
TRANSLATION_4_NBEST_5=2 3 1234 ||| d: 0 lm: -10.4352 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_4_NBEST_6=1234 4 5 ||| d: -3 lm: -7.84275 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_4_NBEST_7=3 4 1234 5 ||| d: -2 lm: -11.6793 I: 0 tm: -9.21034 -9.21034 w: -4
TRANSLATION_4_NBEST_8=1234 5 4 ||| d: -4 lm: -6.99714 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_4_NBEST_9=1 1234 ||| d: 0 lm: -8.4748 I: 0 tm: -4.60517 -4.60517 w: -2
TRANSLATION_4_NBEST_10=4 5 1234 ||| d: 0 lm: -12.0545 I: 0 tm: -6.90776 -6.90776 w: -3
TRANSLATION_5_NBEST_1=1 ||| d: 0 lm: -4.54836 I: 0 tm: -2.30259 -2.30259 w: -1
TRANSLATION_6_NBEST_1=' ||| d: 0 lm: -101.55 I: 0 tm: 0 0 w: -1
TRANSLATION_0_NBEST_1=1 2 ||| d: 0 lm: -4.30484 w: -2 I: 0 tm: -4.60517 -4.60517
TRANSLATION_0_NBEST_2=2 1 ||| d: -3 lm: -9.70915 w: -2 I: 0 tm: -4.60517 -4.60517
TRANSLATION_1_NBEST_1=1 2 3 ||| d: 0 lm: -6.08823 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_1_NBEST_2=12 3 ||| d: 0 lm: -7.98575 w: -2 I: 0 tm: -4.60517 -4.60517
TRANSLATION_1_NBEST_3=2 3 1 ||| d: -2 lm: -9.26988 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_1_NBEST_4=1 3 2 ||| d: -2 lm: -10.7767 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_1_NBEST_5=3 1 2 ||| d: -3 lm: -10.9714 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_1_NBEST_6=2 1 3 ||| d: -1 lm: -13.645 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_1_NBEST_7=3 12 ||| d: -3 lm: -10.0318 w: -2 I: 0 tm: -4.60517 -4.60517
TRANSLATION_1_NBEST_8=3 2 1 ||| d: -4 lm: -12.57 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_2_NBEST_1=1 2 3 ||| d: 0 lm: -6.08823 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_2_NBEST_2=2 3 1 ||| d: -2 lm: -9.26988 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_2_NBEST_3=1 3 2 ||| d: -2 lm: -10.7767 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_2_NBEST_4=3 1 2 ||| d: -3 lm: -10.9714 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_2_NBEST_5=2 1 3 ||| d: -1 lm: -13.645 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_2_NBEST_6=3 2 1 ||| d: -4 lm: -12.57 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_3_NBEST_1=1 2 3 ||| d: 0 lm: -6.08823 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_3_NBEST_2=1 3 2 ||| d: -1 lm: -10.7767 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_3_NBEST_3=1 12 ||| d: 0 lm: -9.28642 w: -2 I: 0 tm: -4.60517 -4.60517
TRANSLATION_3_NBEST_4=2 3 1 ||| d: -3 lm: -9.26988 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_3_NBEST_5=3 1 2 ||| d: -3 lm: -10.9714 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_3_NBEST_6=12 1 ||| d: -3 lm: -8.59503 w: -2 I: 0 tm: -4.60517 -4.60517
TRANSLATION_3_NBEST_7=3 2 1 ||| d: -4 lm: -12.57 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_3_NBEST_8=2 1 3 ||| d: -3 lm: -13.645 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_4_NBEST_1=1234 3 4 5 ||| d: -3 lm: -8.02249 w: -4 I: 0 tm: -9.21034 -9.21034
TRANSLATION_4_NBEST_2=4 1234 5 ||| d: -2 lm: -7.07413 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_4_NBEST_3=5 4 1234 ||| d: -1 lm: -8.17776 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_4_NBEST_4=3 4 5 1234 ||| d: 0 lm: -12.8014 w: -4 I: 0 tm: -9.21034 -9.21034
TRANSLATION_4_NBEST_5=2 3 1234 ||| d: 0 lm: -10.4352 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_4_NBEST_6=1234 4 5 ||| d: -3 lm: -7.84275 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_4_NBEST_7=3 4 1234 5 ||| d: -2 lm: -11.6793 w: -4 I: 0 tm: -9.21034 -9.21034
TRANSLATION_4_NBEST_8=1234 5 4 ||| d: -4 lm: -6.99714 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_4_NBEST_9=1 1234 ||| d: 0 lm: -8.4748 w: -2 I: 0 tm: -4.60517 -4.60517
TRANSLATION_4_NBEST_10=4 5 1234 ||| d: 0 lm: -12.0545 w: -3 I: 0 tm: -6.90776 -6.90776
TRANSLATION_5_NBEST_1=1 ||| d: 0 lm: -4.54836 w: -1 I: 0 tm: -2.30259 -2.30259
TRANSLATION_6_NBEST_1=' ||| d: 0 lm: -101.55 w: -1 I: 0 tm: 0 0
TOTAL_WALLTIME ~ 0

4
regression-testing/tests/nbest-multi-factor/truth/results.txt
View File

@ -2,6 +2,6 @@ TRANSLATION_0=there are various different opinions .
LMLOAD_TIME ~ 11.00
PTLOAD_TIME ~ 23.00
SCORE_0= -113.734
TRANSLATION_0_NBEST_1=there are various different opinions . ||| d: 0 lm: -21.6664 tm: -3.46226 -10.822 -2.82666 -9.32004 2.99969 -2.44147 -4.91188 -6.12017 -10.5033 2.99969 w: -6 g: -0.0625204 -18.098
TRANSLATION_0_NBEST_2=there are various other opinions . ||| d: 0 lm: -25.3276 tm: -1.94591 -10.3833 -0.693147 -7.77959 1.99979 -0.924822 -4.91188 -3.33314 -10.5033 1.99979 w: -6 g: -0.149241 -16.9924
TRANSLATION_0_NBEST_1=there are various different opinions . ||| d: 0 lm: -21.6664 w: -6 tm: -3.46226 -10.822 -2.82666 -9.32004 2.99969 -2.44147 -4.91188 -6.12017 -10.5033 2.99969 g: -0.0625204 -18.098
TRANSLATION_0_NBEST_2=there are various other opinions . ||| d: 0 lm: -25.3276 w: -6 tm: -1.94591 -10.3833 -0.693147 -7.77959 1.99979 -0.924822 -4.91188 -3.33314 -10.5033 1.99979 g: -0.149241 -16.9924
TOTAL_WALLTIME ~ 23

8
scripts/training/mert-moses-new.pl
View File

@ -73,18 +73,19 @@ my $additional_triples = {
[ 0.3, 0.0, 0.5 ],
[ 0.2, 0.0, 0.5 ],
[ 0.0,-1.0, 1.0 ] ], # ... last weight is phrase penalty
"lex"=> [ [ 0.1, 0.0, 0.2 ] ], # global lexical model
};
# moses.ini file uses FULL names for lambdas, while this training script internally (and on the command line)
# uses ABBR names.
my $ABBR_FULL_MAP = "d=weight-d lm=weight-l tm=weight-t w=weight-w g=weight-generation";
my $ABBR_FULL_MAP = "d=weight-d lm=weight-l tm=weight-t w=weight-w g=weight-generation lex=weight-lex";
my %ABBR2FULL = map {split/=/,$_,2} split /\s+/, $ABBR_FULL_MAP;
my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} split /\s+/, $ABBR_FULL_MAP;
# We parse moses.ini to figure out how many weights do we need to optimize.
# For this, we must know the correspondence between options defining files
# for models and options assigning weights to these models.
my $TABLECONFIG_ABBR_MAP = "ttable-file=tm lmodel-file=lm distortion-file=d generation-file=g";
my $TABLECONFIG_ABBR_MAP = "ttable-file=tm lmodel-file=lm distortion-file=d generation-file=g global-lexical-file=lex";
my %TABLECONFIG2ABBR = map {split(/=/,$_,2)} split /\s+/, $TABLECONFIG_ABBR_MAP;
# There are weights that do not correspond to any input file, they just increase the total number of lambdas we optimize
@ -415,6 +416,7 @@ if ($___DECODER_FLAGS =~ /(^|\s)-(config|f) /
|| $___DECODER_FLAGS =~ /(^|\s)-(distortion-file) /
|| $___DECODER_FLAGS =~ /(^|\s)-(generation-file) /
|| $___DECODER_FLAGS =~ /(^|\s)-(lmodel-file) /
|| $___DECODER_FLAGS =~ /(^|\s)-(global-lexical-file) /
) {
die "It is forbidden to supply any of -config, -ttable-file, -distortion-file, -generation-file or -lmodel-file in the --decoder-flags.\nPlease use only the --config option to give the config file that lists all the supplementary files.";
}
@ -1026,6 +1028,7 @@ sub scan_config {
"generation-file" => 3,
"lmodel-file" => 3,
"distortion-file" => 3,
"global-lexical-file" => 1,
);
# by default, each line of each section means one lambda, but some sections
# explicitly state a custom number of lambdas
@ -1055,6 +1058,7 @@ sub scan_config {
$defined_steps{$1}++ if /^([TG])/ || /^\d+ ([TG])/;
}
if (defined $section && defined $where_is_filename{$section}) {
print "$section -> $where_is_filename{$section}\n";
# this ini section is relevant to lambdas
chomp;
my @flds = split / +/;

7
scripts/training/mert-moses.pl
View File

@ -83,18 +83,19 @@ my $additional_triples = {
[ 0.3, 0.0, 0.5 ],
[ 0.2, 0.0, 0.5 ],
[ 0.0,-1.0, 1.0 ] ], # ... last weight is phrase penalty
"lex"=> [ [ 0.1, 0.0, 0.2 ] ], # global lexical model
};
# moses.ini file uses FULL names for lambdas, while this training script internally (and on the command line)
# uses ABBR names.
my $ABBR_FULL_MAP = "d=weight-d lm=weight-l tm=weight-t w=weight-w g=weight-generation";
my $ABBR_FULL_MAP = "d=weight-d lm=weight-l tm=weight-t w=weight-w g=weight-generation lex=weight-lex";
my %ABBR2FULL = map {split/=/,$_,2} split /\s+/, $ABBR_FULL_MAP;
my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} split /\s+/, $ABBR_FULL_MAP;
# We parse moses.ini to figure out how many weights do we need to optimize.
# For this, we must know the correspondence between options defining files
# for models and options assigning weights to these models.
my $TABLECONFIG_ABBR_MAP = "ttable-file=tm lmodel-file=lm distortion-file=d generation-file=g";
my $TABLECONFIG_ABBR_MAP = "ttable-file=tm lmodel-file=lm distortion-file=d generation-file=g global-lexical-file=lex";
my %TABLECONFIG2ABBR = map {split(/=/,$_,2)} split /\s+/, $TABLECONFIG_ABBR_MAP;
# There are weights that do not correspond to any input file, they just increase the total number of lambdas we optimize
@ -407,6 +408,7 @@ if ($___DECODER_FLAGS =~ /(^|\s)-(config|f) /
|| $___DECODER_FLAGS =~ /(^|\s)-(distortion-file) /
|| $___DECODER_FLAGS =~ /(^|\s)-(generation-file) /
|| $___DECODER_FLAGS =~ /(^|\s)-(lmodel-file) /
|| $___DECODER_FLAGS =~ /(^|\s)-(global-lexical-file) /
) {
die "It is forbidden to supply any of -config, -ttable-file, -distortion-file, -generation-file or -lmodel-file in the --decoder-flags.\nPlease use only the --config option to give the config file that lists all the supplementary files.";
}
@ -1084,6 +1086,7 @@ sub scan_config {
"generation-file" => 3,
"lmodel-file" => 3,
"distortion-file" => 3,
"global-lexical-file" => 1,
);
# by default, each line of each section means one lambda, but some sections
# explicitly state a custom number of lambdas