mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-08-16 06:50:32 +03:00
daily automatic beautifier
This commit is contained in:
parent
2da95fe74a
commit
59365f2bd3
@ -130,8 +130,8 @@ void ChartManager::Decode()
|
|||||||
*/
|
*/
|
||||||
void ChartManager::AddXmlChartOptions()
|
void ChartManager::AddXmlChartOptions()
|
||||||
{
|
{
|
||||||
const std::vector <ChartTranslationOptions*> xmlChartOptionsList
|
const std::vector <ChartTranslationOptions*> xmlChartOptionsList
|
||||||
= m_source.GetXmlChartTranslationOptions();
|
= m_source.GetXmlChartTranslationOptions();
|
||||||
IFVERBOSE(2) {
|
IFVERBOSE(2) {
|
||||||
cerr << "AddXmlChartOptions " << xmlChartOptionsList.size() << endl;
|
cerr << "AddXmlChartOptions " << xmlChartOptionsList.size() << endl;
|
||||||
}
|
}
|
||||||
|
@ -91,8 +91,8 @@ void ChartParserUnknown::Process(const Word &sourceWord, const Range &range, Cha
|
|||||||
//const Word &sourceLHS = staticData.GetInputDefaultNonTerminal();
|
//const Word &sourceLHS = staticData.GetInputDefaultNonTerminal();
|
||||||
Word *targetLHS = new Word(true);
|
Word *targetLHS = new Word(true);
|
||||||
|
|
||||||
targetLHS->CreateFromString(Output, staticData.options().output.factor_order,
|
targetLHS->CreateFromString(Output, staticData.options().output.factor_order,
|
||||||
targetLHSStr, true);
|
targetLHSStr, true);
|
||||||
UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS");
|
UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS");
|
||||||
|
|
||||||
// add to dictionary
|
// add to dictionary
|
||||||
@ -133,8 +133,8 @@ void ChartParserUnknown::Process(const Word &sourceWord, const Range &range, Cha
|
|||||||
//float prob = iterLHS->second;
|
//float prob = iterLHS->second;
|
||||||
|
|
||||||
Word *targetLHS = new Word(true);
|
Word *targetLHS = new Word(true);
|
||||||
targetLHS->CreateFromString(Output, staticData.options().output.factor_order,
|
targetLHS->CreateFromString(Output, staticData.options().output.factor_order,
|
||||||
targetLHSStr, true);
|
targetLHSStr, true);
|
||||||
UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS");
|
UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS");
|
||||||
|
|
||||||
targetPhrase->GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, unknownScore);
|
targetPhrase->GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, unknownScore);
|
||||||
@ -210,12 +210,12 @@ void ChartParser::Create(const Range &range, ChartParserCallback &to)
|
|||||||
ruleLookupManager.GetChartRuleCollection(inputPath, last, to);
|
ruleLookupManager.GetChartRuleCollection(inputPath, last, to);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (range.GetNumWordsCovered() == 1
|
if (range.GetNumWordsCovered() == 1
|
||||||
&& range.GetStartPos() != 0
|
&& range.GetStartPos() != 0
|
||||||
&& range.GetStartPos() != m_source.GetSize()-1) {
|
&& range.GetStartPos() != m_source.GetSize()-1) {
|
||||||
bool always = m_ttask.lock()->options().unk.always_create_direct_transopt;
|
bool always = m_ttask.lock()->options().unk.always_create_direct_transopt;
|
||||||
// bool alwaysCreateDirectTranslationOption
|
// bool alwaysCreateDirectTranslationOption
|
||||||
// = StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
|
// = StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
|
||||||
if (to.Empty() || always) {
|
if (to.Empty() || always) {
|
||||||
// create unknown words for 1 word coverage where we don't have any trans options
|
// create unknown words for 1 word coverage where we don't have any trans options
|
||||||
|
@ -282,10 +282,10 @@ ConfusionNet::
|
|||||||
CreateTranslationOptionCollection(ttasksptr const& ttask) const
|
CreateTranslationOptionCollection(ttasksptr const& ttask) const
|
||||||
{
|
{
|
||||||
size_t maxNoTransOptPerCoverage
|
size_t maxNoTransOptPerCoverage
|
||||||
= ttask->options().search.max_trans_opt_per_cov;
|
= ttask->options().search.max_trans_opt_per_cov;
|
||||||
// StaticData::Instance().GetMaxNoTransOptPerCoverage();
|
// StaticData::Instance().GetMaxNoTransOptPerCoverage();
|
||||||
float translationOptionThreshold
|
float translationOptionThreshold
|
||||||
= ttask->options().search.trans_opt_threshold;
|
= ttask->options().search.trans_opt_threshold;
|
||||||
// StaticData::Instance().GetTranslationOptionThreshold();
|
// StaticData::Instance().GetTranslationOptionThreshold();
|
||||||
TranslationOptionCollection *rv
|
TranslationOptionCollection *rv
|
||||||
= new TranslationOptionCollectionConfusionNet
|
= new TranslationOptionCollectionConfusionNet
|
||||||
|
@ -141,18 +141,16 @@ void SimpleTranslationInterface::DestroyFeatureFunctionStatic()
|
|||||||
|
|
||||||
Parameter params;
|
Parameter params;
|
||||||
|
|
||||||
void
|
void
|
||||||
signal_handler(int signum)
|
signal_handler(int signum)
|
||||||
{
|
{
|
||||||
if (signum == SIGALRM) {
|
if (signum == SIGALRM) {
|
||||||
exit(0); // that's what we expected from the child process after forking
|
exit(0); // that's what we expected from the child process after forking
|
||||||
}
|
} else if (signum == SIGTERM || signum == SIGKILL) {
|
||||||
else if (signum == SIGTERM || signum == SIGKILL) {
|
|
||||||
exit(0);
|
exit(0);
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
std::cerr << "Unexpected signal " << signum << std::endl;
|
std::cerr << "Unexpected signal " << signum << std::endl;
|
||||||
exit(signum);
|
exit(signum);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -161,7 +159,7 @@ int
|
|||||||
run_as_server()
|
run_as_server()
|
||||||
{
|
{
|
||||||
#ifdef HAVE_XMLRPC_C
|
#ifdef HAVE_XMLRPC_C
|
||||||
kill(getppid(),SIGALRM);
|
kill(getppid(),SIGALRM);
|
||||||
MosesServer::Server server(params);
|
MosesServer::Server server(params);
|
||||||
return server.run(); // actually: don't return. see Server::run()
|
return server.run(); // actually: don't return. see Server::run()
|
||||||
#else
|
#else
|
||||||
@ -344,7 +342,10 @@ int decoder_main(int argc, char const** argv)
|
|||||||
pid_t pid;
|
pid_t pid;
|
||||||
if (params.GetParam("daemon")) {
|
if (params.GetParam("daemon")) {
|
||||||
pid = fork();
|
pid = fork();
|
||||||
if (pid) { pause(); exit(0); } // parent process
|
if (pid) {
|
||||||
|
pause(); // parent process
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
// setting "-show-weights" -> just dump out weights and exit
|
// setting "-show-weights" -> just dump out weights and exit
|
||||||
@ -356,7 +357,7 @@ int decoder_main(int argc, char const** argv)
|
|||||||
if (params.GetParam("server")) {
|
if (params.GetParam("server")) {
|
||||||
std::cerr << "RUN SERVER at pid " << pid << std::endl;
|
std::cerr << "RUN SERVER at pid " << pid << std::endl;
|
||||||
return run_as_server();
|
return run_as_server();
|
||||||
} else
|
} else
|
||||||
return batch_run();
|
return batch_run();
|
||||||
}
|
}
|
||||||
#ifdef NDEBUG
|
#ifdef NDEBUG
|
||||||
|
@ -50,8 +50,8 @@ void ConstrainedDecoding::Load(AllOptions const& opts)
|
|||||||
{
|
{
|
||||||
const StaticData &staticData = StaticData::Instance();
|
const StaticData &staticData = StaticData::Instance();
|
||||||
bool addBeginEndWord
|
bool addBeginEndWord
|
||||||
= ((opts.search.algo == CYKPlus) || (opts.search.algo == ChartIncremental));
|
= ((opts.search.algo == CYKPlus) || (opts.search.algo == ChartIncremental));
|
||||||
|
|
||||||
for(size_t i = 0; i < m_paths.size(); ++i) {
|
for(size_t i = 0; i < m_paths.size(); ++i) {
|
||||||
InputFileStream constraintFile(m_paths[i]);
|
InputFileStream constraintFile(m_paths[i]);
|
||||||
std::string line;
|
std::string line;
|
||||||
|
@ -7,11 +7,11 @@ namespace Moses
|
|||||||
InternalTree::InternalTree(const std::string & line, size_t start, size_t len, const bool nonterminal)
|
InternalTree::InternalTree(const std::string & line, size_t start, size_t len, const bool nonterminal)
|
||||||
{
|
{
|
||||||
|
|
||||||
std::vector<FactorType> const& oFactors
|
std::vector<FactorType> const& oFactors
|
||||||
= StaticData::Instance().options().output.factor_order;
|
= StaticData::Instance().options().output.factor_order;
|
||||||
if (len > 0) {
|
if (len > 0) {
|
||||||
m_value.CreateFromString(Output, oFactors, StringPiece(line).substr(start, len),
|
m_value.CreateFromString(Output, oFactors, StringPiece(line).substr(start, len),
|
||||||
nonterminal);
|
nonterminal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -21,9 +21,9 @@ InternalTree::InternalTree(const std::string & line, const bool nonterminal)
|
|||||||
size_t found = line.find_first_of("[] ");
|
size_t found = line.find_first_of("[] ");
|
||||||
|
|
||||||
if (found == line.npos) {
|
if (found == line.npos) {
|
||||||
m_value.CreateFromString(Output,
|
m_value.CreateFromString(Output,
|
||||||
StaticData::Instance().options().output.factor_order,
|
StaticData::Instance().options().output.factor_order,
|
||||||
line, nonterminal);
|
line, nonterminal);
|
||||||
} else {
|
} else {
|
||||||
AddSubTree(line, 0);
|
AddSubTree(line, 0);
|
||||||
}
|
}
|
||||||
@ -49,18 +49,18 @@ size_t InternalTree::AddSubTree(const std::string & line, size_t pos)
|
|||||||
pos = m_children.back()->AddSubTree(line, pos+1);
|
pos = m_children.back()->AddSubTree(line, pos+1);
|
||||||
} else {
|
} else {
|
||||||
if (len > 0) {
|
if (len > 0) {
|
||||||
m_value.CreateFromString(Output,
|
m_value.CreateFromString(Output,
|
||||||
StaticData::Instance().options().output.factor_order,
|
StaticData::Instance().options().output.factor_order,
|
||||||
StringPiece(line).substr(oldpos, len), false);
|
StringPiece(line).substr(oldpos, len), false);
|
||||||
has_value = true;
|
has_value = true;
|
||||||
}
|
}
|
||||||
pos = AddSubTree(line, pos+1);
|
pos = AddSubTree(line, pos+1);
|
||||||
}
|
}
|
||||||
} else if (token == ' ' || token == ']') {
|
} else if (token == ' ' || token == ']') {
|
||||||
if (len > 0 && !has_value) {
|
if (len > 0 && !has_value) {
|
||||||
m_value.CreateFromString(Output,
|
m_value.CreateFromString(Output,
|
||||||
StaticData::Instance().options().output.factor_order,
|
StaticData::Instance().options().output.factor_order,
|
||||||
StringPiece(line).substr(oldpos, len), true);
|
StringPiece(line).substr(oldpos, len), true);
|
||||||
has_value = true;
|
has_value = true;
|
||||||
} else if (len > 0) {
|
} else if (len > 0) {
|
||||||
m_children.push_back(boost::make_shared<InternalTree>(line, oldpos, len, false));
|
m_children.push_back(boost::make_shared<InternalTree>(line, oldpos, len, false));
|
||||||
|
@ -124,8 +124,8 @@ const std::string& SoftMatchingFeature::GetOrSetFeatureName(const Word& RHS, con
|
|||||||
boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
|
boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
|
||||||
#endif
|
#endif
|
||||||
std::string &name = m_nameCache[RHS[0]->GetId()][LHS[0]->GetId()];
|
std::string &name = m_nameCache[RHS[0]->GetId()][LHS[0]->GetId()];
|
||||||
const std::vector<FactorType> & oFactors
|
const std::vector<FactorType> & oFactors
|
||||||
= StaticData::Instance().options().output.factor_order;
|
= StaticData::Instance().options().output.factor_order;
|
||||||
std::string LHS_string = LHS.GetString(oFactors, false);
|
std::string LHS_string = LHS.GetString(oFactors, false);
|
||||||
std::string RHS_string = RHS.GetString(oFactors, false);
|
std::string RHS_string = RHS.GetString(oFactors, false);
|
||||||
name = LHS_string + "->" + RHS_string;
|
name = LHS_string + "->" + RHS_string;
|
||||||
|
@ -342,11 +342,11 @@ OutputAlignment(std::ostream &out, bool recursive=true) const
|
|||||||
// call with head recursion to output things in the right order
|
// call with head recursion to output things in the right order
|
||||||
size_t trg_off = recursive && m_prevHypo ? m_prevHypo->OutputAlignment(out) : 0;
|
size_t trg_off = recursive && m_prevHypo ? m_prevHypo->OutputAlignment(out) : 0;
|
||||||
size_t src_off = GetCurrSourceWordsRange().GetStartPos();
|
size_t src_off = GetCurrSourceWordsRange().GetStartPos();
|
||||||
|
|
||||||
typedef std::pair<size_t,size_t> const* entry;
|
typedef std::pair<size_t,size_t> const* entry;
|
||||||
std::vector<entry> alnvec = tp.GetAlignTerm().GetSortedAlignments(waso);
|
std::vector<entry> alnvec = tp.GetAlignTerm().GetSortedAlignments(waso);
|
||||||
BOOST_FOREACH(entry e, alnvec)
|
BOOST_FOREACH(entry e, alnvec)
|
||||||
out << e->first + src_off << "-" << e->second + trg_off << " ";
|
out << e->first + src_off << "-" << e->second + trg_off << " ";
|
||||||
return trg_off + tp.GetSize();
|
return trg_off + tp.GetSize();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -88,7 +88,7 @@ IOWrapper::IOWrapper()
|
|||||||
m_currentLine = staticData.GetStartTranslationId();
|
m_currentLine = staticData.GetStartTranslationId();
|
||||||
|
|
||||||
m_inputFactorOrder = &staticData.options().input.factor_order;
|
m_inputFactorOrder = &staticData.options().input.factor_order;
|
||||||
|
|
||||||
size_t nBestSize = staticData.options().nbest.nbest_size;
|
size_t nBestSize = staticData.options().nbest.nbest_size;
|
||||||
string nBestFilePath = staticData.options().nbest.output_file_path;
|
string nBestFilePath = staticData.options().nbest.output_file_path;
|
||||||
|
|
||||||
|
@ -329,7 +329,7 @@ OutputNBestList(OutputCollector *collector,
|
|||||||
{
|
{
|
||||||
const StaticData &staticData = StaticData::Instance();
|
const StaticData &staticData = StaticData::Instance();
|
||||||
const std::vector<Moses::FactorType> &outputFactorOrder
|
const std::vector<Moses::FactorType> &outputFactorOrder
|
||||||
= options().output.factor_order;
|
= options().output.factor_order;
|
||||||
|
|
||||||
std::ostringstream out;
|
std::ostringstream out;
|
||||||
// wtf? copied from the original OutputNBestList
|
// wtf? copied from the original OutputNBestList
|
||||||
|
@ -561,7 +561,7 @@ void Manager::OutputWordGraph(std::ostream &outputWordGraphStream, const Hypothe
|
|||||||
}
|
}
|
||||||
|
|
||||||
// output both source and target phrases in the word graph
|
// output both source and target phrases in the word graph
|
||||||
outputWordGraphStream << "\tw=" << hypo->GetSourcePhraseStringRep()
|
outputWordGraphStream << "\tw=" << hypo->GetSourcePhraseStringRep()
|
||||||
<< "|" << hypo->GetCurrTargetPhrase();
|
<< "|" << hypo->GetCurrTargetPhrase();
|
||||||
|
|
||||||
outputWordGraphStream << endl;
|
outputWordGraphStream << endl;
|
||||||
@ -821,9 +821,9 @@ size_t Manager::OutputFeatureWeightsForSLF(size_t index, const FeatureFunction*
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t
|
size_t
|
||||||
Manager::
|
Manager::
|
||||||
OutputFeatureValuesForSLF(size_t index, bool zeros, const Hypothesis* hypo,
|
OutputFeatureValuesForSLF(size_t index, bool zeros, const Hypothesis* hypo,
|
||||||
const FeatureFunction* ff, std::ostream &out) const
|
const FeatureFunction* ff, std::ostream &out) const
|
||||||
{
|
{
|
||||||
const ScoreComponentCollection& scoreCollection = hypo->GetScoreBreakdown();
|
const ScoreComponentCollection& scoreCollection = hypo->GetScoreBreakdown();
|
||||||
@ -836,7 +836,7 @@ OutputFeatureValuesForSLF(size_t index, bool zeros, const Hypothesis* hypo,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**! Output search graph in hypergraph format of Kenneth Heafield's lazy hypergraph decoder */
|
/**! Output search graph in hypergraph format of Kenneth Heafield's lazy hypergraph decoder */
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const
|
OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const
|
||||||
{
|
{
|
||||||
@ -1112,7 +1112,7 @@ OutputSearchNode(AllOptions const& opts, long translationId,
|
|||||||
<< " back=" << prevHypo->GetId()
|
<< " back=" << prevHypo->GetId()
|
||||||
<< " score=" << searchNode.hypo->GetScore()
|
<< " score=" << searchNode.hypo->GetScore()
|
||||||
<< " transition=" << (searchNode.hypo->GetScore() - prevHypo->GetScore());
|
<< " transition=" << (searchNode.hypo->GetScore() - prevHypo->GetScore());
|
||||||
|
|
||||||
if (searchNode.recombinationHypo != NULL)
|
if (searchNode.recombinationHypo != NULL)
|
||||||
out << " recombined=" << searchNode.recombinationHypo->GetId();
|
out << " recombined=" << searchNode.recombinationHypo->GetId();
|
||||||
|
|
||||||
@ -1132,7 +1132,7 @@ OutputSearchNode(AllOptions const& opts, long translationId,
|
|||||||
|
|
||||||
if (searchNode.recombinationHypo != NULL)
|
if (searchNode.recombinationHypo != NULL)
|
||||||
out << " recombined=" << searchNode.recombinationHypo->GetId();
|
out << " recombined=" << searchNode.recombinationHypo->GetId();
|
||||||
|
|
||||||
out << " forward=" << searchNode.forward << " fscore=" << searchNode.fscore
|
out << " forward=" << searchNode.forward << " fscore=" << searchNode.fscore
|
||||||
<< " covered=" << searchNode.hypo->GetCurrSourceWordsRange().GetStartPos()
|
<< " covered=" << searchNode.hypo->GetCurrSourceWordsRange().GetStartPos()
|
||||||
<< "-" << searchNode.hypo->GetCurrSourceWordsRange().GetEndPos();
|
<< "-" << searchNode.hypo->GetCurrSourceWordsRange().GetEndPos();
|
||||||
@ -1141,7 +1141,7 @@ OutputSearchNode(AllOptions const& opts, long translationId,
|
|||||||
ScoreComponentCollection scoreBreakdown = searchNode.hypo->GetScoreBreakdown();
|
ScoreComponentCollection scoreBreakdown = searchNode.hypo->GetScoreBreakdown();
|
||||||
scoreBreakdown.MinusEquals( prevHypo->GetScoreBreakdown() );
|
scoreBreakdown.MinusEquals( prevHypo->GetScoreBreakdown() );
|
||||||
out << " scores=\"" << scoreBreakdown << "\""
|
out << " scores=\"" << scoreBreakdown << "\""
|
||||||
<< " out=\"" << searchNode.hypo->GetSourcePhraseStringRep()
|
<< " out=\"" << searchNode.hypo->GetSourcePhraseStringRep()
|
||||||
<< "|" << searchNode.hypo->GetCurrTargetPhrase().GetStringRep(outputFactorOrder) << "\"" << endl;
|
<< "|" << searchNode.hypo->GetCurrTargetPhrase().GetStringRep(outputFactorOrder) << "\"" << endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1153,8 +1153,8 @@ void Manager::GetConnectedGraph(
|
|||||||
std::vector< const Hypothesis *>& connectedList = *pConnectedList;
|
std::vector< const Hypothesis *>& connectedList = *pConnectedList;
|
||||||
|
|
||||||
// start with the ones in the final stack
|
// start with the ones in the final stack
|
||||||
const std::vector < HypothesisStack* > &hypoStackColl
|
const std::vector < HypothesisStack* > &hypoStackColl
|
||||||
= m_search->GetHypothesisStacks();
|
= m_search->GetHypothesisStacks();
|
||||||
const HypothesisStack &finalStack = *hypoStackColl.back();
|
const HypothesisStack &finalStack = *hypoStackColl.back();
|
||||||
HypothesisStack::const_iterator iterHypo;
|
HypothesisStack::const_iterator iterHypo;
|
||||||
for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
|
for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
|
||||||
@ -1620,7 +1620,7 @@ OutputNBest(std::ostream& out, Moses::TrellisPathList const& nBestList) const
|
|||||||
out << m_source.GetTranslationId() << " ||| ";
|
out << m_source.GetTranslationId() << " ||| ";
|
||||||
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
|
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
|
||||||
const Hypothesis &edge = *edges[currEdge];
|
const Hypothesis &edge = *edges[currEdge];
|
||||||
OutputSurface(out, edge);
|
OutputSurface(out, edge);
|
||||||
}
|
}
|
||||||
out << " |||";
|
out << " |||";
|
||||||
|
|
||||||
@ -1853,11 +1853,11 @@ void Manager::OutputAlignment(OutputCollector *collector) const
|
|||||||
}
|
}
|
||||||
out << std::endl; // Used by --alignment-output-file so requires endl
|
out << std::endl; // Used by --alignment-output-file so requires endl
|
||||||
collector->Write(m_source.GetTranslationId(), out.str());
|
collector->Write(m_source.GetTranslationId(), out.str());
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputDetailedTranslationReport(OutputCollector *collector) const
|
OutputDetailedTranslationReport(OutputCollector *collector) const
|
||||||
{
|
{
|
||||||
@ -1870,7 +1870,7 @@ OutputDetailedTranslationReport(OutputCollector *collector) const
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputUnknowns(OutputCollector *collector) const
|
OutputUnknowns(OutputCollector *collector) const
|
||||||
{
|
{
|
||||||
@ -1887,7 +1887,7 @@ OutputUnknowns(OutputCollector *collector) const
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputWordGraph(OutputCollector *collector) const
|
OutputWordGraph(OutputCollector *collector) const
|
||||||
{
|
{
|
||||||
@ -1900,7 +1900,7 @@ OutputWordGraph(OutputCollector *collector) const
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputSearchGraph(OutputCollector *collector) const
|
OutputSearchGraph(OutputCollector *collector) const
|
||||||
{
|
{
|
||||||
@ -1976,7 +1976,7 @@ void Manager::OutputLatticeMBRNBest(std::ostream& out, const vector<LatticeMBRSo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputBestHypo(const std::vector<Word>& mbrBestHypo, ostream& out) const
|
OutputBestHypo(const std::vector<Word>& mbrBestHypo, ostream& out) const
|
||||||
{
|
{
|
||||||
@ -2008,7 +2008,7 @@ OutputAlignment(std::ostringstream &out, const TrellisPath &path) const
|
|||||||
{
|
{
|
||||||
WordAlignmentSort waso = options().output.WA_SortOrder;
|
WordAlignmentSort waso = options().output.WA_SortOrder;
|
||||||
BOOST_REVERSE_FOREACH(Hypothesis const* e, path.GetEdges())
|
BOOST_REVERSE_FOREACH(Hypothesis const* e, path.GetEdges())
|
||||||
e->OutputAlignment(out, false);
|
e->OutputAlignment(out, false);
|
||||||
// Hypothesis::OutputAlignment(out, path.GetEdges(), waso);
|
// Hypothesis::OutputAlignment(out, path.GetEdges(), waso);
|
||||||
// Used by --alignment-output-file so requires endl
|
// Used by --alignment-output-file so requires endl
|
||||||
out << std::endl;
|
out << std::endl;
|
||||||
|
@ -133,16 +133,16 @@ protected:
|
|||||||
mutable std::ostringstream m_alignmentOut;
|
mutable std::ostringstream m_alignmentOut;
|
||||||
public:
|
public:
|
||||||
void OutputNBest(std::ostream& out, const Moses::TrellisPathList &nBestList) const;
|
void OutputNBest(std::ostream& out, const Moses::TrellisPathList &nBestList) const;
|
||||||
void OutputSurface(std::ostream &out,
|
void OutputSurface(std::ostream &out,
|
||||||
Hypothesis const& edge,
|
Hypothesis const& edge,
|
||||||
bool const recursive=false) const;
|
bool const recursive=false) const;
|
||||||
|
|
||||||
void OutputAlignment(std::ostream &out, const AlignmentInfo &ai, size_t sourceOffset, size_t targetOffset) const;
|
void OutputAlignment(std::ostream &out, const AlignmentInfo &ai, size_t sourceOffset, size_t targetOffset) const;
|
||||||
void OutputInput(std::ostream& os, const Hypothesis* hypo) const;
|
void OutputInput(std::ostream& os, const Hypothesis* hypo) const;
|
||||||
void OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo) const;
|
void OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo) const;
|
||||||
void OutputPassthroughInformation(std::ostream& os, const Hypothesis* hypo) const;
|
void OutputPassthroughInformation(std::ostream& os, const Hypothesis* hypo) const;
|
||||||
|
|
||||||
std::map<size_t, const Factor*>
|
std::map<size_t, const Factor*>
|
||||||
GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor) const;
|
GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor) const;
|
||||||
|
|
||||||
void OutputWordGraph(std::ostream &outputWordGraphStream, const Hypothesis *hypo, size_t &linkId) const;
|
void OutputWordGraph(std::ostream &outputWordGraphStream, const Hypothesis *hypo, size_t &linkId) const;
|
||||||
@ -192,11 +192,11 @@ public:
|
|||||||
/***
|
/***
|
||||||
*For Lattice MBR
|
*For Lattice MBR
|
||||||
*/
|
*/
|
||||||
void
|
void
|
||||||
GetForwardBackwardSearchGraph
|
GetForwardBackwardSearchGraph
|
||||||
( std::map< int, bool >* pConnected,
|
( std::map< int, bool >* pConnected,
|
||||||
std::vector< const Hypothesis* >* pConnectedList,
|
std::vector< const Hypothesis* >* pConnectedList,
|
||||||
std::map < const Hypothesis*, std::set < const Hypothesis* > >* pOutgoingHyps,
|
std::map < const Hypothesis*, std::set < const Hypothesis* > >* pOutgoingHyps,
|
||||||
std::vector< float>* pFwdBwdScores) const;
|
std::vector< float>* pFwdBwdScores) const;
|
||||||
|
|
||||||
// outputs
|
// outputs
|
||||||
|
@ -65,7 +65,7 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
PartialTranslOptColl(size_t const maxSize);
|
PartialTranslOptColl(size_t const maxSize);
|
||||||
|
|
||||||
/** destructor, cleans out list */
|
/** destructor, cleans out list */
|
||||||
~PartialTranslOptColl() {
|
~PartialTranslOptColl() {
|
||||||
RemoveAllInColl( m_list );
|
RemoveAllInColl( m_list );
|
||||||
|
@ -271,9 +271,9 @@ CreateTranslationOptionCollection(ttasksptr const& ttask) const
|
|||||||
float transOptThreshold = ttask->options().search.trans_opt_threshold;
|
float transOptThreshold = ttask->options().search.trans_opt_threshold;
|
||||||
// StaticData::Instance().GetTranslationOptionThreshold();
|
// StaticData::Instance().GetTranslationOptionThreshold();
|
||||||
TranslationOptionCollection *rv
|
TranslationOptionCollection *rv
|
||||||
= new TranslationOptionCollectionText(ttask, *this,
|
= new TranslationOptionCollectionText(ttask, *this,
|
||||||
maxNoTransOptPerCoverage,
|
maxNoTransOptPerCoverage,
|
||||||
transOptThreshold);
|
transOptThreshold);
|
||||||
assert(rv);
|
assert(rv);
|
||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
|
@ -63,7 +63,7 @@ StaticData StaticData::s_instance;
|
|||||||
StaticData::StaticData()
|
StaticData::StaticData()
|
||||||
: m_sourceStartPosMattersForRecombination(false)
|
: m_sourceStartPosMattersForRecombination(false)
|
||||||
, m_requireSortingAfterSourceContext(false)
|
, m_requireSortingAfterSourceContext(false)
|
||||||
// , m_isAlwaysCreateDirectTranslationOption(false)
|
// , m_isAlwaysCreateDirectTranslationOption(false)
|
||||||
, m_currentWeightSetting("default")
|
, m_currentWeightSetting("default")
|
||||||
, m_treeStructure(NULL)
|
, m_treeStructure(NULL)
|
||||||
{
|
{
|
||||||
|
@ -229,63 +229,63 @@ public:
|
|||||||
return m_options;
|
return m_options;
|
||||||
}
|
}
|
||||||
|
|
||||||
inline bool
|
inline bool
|
||||||
GetSourceStartPosMattersForRecombination() const {
|
GetSourceStartPosMattersForRecombination() const {
|
||||||
return m_sourceStartPosMattersForRecombination;
|
return m_sourceStartPosMattersForRecombination;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool
|
bool
|
||||||
IsWordDeletionEnabled() const {
|
IsWordDeletionEnabled() const {
|
||||||
// return m_wordDeletionEnabled;
|
// return m_wordDeletionEnabled;
|
||||||
return m_options.unk.word_deletion_enabled;
|
return m_options.unk.word_deletion_enabled;
|
||||||
}
|
}
|
||||||
|
|
||||||
int
|
int
|
||||||
GetMaxDistortion() const {
|
GetMaxDistortion() const {
|
||||||
return m_options.reordering.max_distortion;
|
return m_options.reordering.max_distortion;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool
|
bool
|
||||||
UseReorderingConstraint() const {
|
UseReorderingConstraint() const {
|
||||||
return m_reorderingConstraint;
|
return m_reorderingConstraint;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool
|
bool
|
||||||
UseEarlyDiscarding() const {
|
UseEarlyDiscarding() const {
|
||||||
return m_options.search.early_discarding_threshold
|
return m_options.search.early_discarding_threshold
|
||||||
!= -std::numeric_limits<float>::infinity();
|
!= -std::numeric_limits<float>::infinity();
|
||||||
}
|
}
|
||||||
|
|
||||||
bool
|
bool
|
||||||
UseEarlyDistortionCost() const {
|
UseEarlyDistortionCost() const {
|
||||||
return m_options.reordering.use_early_distortion_cost;
|
return m_options.reordering.use_early_distortion_cost;
|
||||||
}
|
}
|
||||||
|
|
||||||
float
|
float
|
||||||
GetTranslationOptionThreshold() const {
|
GetTranslationOptionThreshold() const {
|
||||||
return m_options.search.trans_opt_threshold;
|
return m_options.search.trans_opt_threshold;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t
|
size_t
|
||||||
GetVerboseLevel() const {
|
GetVerboseLevel() const {
|
||||||
return m_verboseLevel;
|
return m_verboseLevel;
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
SetVerboseLevel(int x) const {
|
SetVerboseLevel(int x) const {
|
||||||
m_verboseLevel = x;
|
m_verboseLevel = x;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool
|
bool
|
||||||
UseMinphrInMemory() const {
|
UseMinphrInMemory() const {
|
||||||
return m_minphrMemory;
|
return m_minphrMemory;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool
|
bool
|
||||||
UseMinlexrInMemory() const {
|
UseMinlexrInMemory() const {
|
||||||
return m_minlexrMemory;
|
return m_minlexrMemory;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool IsSyntax(SearchAlgorithm algo = DefaultSearchAlgorithm) const {
|
bool IsSyntax(SearchAlgorithm algo = DefaultSearchAlgorithm) const {
|
||||||
if (algo == DefaultSearchAlgorithm)
|
if (algo == DefaultSearchAlgorithm)
|
||||||
algo = m_options.search.algo;
|
algo = m_options.search.algo;
|
||||||
|
@ -40,7 +40,7 @@ namespace F2S
|
|||||||
{
|
{
|
||||||
|
|
||||||
bool HyperTreeLoader::Load(AllOptions const& opts,
|
bool HyperTreeLoader::Load(AllOptions const& opts,
|
||||||
const std::vector<FactorType> &input,
|
const std::vector<FactorType> &input,
|
||||||
const std::vector<FactorType> &output,
|
const std::vector<FactorType> &output,
|
||||||
const std::string &inFile,
|
const std::string &inFile,
|
||||||
const RuleTableFF &ff,
|
const RuleTableFF &ff,
|
||||||
|
@ -24,7 +24,7 @@ class HyperTreeLoader : public HyperTreeCreator
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
bool Load(AllOptions const& opts,
|
bool Load(AllOptions const& opts,
|
||||||
const std::vector<FactorType> &input,
|
const std::vector<FactorType> &input,
|
||||||
const std::vector<FactorType> &output,
|
const std::vector<FactorType> &output,
|
||||||
const std::string &inFile,
|
const std::string &inFile,
|
||||||
const RuleTableFF &,
|
const RuleTableFF &,
|
||||||
|
@ -74,9 +74,9 @@ void Manager<RuleMatcher>::Decode()
|
|||||||
RuleMatcherCallback callback(m_stackMap, ruleLimit);
|
RuleMatcherCallback callback(m_stackMap, ruleLimit);
|
||||||
|
|
||||||
// Create a glue rule synthesizer.
|
// Create a glue rule synthesizer.
|
||||||
GlueRuleSynthesizer glueRuleSynthesizer(*m_glueRuleTrie,
|
GlueRuleSynthesizer glueRuleSynthesizer(*m_glueRuleTrie,
|
||||||
options().input.factor_order);
|
options().input.factor_order);
|
||||||
|
|
||||||
// Sort the input forest's vertices into bottom-up topological order.
|
// Sort the input forest's vertices into bottom-up topological order.
|
||||||
std::vector<const Forest::Vertex *> sortedVertices;
|
std::vector<const Forest::Vertex *> sortedVertices;
|
||||||
TopologicalSorter sorter;
|
TopologicalSorter sorter;
|
||||||
|
@ -37,7 +37,7 @@ namespace S2T
|
|||||||
{
|
{
|
||||||
|
|
||||||
bool RuleTrieLoader::Load(Moses::AllOptions const& opts,
|
bool RuleTrieLoader::Load(Moses::AllOptions const& opts,
|
||||||
const std::vector<FactorType> &input,
|
const std::vector<FactorType> &input,
|
||||||
const std::vector<FactorType> &output,
|
const std::vector<FactorType> &output,
|
||||||
const std::string &inFile,
|
const std::string &inFile,
|
||||||
const RuleTableFF &ff,
|
const RuleTableFF &ff,
|
||||||
|
@ -22,7 +22,7 @@ class RuleTrieLoader : public RuleTrieCreator
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
bool Load(Moses::AllOptions const& opts,
|
bool Load(Moses::AllOptions const& opts,
|
||||||
const std::vector<FactorType> &input,
|
const std::vector<FactorType> &input,
|
||||||
const std::vector<FactorType> &output,
|
const std::vector<FactorType> &output,
|
||||||
const std::string &inFile,
|
const std::string &inFile,
|
||||||
const RuleTableFF &,
|
const RuleTableFF &,
|
||||||
|
@ -36,7 +36,7 @@ namespace T2S
|
|||||||
{
|
{
|
||||||
|
|
||||||
bool RuleTrieLoader::Load(Moses::AllOptions const& opts,
|
bool RuleTrieLoader::Load(Moses::AllOptions const& opts,
|
||||||
const std::vector<FactorType> &input,
|
const std::vector<FactorType> &input,
|
||||||
const std::vector<FactorType> &output,
|
const std::vector<FactorType> &output,
|
||||||
const std::string &inFile,
|
const std::string &inFile,
|
||||||
const RuleTableFF &ff,
|
const RuleTableFF &ff,
|
||||||
|
@ -21,7 +21,7 @@ class RuleTrieLoader : public RuleTrieCreator
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
bool Load(Moses::AllOptions const& opts,
|
bool Load(Moses::AllOptions const& opts,
|
||||||
const std::vector<FactorType> &input,
|
const std::vector<FactorType> &input,
|
||||||
const std::vector<FactorType> &output,
|
const std::vector<FactorType> &output,
|
||||||
const std::string &inFile,
|
const std::string &inFile,
|
||||||
const RuleTableFF &,
|
const RuleTableFF &,
|
||||||
|
@ -336,8 +336,8 @@ void PhraseDictionaryDynamicCacheBased::ClearEntries(std::string sourcePhraseStr
|
|||||||
//target
|
//target
|
||||||
targetPhrase.Clear();
|
targetPhrase.Clear();
|
||||||
VERBOSE(3, "targetPhraseString:|" << targetPhraseString << "|" << std::endl);
|
VERBOSE(3, "targetPhraseString:|" << targetPhraseString << "|" << std::endl);
|
||||||
targetPhrase.CreateFromString(Output, staticData.options().output.factor_order,
|
targetPhrase.CreateFromString(Output, staticData.options().output.factor_order,
|
||||||
targetPhraseString, /*factorDelimiter,*/ NULL);
|
targetPhraseString, /*factorDelimiter,*/ NULL);
|
||||||
VERBOSE(2, "targetPhrase:|" << targetPhrase << "|" << std::endl);
|
VERBOSE(2, "targetPhrase:|" << targetPhrase << "|" << std::endl);
|
||||||
|
|
||||||
//TODO: Would be better to reuse source phrases, but ownership has to be
|
//TODO: Would be better to reuse source phrases, but ownership has to be
|
||||||
@ -345,7 +345,7 @@ void PhraseDictionaryDynamicCacheBased::ClearEntries(std::string sourcePhraseStr
|
|||||||
sourcePhrase.Clear();
|
sourcePhrase.Clear();
|
||||||
VERBOSE(3, "sourcePhraseString:|" << sourcePhraseString << "|" << std::endl);
|
VERBOSE(3, "sourcePhraseString:|" << sourcePhraseString << "|" << std::endl);
|
||||||
sourcePhrase.CreateFromString(Input, staticData.options().input.factor_order,
|
sourcePhrase.CreateFromString(Input, staticData.options().input.factor_order,
|
||||||
sourcePhraseString, /*factorDelimiter,*/ NULL);
|
sourcePhraseString, /*factorDelimiter,*/ NULL);
|
||||||
VERBOSE(3, "sourcePhrase:|" << sourcePhrase << "|" << std::endl);
|
VERBOSE(3, "sourcePhrase:|" << sourcePhrase << "|" << std::endl);
|
||||||
ClearEntries(sourcePhrase, targetPhrase);
|
ClearEntries(sourcePhrase, targetPhrase);
|
||||||
|
|
||||||
@ -435,8 +435,8 @@ void PhraseDictionaryDynamicCacheBased::ClearSource(std::vector<std::string> ent
|
|||||||
|
|
||||||
sourcePhrase.Clear();
|
sourcePhrase.Clear();
|
||||||
VERBOSE(3, "sourcePhraseString:|" << (*it) << "|" << std::endl);
|
VERBOSE(3, "sourcePhraseString:|" << (*it) << "|" << std::endl);
|
||||||
sourcePhrase.CreateFromString(Input, staticData.options().input.factor_order,
|
sourcePhrase.CreateFromString(Input, staticData.options().input.factor_order,
|
||||||
*it, /*factorDelimiter,*/ NULL);
|
*it, /*factorDelimiter,*/ NULL);
|
||||||
VERBOSE(3, "sourcePhrase:|" << sourcePhrase << "|" << std::endl);
|
VERBOSE(3, "sourcePhrase:|" << sourcePhrase << "|" << std::endl);
|
||||||
|
|
||||||
ClearSource(sourcePhrase);
|
ClearSource(sourcePhrase);
|
||||||
@ -529,10 +529,10 @@ void PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, s
|
|||||||
//target
|
//target
|
||||||
targetPhrase.Clear();
|
targetPhrase.Clear();
|
||||||
VERBOSE(3, "targetPhraseString:|" << targetPhraseString << "|" << std::endl);
|
VERBOSE(3, "targetPhraseString:|" << targetPhraseString << "|" << std::endl);
|
||||||
targetPhrase.CreateFromString(Output, staticData.options().output.factor_order,
|
targetPhrase.CreateFromString(Output, staticData.options().output.factor_order,
|
||||||
targetPhraseString, /*factorDelimiter,*/ NULL);
|
targetPhraseString, /*factorDelimiter,*/ NULL);
|
||||||
VERBOSE(3, "targetPhrase:|" << targetPhrase << "|" << std::endl);
|
VERBOSE(3, "targetPhrase:|" << targetPhrase << "|" << std::endl);
|
||||||
|
|
||||||
//TODO: Would be better to reuse source phrases, but ownership has to be
|
//TODO: Would be better to reuse source phrases, but ownership has to be
|
||||||
//consistent across phrase table implementations
|
//consistent across phrase table implementations
|
||||||
sourcePhrase.Clear();
|
sourcePhrase.Clear();
|
||||||
|
@ -37,7 +37,7 @@ public:
|
|||||||
virtual ~RuleTableLoader() {}
|
virtual ~RuleTableLoader() {}
|
||||||
|
|
||||||
virtual bool Load(AllOptions const& opts,
|
virtual bool Load(AllOptions const& opts,
|
||||||
const std::vector<FactorType> &input,
|
const std::vector<FactorType> &input,
|
||||||
const std::vector<FactorType> &output,
|
const std::vector<FactorType> &output,
|
||||||
const std::string &inFile,
|
const std::string &inFile,
|
||||||
size_t tableLimit,
|
size_t tableLimit,
|
||||||
|
@ -32,8 +32,8 @@
|
|||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
|
|
||||||
bool RuleTableLoaderCompact::Load(AllOptions const& opts,
|
bool RuleTableLoaderCompact::Load(AllOptions const& opts,
|
||||||
const std::vector<FactorType> &input,
|
const std::vector<FactorType> &input,
|
||||||
const std::vector<FactorType> &output,
|
const std::vector<FactorType> &output,
|
||||||
const std::string &inFile,
|
const std::string &inFile,
|
||||||
size_t /* tableLimit */,
|
size_t /* tableLimit */,
|
||||||
|
@ -37,7 +37,7 @@ class RuleTableLoaderCompact : public RuleTableLoader
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
bool Load(AllOptions const& opts,
|
bool Load(AllOptions const& opts,
|
||||||
const std::vector<FactorType> &input,
|
const std::vector<FactorType> &input,
|
||||||
const std::vector<FactorType> &output,
|
const std::vector<FactorType> &output,
|
||||||
const std::string &inFile,
|
const std::string &inFile,
|
||||||
size_t tableLimit,
|
size_t tableLimit,
|
||||||
|
@ -35,7 +35,7 @@ namespace Moses
|
|||||||
|
|
||||||
// Determines the rule table type by peeking inside the file then creates
|
// Determines the rule table type by peeking inside the file then creates
|
||||||
// a suitable RuleTableLoader object.
|
// a suitable RuleTableLoader object.
|
||||||
std::auto_ptr<RuleTableLoader>
|
std::auto_ptr<RuleTableLoader>
|
||||||
RuleTableLoaderFactory::
|
RuleTableLoaderFactory::
|
||||||
Create(const std::string &path)
|
Create(const std::string &path)
|
||||||
{
|
{
|
||||||
|
@ -15,7 +15,7 @@ namespace Moses
|
|||||||
{
|
{
|
||||||
|
|
||||||
bool RuleTableLoaderHiero::Load(AllOptions const& opts,
|
bool RuleTableLoaderHiero::Load(AllOptions const& opts,
|
||||||
const std::vector<FactorType> &input,
|
const std::vector<FactorType> &input,
|
||||||
const std::vector<FactorType> &output,
|
const std::vector<FactorType> &output,
|
||||||
const std::string &inFile,
|
const std::string &inFile,
|
||||||
size_t tableLimit,
|
size_t tableLimit,
|
||||||
|
@ -19,7 +19,7 @@ class RuleTableLoaderHiero : public RuleTableLoaderStandard
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
bool Load(AllOptions const& opts,
|
bool Load(AllOptions const& opts,
|
||||||
const std::vector<FactorType> &input,
|
const std::vector<FactorType> &input,
|
||||||
const std::vector<FactorType> &output,
|
const std::vector<FactorType> &output,
|
||||||
const std::string &inFile,
|
const std::string &inFile,
|
||||||
size_t tableLimit,
|
size_t tableLimit,
|
||||||
|
@ -47,8 +47,8 @@ using namespace boost::algorithm;
|
|||||||
|
|
||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
|
|
||||||
bool
|
bool
|
||||||
RuleTableLoaderStandard::
|
RuleTableLoaderStandard::
|
||||||
Load(AllOptions const& opts
|
Load(AllOptions const& opts
|
||||||
, const std::vector<FactorType> &input
|
, const std::vector<FactorType> &input
|
||||||
|
@ -30,7 +30,7 @@ class RuleTableLoaderStandard : public RuleTableLoader
|
|||||||
protected:
|
protected:
|
||||||
|
|
||||||
bool Load(AllOptions const& opts,
|
bool Load(AllOptions const& opts,
|
||||||
FormatType format,
|
FormatType format,
|
||||||
const std::vector<FactorType> &input,
|
const std::vector<FactorType> &input,
|
||||||
const std::vector<FactorType> &output,
|
const std::vector<FactorType> &output,
|
||||||
const std::string &inFile,
|
const std::string &inFile,
|
||||||
@ -38,7 +38,7 @@ protected:
|
|||||||
RuleTableTrie &);
|
RuleTableTrie &);
|
||||||
public:
|
public:
|
||||||
bool Load(AllOptions const& opts,
|
bool Load(AllOptions const& opts,
|
||||||
const std::vector<FactorType> &input,
|
const std::vector<FactorType> &input,
|
||||||
const std::vector<FactorType> &output,
|
const std::vector<FactorType> &output,
|
||||||
const std::string &inFile,
|
const std::string &inFile,
|
||||||
size_t tableLimit,
|
size_t tableLimit,
|
||||||
|
@ -50,8 +50,8 @@ void PhraseDictionaryALSuffixArray::InitializeForInput(ttasksptr const& ttask)
|
|||||||
AllOptions const& opts = ttask->options();
|
AllOptions const& opts = ttask->options();
|
||||||
bool ret = loader->Load(opts, m_input, m_output, grammarFile, m_tableLimit, *this);
|
bool ret = loader->Load(opts, m_input, m_output, grammarFile, m_tableLimit, *this);
|
||||||
|
|
||||||
UTIL_THROW_IF2(!ret, "Rules not successfully loaded for sentence id "
|
UTIL_THROW_IF2(!ret, "Rules not successfully loaded for sentence id "
|
||||||
<< translationId);
|
<< translationId);
|
||||||
}
|
}
|
||||||
|
|
||||||
void PhraseDictionaryALSuffixArray::CleanUpAfterSentenceProcessing(const InputType &source)
|
void PhraseDictionaryALSuffixArray::CleanUpAfterSentenceProcessing(const InputType &source)
|
||||||
|
@ -218,13 +218,13 @@ CreateTranslationOptionsForRangeNew
|
|||||||
|
|
||||||
bool
|
bool
|
||||||
TranslationOptionCollectionConfusionNet::
|
TranslationOptionCollectionConfusionNet::
|
||||||
CreateTranslationOptionsForRangeLEGACY(const DecodeGraph &decodeGraph,
|
CreateTranslationOptionsForRangeLEGACY(const DecodeGraph &decodeGraph,
|
||||||
size_t startPos, size_t endPos,
|
size_t startPos, size_t endPos,
|
||||||
bool adhereTableLimit, size_t graphInd)
|
bool adhereTableLimit, size_t graphInd)
|
||||||
{
|
{
|
||||||
bool retval = true;
|
bool retval = true;
|
||||||
size_t const max_phrase_length
|
size_t const max_phrase_length
|
||||||
= StaticData::Instance().options().search.max_phrase_length;
|
= StaticData::Instance().options().search.max_phrase_length;
|
||||||
XmlInputType intype = m_ttask.lock()->options().input.xml_policy;
|
XmlInputType intype = m_ttask.lock()->options().input.xml_policy;
|
||||||
if ((intype != XmlExclusive) || !HasXmlOptionsOverlappingRange(startPos,endPos)) {
|
if ((intype != XmlExclusive) || !HasXmlOptionsOverlappingRange(startPos,endPos)) {
|
||||||
InputPathList &inputPathList = GetInputPathList(startPos, endPos);
|
InputPathList &inputPathList = GetInputPathList(startPos, endPos);
|
||||||
|
@ -70,14 +70,14 @@ TranslationOptionCollectionLattice
|
|||||||
|
|
||||||
// recursive
|
// recursive
|
||||||
Extend(*path, input, ttask->options().search.max_phrase_length);
|
Extend(*path, input, ttask->options().search.max_phrase_length);
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
TranslationOptionCollectionLattice::
|
TranslationOptionCollectionLattice::
|
||||||
Extend(const InputPath &prevPath, const WordLattice &input,
|
Extend(const InputPath &prevPath, const WordLattice &input,
|
||||||
size_t const maxPhraseLength)
|
size_t const maxPhraseLength)
|
||||||
{
|
{
|
||||||
size_t nextPos = prevPath.GetWordsRange().GetEndPos() + 1;
|
size_t nextPos = prevPath.GetWordsRange().GetEndPos() + 1;
|
||||||
|
@ -33,7 +33,7 @@ public:
|
|||||||
|
|
||||||
protected:
|
protected:
|
||||||
void Extend(const InputPath &prevPath, const WordLattice &input,
|
void Extend(const InputPath &prevPath, const WordLattice &input,
|
||||||
size_t const maxPhraseLength);
|
size_t const maxPhraseLength);
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -169,15 +169,15 @@ TrellisPath::
|
|||||||
GetScoreBreakdown() const
|
GetScoreBreakdown() const
|
||||||
{
|
{
|
||||||
if (!m_scoreBreakdown) {
|
if (!m_scoreBreakdown) {
|
||||||
float totalScore = m_path[0]->GetWinningHypo()->GetFutureScore();
|
float totalScore = m_path[0]->GetWinningHypo()->GetFutureScore();
|
||||||
// calculated for sanity check only
|
// calculated for sanity check only
|
||||||
|
|
||||||
m_scoreBreakdown.reset(new ScoreComponentCollection());
|
m_scoreBreakdown.reset(new ScoreComponentCollection());
|
||||||
m_scoreBreakdown->PlusEquals(m_path[0]->GetWinningHypo()->GetScoreBreakdown());
|
m_scoreBreakdown->PlusEquals(m_path[0]->GetWinningHypo()->GetScoreBreakdown());
|
||||||
|
|
||||||
// adjust score
|
// adjust score
|
||||||
// I assume things are done this way on the assumption that most hypothesis edges
|
// I assume things are done this way on the assumption that most hypothesis edges
|
||||||
// are shared with the winning path, so that score adjustments are cheaper than
|
// are shared with the winning path, so that score adjustments are cheaper than
|
||||||
// recomputing the score from scratch. UG
|
// recomputing the score from scratch. UG
|
||||||
size_t sizePath = m_path.size();
|
size_t sizePath = m_path.size();
|
||||||
for (size_t pos = 0 ; pos < sizePath ; pos++) {
|
for (size_t pos = 0 ; pos < sizePath ; pos++) {
|
||||||
|
@ -163,8 +163,8 @@ Read(std::istream& in,
|
|||||||
}
|
}
|
||||||
|
|
||||||
PCN::CN cn = PCN::parsePCN(line);
|
PCN::CN cn = PCN::parsePCN(line);
|
||||||
return InitializeFromPCNDataType(cn, opts.search.max_phrase_length,
|
return InitializeFromPCNDataType(cn, opts.search.max_phrase_length,
|
||||||
factorOrder, line);
|
factorOrder, line);
|
||||||
}
|
}
|
||||||
|
|
||||||
void WordLattice::GetAsEdgeMatrix(std::vector<std::vector<bool> >& edges) const
|
void WordLattice::GetAsEdgeMatrix(std::vector<std::vector<bool> >& edges) const
|
||||||
@ -231,10 +231,10 @@ WordLattice
|
|||||||
// size_t maxNoTransOptPerCoverage = StaticData::Instance().GetMaxNoTransOptPerCoverage();
|
// size_t maxNoTransOptPerCoverage = StaticData::Instance().GetMaxNoTransOptPerCoverage();
|
||||||
// float translationOptionThreshold = StaticData::Instance().GetTranslationOptionThreshold();
|
// float translationOptionThreshold = StaticData::Instance().GetTranslationOptionThreshold();
|
||||||
|
|
||||||
size_t maxNoTransOptPerCoverage = ttask->options().search.max_trans_opt_per_cov;
|
size_t maxNoTransOptPerCoverage = ttask->options().search.max_trans_opt_per_cov;
|
||||||
// StaticData::Instance().GetMaxNoTransOptPerCoverage();
|
// StaticData::Instance().GetMaxNoTransOptPerCoverage();
|
||||||
float translationOptionThreshold = ttask->options().search.trans_opt_threshold;
|
float translationOptionThreshold = ttask->options().search.trans_opt_threshold;
|
||||||
// StaticData::Instance().GetTranslationOptionThreshold();
|
// StaticData::Instance().GetTranslationOptionThreshold();
|
||||||
|
|
||||||
|
|
||||||
TranslationOptionCollection *rv = NULL;
|
TranslationOptionCollection *rv = NULL;
|
||||||
|
@ -40,9 +40,9 @@ public:
|
|||||||
/** Given a lattice represented using the PCN::CN data type (topologically sorted agency list
|
/** Given a lattice represented using the PCN::CN data type (topologically sorted agency list
|
||||||
* representation), initialize the WordLattice object
|
* representation), initialize the WordLattice object
|
||||||
*/
|
*/
|
||||||
int InitializeFromPCNDataType(const PCN::CN& cn, size_t const maxPhraseLength,
|
int InitializeFromPCNDataType(const PCN::CN& cn, size_t const maxPhraseLength,
|
||||||
const std::vector<FactorType>& factorOrder,
|
const std::vector<FactorType>& factorOrder,
|
||||||
const std::string& debug_line = "");
|
const std::string& debug_line = "");
|
||||||
/** Read from PLF format (1 lattice per line)
|
/** Read from PLF format (1 lattice per line)
|
||||||
*/
|
*/
|
||||||
int Read(std::istream& in,
|
int Read(std::istream& in,
|
||||||
|
@ -119,7 +119,7 @@ const TrellisPath doMBR(const TrellisPathList& nBestList, AllOptions const& opts
|
|||||||
// get words in translation
|
// get words in translation
|
||||||
vector<const Factor*> translation;
|
vector<const Factor*> translation;
|
||||||
GetOutputFactors(path, oFactors[0], translation);
|
GetOutputFactors(path, oFactors[0], translation);
|
||||||
|
|
||||||
// collect n-gram counts
|
// collect n-gram counts
|
||||||
map < vector < const Factor *>, int > counts;
|
map < vector < const Factor *>, int > counts;
|
||||||
extract_ngrams(translation,counts);
|
extract_ngrams(translation,counts);
|
||||||
@ -158,9 +158,9 @@ const TrellisPath doMBR(const TrellisPathList& nBestList, AllOptions const& opts
|
|||||||
//return translations[minMBRLossIdx];
|
//return translations[minMBRLossIdx];
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
GetOutputFactors(const TrellisPath &path, FactorType const oFactor,
|
GetOutputFactors(const TrellisPath &path, FactorType const oFactor,
|
||||||
vector <const Factor*> &translation)
|
vector <const Factor*> &translation)
|
||||||
{
|
{
|
||||||
const std::vector<const Hypothesis *> &edges = path.GetEdges();
|
const std::vector<const Hypothesis *> &edges = path.GetEdges();
|
||||||
|
|
||||||
|
16
moses/mbr.h
16
moses/mbr.h
@ -26,14 +26,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
Moses::TrellisPath const
|
Moses::TrellisPath const
|
||||||
doMBR(Moses::TrellisPathList const& nBestList, Moses::AllOptions const& opts);
|
doMBR(Moses::TrellisPathList const& nBestList, Moses::AllOptions const& opts);
|
||||||
|
|
||||||
void
|
void
|
||||||
GetOutputFactors(const Moses::TrellisPath &path, Moses::FactorType const f,
|
GetOutputFactors(const Moses::TrellisPath &path, Moses::FactorType const f,
|
||||||
std::vector <const Moses::Factor*> &translation);
|
std::vector <const Moses::Factor*> &translation);
|
||||||
|
|
||||||
float
|
float
|
||||||
calculate_score(const std::vector< std::vector<const Moses::Factor*> > & sents,
|
calculate_score(const std::vector< std::vector<const Moses::Factor*> > & sents,
|
||||||
int ref, int hyp,
|
int ref, int hyp,
|
||||||
std::vector<std::map<std::vector<const Moses::Factor*>,int> > &
|
std::vector<std::map<std::vector<const Moses::Factor*>,int> > &
|
||||||
ngram_stats );
|
ngram_stats );
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
Loading…
Reference in New Issue
Block a user