mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-08-17 07:20:48 +03:00
daily automatic beautifier
This commit is contained in:
parent
8424fb6e2a
commit
f38976aba0
@ -202,8 +202,8 @@ int main(int argc, char* argv[])
|
|||||||
<< " ||| ";
|
<< " ||| ";
|
||||||
vector<Word> mbrBestHypo = doLatticeMBR(manager,nBestList);
|
vector<Word> mbrBestHypo = doLatticeMBR(manager,nBestList);
|
||||||
manager.OutputBestHypo(mbrBestHypo, lineCount,
|
manager.OutputBestHypo(mbrBestHypo, lineCount,
|
||||||
manager.options().output.ReportSegmentation,
|
manager.options().output.ReportSegmentation,
|
||||||
manager.options().output.ReportAllFactors,
|
manager.options().output.ReportAllFactors,
|
||||||
cout);
|
cout);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -106,9 +106,9 @@ std::set<size_t> AlignmentInfo::GetAlignmentsForTarget(size_t targetPos) const
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool
|
bool
|
||||||
compare_target(std::pair<size_t,size_t> const* a,
|
compare_target(std::pair<size_t,size_t> const* a,
|
||||||
std::pair<size_t,size_t> const* b)
|
std::pair<size_t,size_t> const* b)
|
||||||
{
|
{
|
||||||
if(a->second < b->second) return true;
|
if(a->second < b->second) return true;
|
||||||
if(a->second == b->second) return (a->first < b->first);
|
if(a->second == b->second) return (a->first < b->first);
|
||||||
@ -116,29 +116,29 @@ compare_target(std::pair<size_t,size_t> const* a,
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
std::vector< const std::pair<size_t,size_t>* >
|
std::vector< const std::pair<size_t,size_t>* >
|
||||||
AlignmentInfo::
|
AlignmentInfo::
|
||||||
GetSortedAlignments(WordAlignmentSort SortOrder) const
|
GetSortedAlignments(WordAlignmentSort SortOrder) const
|
||||||
{
|
{
|
||||||
std::vector< const std::pair<size_t,size_t>* > ret;
|
std::vector< const std::pair<size_t,size_t>* > ret;
|
||||||
|
|
||||||
CollType::const_iterator iter;
|
CollType::const_iterator iter;
|
||||||
for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
|
for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
|
||||||
const std::pair<size_t,size_t> &alignPair = *iter;
|
const std::pair<size_t,size_t> &alignPair = *iter;
|
||||||
ret.push_back(&alignPair);
|
ret.push_back(&alignPair);
|
||||||
}
|
}
|
||||||
|
|
||||||
switch (SortOrder) {
|
switch (SortOrder) {
|
||||||
case NoSort:
|
case NoSort:
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case TargetOrder:
|
case TargetOrder:
|
||||||
std::sort(ret.begin(), ret.end(), compare_target);
|
std::sort(ret.begin(), ret.end(), compare_target);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
UTIL_THROW(util::Exception, "Unknown word alignment sort option: "
|
UTIL_THROW(util::Exception, "Unknown word alignment sort option: "
|
||||||
<< SortOrder);
|
<< SortOrder);
|
||||||
}
|
}
|
||||||
|
|
||||||
return ret;
|
return ret;
|
||||||
|
@ -83,7 +83,7 @@ public:
|
|||||||
return m_collection.size();
|
return m_collection.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector< const std::pair<size_t,size_t>* >
|
std::vector< const std::pair<size_t,size_t>* >
|
||||||
GetSortedAlignments(WordAlignmentSort SortOrder) const;
|
GetSortedAlignments(WordAlignmentSort SortOrder) const;
|
||||||
|
|
||||||
std::vector<size_t> GetSourceIndex2PosMap() const;
|
std::vector<size_t> GetSourceIndex2PosMap() const;
|
||||||
|
@ -65,12 +65,11 @@ bool ChartCell::AddHypothesis(ChartHypothesis *hypo)
|
|||||||
{
|
{
|
||||||
const Word &targetLHS = hypo->GetTargetLHS();
|
const Word &targetLHS = hypo->GetTargetLHS();
|
||||||
MapType::iterator m = m_hypoColl.find(targetLHS);
|
MapType::iterator m = m_hypoColl.find(targetLHS);
|
||||||
if (m == m_hypoColl.end())
|
if (m == m_hypoColl.end()) {
|
||||||
{
|
std::pair<Word, ChartHypothesisCollection>
|
||||||
std::pair<Word, ChartHypothesisCollection>
|
e(targetLHS, ChartHypothesisCollection(m_manager.options()));
|
||||||
e(targetLHS, ChartHypothesisCollection(m_manager.options()));
|
m = m_hypoColl.insert(e).first;
|
||||||
m = m_hypoColl.insert(e).first;
|
}
|
||||||
}
|
|
||||||
return m->second.AddHypothesis(hypo, m_manager);
|
return m->second.AddHypothesis(hypo, m_manager);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -29,7 +29,7 @@ namespace Moses
|
|||||||
{
|
{
|
||||||
|
|
||||||
class ChartSearchGraphWriter;
|
class ChartSearchGraphWriter;
|
||||||
class AllOptions;
|
class AllOptions;
|
||||||
|
|
||||||
//! functor to compare (chart) hypotheses by (descending) score
|
//! functor to compare (chart) hypotheses by (descending) score
|
||||||
class ChartHypothesisScoreOrderer
|
class ChartHypothesisScoreOrderer
|
||||||
|
@ -108,12 +108,12 @@ void ChartParserUnknown::Process(const Word &sourceWord, const Range &range, Cha
|
|||||||
targetPhrase->EvaluateInIsolation(*unksrc);
|
targetPhrase->EvaluateInIsolation(*unksrc);
|
||||||
|
|
||||||
AllOptions const& opts = staticData.options();
|
AllOptions const& opts = staticData.options();
|
||||||
if (!opts.output.detailed_tree_transrep_filepath.empty() ||
|
if (!opts.output.detailed_tree_transrep_filepath.empty() ||
|
||||||
opts.nbest.print_trees || staticData.GetTreeStructure() != NULL) {
|
opts.nbest.print_trees || staticData.GetTreeStructure() != NULL) {
|
||||||
std::string prop = "[ ";
|
std::string prop = "[ ";
|
||||||
prop += (*targetLHS)[0]->GetString().as_string() + " ";
|
prop += (*targetLHS)[0]->GetString().as_string() + " ";
|
||||||
prop += sourceWord[0]->GetString().as_string() + " ]";
|
prop += sourceWord[0]->GetString().as_string() + " ]";
|
||||||
targetPhrase->SetProperty("Tree", prop);
|
targetPhrase->SetProperty("Tree", prop);
|
||||||
}
|
}
|
||||||
|
|
||||||
// chart rule
|
// chart rule
|
||||||
|
@ -68,7 +68,7 @@ public:
|
|||||||
virtual void Print(std::ostream&) const;
|
virtual void Print(std::ostream&) const;
|
||||||
|
|
||||||
int Read(std::istream& in,const std::vector<FactorType>& factorOrder,
|
int Read(std::istream& in,const std::vector<FactorType>& factorOrder,
|
||||||
AllOptions const& opts);
|
AllOptions const& opts);
|
||||||
|
|
||||||
Phrase GetSubString(const Range&) const; //TODO not defined
|
Phrase GetSubString(const Range&) const; //TODO not defined
|
||||||
std::string GetStringRep(const std::vector<FactorType> factorsToPrint) const; //TODO not defined
|
std::string GetStringRep(const std::vector<FactorType> factorsToPrint) const; //TODO not defined
|
||||||
|
@ -100,14 +100,14 @@ void DecodeStepTranslation::Process(const TranslationOption &inputPartialTranslO
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
DecodeStepTranslation::
|
DecodeStepTranslation::
|
||||||
ProcessInitialTranslation(InputType const& source,
|
ProcessInitialTranslation(InputType const& source,
|
||||||
PartialTranslOptColl &outputPartialTranslOptColl,
|
PartialTranslOptColl &outputPartialTranslOptColl,
|
||||||
size_t startPos, size_t endPos,
|
size_t startPos, size_t endPos,
|
||||||
bool adhereTableLimit,
|
bool adhereTableLimit,
|
||||||
InputPath const& inputPath,
|
InputPath const& inputPath,
|
||||||
TargetPhraseCollection::shared_ptr phraseColl) const
|
TargetPhraseCollection::shared_ptr phraseColl) const
|
||||||
{
|
{
|
||||||
const PhraseDictionary* phraseDictionary = GetPhraseDictionaryFeature();
|
const PhraseDictionary* phraseDictionary = GetPhraseDictionaryFeature();
|
||||||
const size_t tableLimit = phraseDictionary->GetTableLimit();
|
const size_t tableLimit = phraseDictionary->GetTableLimit();
|
||||||
@ -117,12 +117,12 @@ ProcessInitialTranslation(InputType const& source,
|
|||||||
if (phraseColl != NULL) {
|
if (phraseColl != NULL) {
|
||||||
IFVERBOSE(3) {
|
IFVERBOSE(3) {
|
||||||
if(source.GetType() == SentenceInput)
|
if(source.GetType() == SentenceInput)
|
||||||
TRACE_ERR("[" << source.GetSubString(range) << "; "
|
TRACE_ERR("[" << source.GetSubString(range) << "; "
|
||||||
<< startPos << "-" << endPos << "]\n");
|
<< startPos << "-" << endPos << "]\n");
|
||||||
else
|
else
|
||||||
TRACE_ERR("[" << startPos << "-" << endPos << "]" << std::endl);
|
TRACE_ERR("[" << startPos << "-" << endPos << "]" << std::endl);
|
||||||
}
|
}
|
||||||
|
|
||||||
TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
|
TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
|
||||||
iterEnd = (!adhereTableLimit || tableLimit == 0 || phraseColl->GetSize() < tableLimit) ? phraseColl->end() : phraseColl->begin() + tableLimit;
|
iterEnd = (!adhereTableLimit || tableLimit == 0 || phraseColl->GetSize() < tableLimit) ? phraseColl->end() : phraseColl->begin() + tableLimit;
|
||||||
|
|
||||||
@ -140,13 +140,13 @@ ProcessInitialTranslation(InputType const& source,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
DecodeStepTranslation::
|
DecodeStepTranslation::
|
||||||
ProcessInitialTransLEGACY(InputType const& source,
|
ProcessInitialTransLEGACY(InputType const& source,
|
||||||
PartialTranslOptColl &outputPartialTranslOptColl,
|
PartialTranslOptColl &outputPartialTranslOptColl,
|
||||||
size_t startPos, size_t endPos,
|
size_t startPos, size_t endPos,
|
||||||
bool adhereTableLimit,
|
bool adhereTableLimit,
|
||||||
InputPathList const& inputPathList) const
|
InputPathList const& inputPathList) const
|
||||||
{
|
{
|
||||||
const PhraseDictionary* phraseDictionary = GetPhraseDictionaryFeature();
|
const PhraseDictionary* phraseDictionary = GetPhraseDictionaryFeature();
|
||||||
const size_t tableLimit = phraseDictionary->GetTableLimit();
|
const size_t tableLimit = phraseDictionary->GetTableLimit();
|
||||||
@ -158,12 +158,12 @@ ProcessInitialTransLEGACY(InputType const& source,
|
|||||||
if (phraseColl != NULL) {
|
if (phraseColl != NULL) {
|
||||||
IFVERBOSE(3) {
|
IFVERBOSE(3) {
|
||||||
if(source.GetType() == SentenceInput)
|
if(source.GetType() == SentenceInput)
|
||||||
TRACE_ERR("[" << source.GetSubString(range) << "; "
|
TRACE_ERR("[" << source.GetSubString(range) << "; "
|
||||||
<< startPos << "-" << endPos << "]\n");
|
<< startPos << "-" << endPos << "]\n");
|
||||||
else
|
else
|
||||||
TRACE_ERR("[" << startPos << "-" << endPos << "]" << std::endl);
|
TRACE_ERR("[" << startPos << "-" << endPos << "]" << std::endl);
|
||||||
}
|
}
|
||||||
|
|
||||||
const std::vector<Phrase> &sourcePhrases = phraseColl->GetSourcePhrases();
|
const std::vector<Phrase> &sourcePhrases = phraseColl->GetSourcePhrases();
|
||||||
|
|
||||||
TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
|
TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
|
||||||
|
@ -61,12 +61,12 @@ public:
|
|||||||
, TargetPhraseCollection::shared_ptr phraseColl) const;
|
, TargetPhraseCollection::shared_ptr phraseColl) const;
|
||||||
|
|
||||||
// legacy
|
// legacy
|
||||||
void
|
void
|
||||||
ProcessInitialTransLEGACY(InputType const& source,
|
ProcessInitialTransLEGACY(InputType const& source,
|
||||||
PartialTranslOptColl &outputPartialTranslOptColl,
|
PartialTranslOptColl &outputPartialTranslOptColl,
|
||||||
size_t startPos, size_t endPos,
|
size_t startPos, size_t endPos,
|
||||||
bool adhereTableLimit,
|
bool adhereTableLimit,
|
||||||
InputPathList const& inputPathList) const;
|
InputPathList const& inputPathList) const;
|
||||||
|
|
||||||
void ProcessLEGACY(const TranslationOption &inputPartialTranslOpt
|
void ProcessLEGACY(const TranslationOption &inputPartialTranslOpt
|
||||||
, const DecodeStep &decodeStep
|
, const DecodeStep &decodeStep
|
||||||
|
@ -28,9 +28,9 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
//! populate this InputType with data from in stream
|
//! populate this InputType with data from in stream
|
||||||
virtual int
|
virtual int
|
||||||
Read(std::istream& in,
|
Read(std::istream& in,
|
||||||
std::vector<FactorType> const& factorOrder,
|
std::vector<FactorType> const& factorOrder,
|
||||||
AllOptions const& opts);
|
AllOptions const& opts);
|
||||||
|
|
||||||
//! Output debugging info to stream out
|
//! Output debugging info to stream out
|
||||||
|
@ -380,16 +380,16 @@ OutputAlignment(std::ostream &out) const
|
|||||||
edges.push_back(currentHypo);
|
edges.push_back(currentHypo);
|
||||||
currentHypo = currentHypo->GetPrevHypo();
|
currentHypo = currentHypo->GetPrevHypo();
|
||||||
}
|
}
|
||||||
|
|
||||||
OutputAlignment(out, edges, m_manager.options().output.WA_SortOrder);
|
OutputAlignment(out, edges, m_manager.options().output.WA_SortOrder);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Hypothesis::
|
Hypothesis::
|
||||||
OutputAlignment(ostream &out,
|
OutputAlignment(ostream &out,
|
||||||
vector<const Hypothesis *> const& edges,
|
vector<const Hypothesis *> const& edges,
|
||||||
WordAlignmentSort waso)
|
WordAlignmentSort waso)
|
||||||
{
|
{
|
||||||
size_t targetOffset = 0;
|
size_t targetOffset = 0;
|
||||||
|
|
||||||
@ -409,7 +409,7 @@ void
|
|||||||
Hypothesis::
|
Hypothesis::
|
||||||
OutputAlignment(ostream &out, const AlignmentInfo &ai,
|
OutputAlignment(ostream &out, const AlignmentInfo &ai,
|
||||||
size_t sourceOffset, size_t targetOffset,
|
size_t sourceOffset, size_t targetOffset,
|
||||||
WordAlignmentSort waso)
|
WordAlignmentSort waso)
|
||||||
{
|
{
|
||||||
typedef std::vector< const std::pair<size_t,size_t>* > AlignVec;
|
typedef std::vector< const std::pair<size_t,size_t>* > AlignVec;
|
||||||
AlignVec alignments = ai.GetSortedAlignments(waso);
|
AlignVec alignments = ai.GetSortedAlignments(waso);
|
||||||
@ -417,8 +417,8 @@ OutputAlignment(ostream &out, const AlignmentInfo &ai,
|
|||||||
AlignVec::const_iterator it;
|
AlignVec::const_iterator it;
|
||||||
for (it = alignments.begin(); it != alignments.end(); ++it) {
|
for (it = alignments.begin(); it != alignments.end(); ++it) {
|
||||||
const std::pair<size_t,size_t> &alignment = **it;
|
const std::pair<size_t,size_t> &alignment = **it;
|
||||||
out << alignment.first + sourceOffset << "-"
|
out << alignment.first + sourceOffset << "-"
|
||||||
<< alignment.second + targetOffset << " ";
|
<< alignment.second + targetOffset << " ";
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
@ -608,7 +608,7 @@ OutputLocalWordAlignment(vector<xmlrpc_c::value>& dest) const
|
|||||||
using namespace std;
|
using namespace std;
|
||||||
Range const& src = this->GetCurrSourceWordsRange();
|
Range const& src = this->GetCurrSourceWordsRange();
|
||||||
Range const& trg = this->GetCurrTargetWordsRange();
|
Range const& trg = this->GetCurrTargetWordsRange();
|
||||||
|
|
||||||
WordAlignmentSort waso = m_manager.options().output.WA_SortOrder;
|
WordAlignmentSort waso = m_manager.options().output.WA_SortOrder;
|
||||||
vector<pair<size_t,size_t> const* > a
|
vector<pair<size_t,size_t> const* > a
|
||||||
= this->GetCurrTargetPhrase().GetAlignTerm().GetSortedAlignments(waso);
|
= this->GetCurrTargetPhrase().GetAlignTerm().GetSortedAlignments(waso);
|
||||||
|
@ -251,18 +251,18 @@ public:
|
|||||||
return m_transOpt;
|
return m_transOpt;
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
OutputAlignment(std::ostream &out) const;
|
OutputAlignment(std::ostream &out) const;
|
||||||
|
|
||||||
static void
|
static void
|
||||||
OutputAlignment(std::ostream &out,
|
OutputAlignment(std::ostream &out,
|
||||||
const std::vector<const Hypothesis *> &edges,
|
const std::vector<const Hypothesis *> &edges,
|
||||||
WordAlignmentSort waso);
|
WordAlignmentSort waso);
|
||||||
|
|
||||||
static void
|
static void
|
||||||
OutputAlignment(std::ostream &out, const Moses::AlignmentInfo &ai,
|
OutputAlignment(std::ostream &out, const Moses::AlignmentInfo &ai,
|
||||||
size_t sourceOffset, size_t targetOffset,
|
size_t sourceOffset, size_t targetOffset,
|
||||||
WordAlignmentSort waso);
|
WordAlignmentSort waso);
|
||||||
|
|
||||||
void OutputInput(std::ostream& os) const;
|
void OutputInput(std::ostream& os) const;
|
||||||
static void OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo);
|
static void OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo);
|
||||||
|
@ -95,7 +95,7 @@ IOWrapper::IOWrapper()
|
|||||||
m_look_back = staticData.options().context.look_back;
|
m_look_back = staticData.options().context.look_back;
|
||||||
|
|
||||||
m_inputType = staticData.options().input.input_type;
|
m_inputType = staticData.options().input.input_type;
|
||||||
|
|
||||||
UTIL_THROW_IF2((m_look_ahead || m_look_back) && m_inputType != SentenceInput,
|
UTIL_THROW_IF2((m_look_ahead || m_look_back) && m_inputType != SentenceInput,
|
||||||
"Context-sensitive decoding currently works only with sentence input.");
|
"Context-sensitive decoding currently works only with sentence input.");
|
||||||
|
|
||||||
|
@ -320,15 +320,15 @@ void Manager::OutputNBest(OutputCollector *collector) const
|
|||||||
OutputNBestList(collector, *completed_nbest_, m_source.GetTranslationId());
|
OutputNBestList(collector, *completed_nbest_, m_source.GetTranslationId());
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputNBestList(OutputCollector *collector,
|
OutputNBestList(OutputCollector *collector,
|
||||||
std::vector<search::Applied> const& nbest,
|
std::vector<search::Applied> const& nbest,
|
||||||
long translationId) const
|
long translationId) const
|
||||||
{
|
{
|
||||||
const StaticData &staticData = StaticData::Instance();
|
const StaticData &staticData = StaticData::Instance();
|
||||||
const std::vector<Moses::FactorType> &outputFactorOrder
|
const std::vector<Moses::FactorType> &outputFactorOrder
|
||||||
= staticData.GetOutputFactorOrder();
|
= staticData.GetOutputFactorOrder();
|
||||||
|
|
||||||
std::ostringstream out;
|
std::ostringstream out;
|
||||||
// wtf? copied from the original OutputNBestList
|
// wtf? copied from the original OutputNBestList
|
||||||
@ -337,13 +337,13 @@ OutputNBestList(OutputCollector *collector,
|
|||||||
}
|
}
|
||||||
Phrase outputPhrase;
|
Phrase outputPhrase;
|
||||||
ScoreComponentCollection features;
|
ScoreComponentCollection features;
|
||||||
for (std::vector<search::Applied>::const_iterator i = nbest.begin();
|
for (std::vector<search::Applied>::const_iterator i = nbest.begin();
|
||||||
i != nbest.end(); ++i) {
|
i != nbest.end(); ++i) {
|
||||||
Incremental::PhraseAndFeatures(*i, outputPhrase, features);
|
Incremental::PhraseAndFeatures(*i, outputPhrase, features);
|
||||||
// <s> and </s>
|
// <s> and </s>
|
||||||
UTIL_THROW_IF2(outputPhrase.GetSize() < 2,
|
UTIL_THROW_IF2(outputPhrase.GetSize() < 2,
|
||||||
"Output phrase should have contained at least 2 words "
|
"Output phrase should have contained at least 2 words "
|
||||||
<< "(beginning and end-of-sentence)");
|
<< "(beginning and end-of-sentence)");
|
||||||
|
|
||||||
outputPhrase.RemoveWord(0);
|
outputPhrase.RemoveWord(0);
|
||||||
outputPhrase.RemoveWord(outputPhrase.GetSize() - 1);
|
outputPhrase.RemoveWord(outputPhrase.GetSize() - 1);
|
||||||
@ -359,7 +359,7 @@ OutputNBestList(OutputCollector *collector,
|
|||||||
collector->Write(translationId, out.str());
|
collector->Write(translationId, out.str());
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputDetailedTranslationReport(OutputCollector *collector) const
|
OutputDetailedTranslationReport(OutputCollector *collector) const
|
||||||
{
|
{
|
||||||
@ -525,7 +525,7 @@ void Manager::OutputBestHypo(OutputCollector *collector, search::Applied applied
|
|||||||
VERBOSE(1,"BEST TRANSLATION: " << outPhrase << "[total=" << applied.GetScore() << "]" << std::endl);
|
VERBOSE(1,"BEST TRANSLATION: " << outPhrase << "[total=" << applied.GetScore() << "]" << std::endl);
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputBestNone(OutputCollector *collector, long translationId) const
|
OutputBestNone(OutputCollector *collector, long translationId) const
|
||||||
{
|
{
|
||||||
|
@ -184,8 +184,8 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
//! populate this InputType with data from in stream
|
//! populate this InputType with data from in stream
|
||||||
virtual int
|
virtual int
|
||||||
Read(std::istream& in,
|
Read(std::istream& in,
|
||||||
std::vector<FactorType> const& factorOrder,
|
std::vector<FactorType> const& factorOrder,
|
||||||
AllOptions const& opts) =0;
|
AllOptions const& opts) =0;
|
||||||
|
|
||||||
|
@ -97,11 +97,11 @@ public:
|
|||||||
virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const;
|
virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const;
|
||||||
|
|
||||||
virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const;
|
virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const;
|
||||||
/*
|
/*
|
||||||
virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const;
|
virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const;
|
||||||
|
|
||||||
virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const;
|
virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const;
|
||||||
*/
|
*/
|
||||||
|
|
||||||
void InitializeForInput(ttasksptr const& ttask);
|
void InitializeForInput(ttasksptr const& ttask);
|
||||||
void CleanUpAfterSentenceProcessing(const InputType& source);
|
void CleanUpAfterSentenceProcessing(const InputType& source);
|
||||||
|
@ -100,8 +100,8 @@ Manager::GetSource() const
|
|||||||
void Manager::Decode()
|
void Manager::Decode()
|
||||||
{
|
{
|
||||||
|
|
||||||
std::cerr << options().nbest.nbest_size << " "
|
std::cerr << options().nbest.nbest_size << " "
|
||||||
<< options().nbest.enabled << " " << std::endl;
|
<< options().nbest.enabled << " " << std::endl;
|
||||||
|
|
||||||
// initialize statistics
|
// initialize statistics
|
||||||
ResetSentenceStats(m_source);
|
ResetSentenceStats(m_source);
|
||||||
@ -128,8 +128,8 @@ void Manager::Decode()
|
|||||||
// some reporting on how long this took
|
// some reporting on how long this took
|
||||||
IFVERBOSE(1) {
|
IFVERBOSE(1) {
|
||||||
GetSentenceStats().StopTimeCollectOpts();
|
GetSentenceStats().StopTimeCollectOpts();
|
||||||
TRACE_ERR("Line "<< m_source.GetTranslationId()
|
TRACE_ERR("Line "<< m_source.GetTranslationId()
|
||||||
<< ": Collecting options took "
|
<< ": Collecting options took "
|
||||||
<< GetSentenceStats().GetTimeCollectOpts() << " seconds at "
|
<< GetSentenceStats().GetTimeCollectOpts() << " seconds at "
|
||||||
<< __FILE__ << ":" << __LINE__ << endl);
|
<< __FILE__ << ":" << __LINE__ << endl);
|
||||||
}
|
}
|
||||||
@ -1118,10 +1118,10 @@ void Manager::OutputSearchGraphAsSLF(long translationId, std::ostream &outputSea
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
OutputSearchNode(AllOptions const& opts, long translationId,
|
OutputSearchNode(AllOptions const& opts, long translationId,
|
||||||
std::ostream &outputSearchGraphStream,
|
std::ostream &outputSearchGraphStream,
|
||||||
SearchGraphNode const& searchNode)
|
SearchGraphNode const& searchNode)
|
||||||
{
|
{
|
||||||
const vector<FactorType> &outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
|
const vector<FactorType> &outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
|
||||||
bool extendedFormat = opts.output.SearchGraphExtended.size();
|
bool extendedFormat = opts.output.SearchGraphExtended.size();
|
||||||
@ -1377,7 +1377,7 @@ void Manager::SerializeSearchGraphPB(
|
|||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputSearchGraph(long translationId, std::ostream &out) const
|
OutputSearchGraph(long translationId, std::ostream &out) const
|
||||||
{
|
{
|
||||||
@ -1388,12 +1388,12 @@ OutputSearchGraph(long translationId, std::ostream &out) const
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
GetForwardBackwardSearchGraph
|
GetForwardBackwardSearchGraph
|
||||||
( std::map< int, bool >* pConnected,
|
( std::map< int, bool >* pConnected,
|
||||||
std::vector<Hypothesis const* >* pConnectedList,
|
std::vector<Hypothesis const* >* pConnectedList,
|
||||||
std::map<Hypothesis const*, set<Hypothesis const*> >* pOutgoingHyps,
|
std::map<Hypothesis const*, set<Hypothesis const*> >* pOutgoingHyps,
|
||||||
vector< float>* pFwdBwdScores) const
|
vector< float>* pFwdBwdScores) const
|
||||||
{
|
{
|
||||||
std::map < int, bool > &connected = *pConnected;
|
std::map < int, bool > &connected = *pConnected;
|
||||||
@ -1401,8 +1401,8 @@ GetForwardBackwardSearchGraph
|
|||||||
std::map < int, int > forward;
|
std::map < int, int > forward;
|
||||||
std::map < int, double > forwardScore;
|
std::map < int, double > forwardScore;
|
||||||
|
|
||||||
std::map < const Hypothesis*, set <const Hypothesis*> > & outgoingHyps
|
std::map < const Hypothesis*, set <const Hypothesis*> > & outgoingHyps
|
||||||
= *pOutgoingHyps;
|
= *pOutgoingHyps;
|
||||||
vector< float> & estimatedScores = *pFwdBwdScores;
|
vector< float> & estimatedScores = *pFwdBwdScores;
|
||||||
|
|
||||||
// *** find connected hypotheses ***
|
// *** find connected hypotheses ***
|
||||||
@ -1411,8 +1411,8 @@ GetForwardBackwardSearchGraph
|
|||||||
// ** compute best forward path for each hypothesis *** //
|
// ** compute best forward path for each hypothesis *** //
|
||||||
|
|
||||||
// forward cost of hypotheses on final stack is 0
|
// forward cost of hypotheses on final stack is 0
|
||||||
const std::vector < HypothesisStack* > &hypoStackColl
|
const std::vector < HypothesisStack* > &hypoStackColl
|
||||||
= m_search->GetHypothesisStacks();
|
= m_search->GetHypothesisStacks();
|
||||||
const HypothesisStack &finalStack = *hypoStackColl.back();
|
const HypothesisStack &finalStack = *hypoStackColl.back();
|
||||||
HypothesisStack::const_iterator iterHypo;
|
HypothesisStack::const_iterator iterHypo;
|
||||||
for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
|
for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
|
||||||
@ -1531,12 +1531,12 @@ void Manager::OutputBest(OutputCollector *collector) const
|
|||||||
|
|
||||||
// const PARAM_VEC *params = staticData.GetParameter().GetParam("print-id");
|
// const PARAM_VEC *params = staticData.GetParameter().GetParam("print-id");
|
||||||
if (options().output.PrintID) {
|
if (options().output.PrintID) {
|
||||||
out << translationId << " ";
|
out << translationId << " ";
|
||||||
}
|
}
|
||||||
|
|
||||||
// VN : I put back the code for OutputPassthroughInformation
|
// VN : I put back the code for OutputPassthroughInformation
|
||||||
if (options().output.PrintPassThrough) {
|
if (options().output.PrintPassThrough) {
|
||||||
OutputPassthroughInformation(out, bestHypo);
|
OutputPassthroughInformation(out, bestHypo);
|
||||||
}
|
}
|
||||||
// end of add back
|
// end of add back
|
||||||
|
|
||||||
@ -1589,8 +1589,8 @@ void Manager::OutputBest(OutputCollector *collector) const
|
|||||||
} else {
|
} else {
|
||||||
//Lattice MBR decoding
|
//Lattice MBR decoding
|
||||||
vector<Word> mbrBestHypo = doLatticeMBR(*this,nBestList);
|
vector<Word> mbrBestHypo = doLatticeMBR(*this,nBestList);
|
||||||
OutputBestHypo(mbrBestHypo, translationId,
|
OutputBestHypo(mbrBestHypo, translationId,
|
||||||
options().output.ReportSegmentation,
|
options().output.ReportSegmentation,
|
||||||
options().output.ReportAllFactors, out);
|
options().output.ReportAllFactors, out);
|
||||||
IFVERBOSE(2) {
|
IFVERBOSE(2) {
|
||||||
PrintUserTime("finished Lattice MBR decoding");
|
PrintUserTime("finished Lattice MBR decoding");
|
||||||
@ -1614,7 +1614,7 @@ void Manager::OutputBest(OutputCollector *collector) const
|
|||||||
else {
|
else {
|
||||||
const TrellisPath &mbrBestHypo = doMBR(nBestList);
|
const TrellisPath &mbrBestHypo = doMBR(nBestList);
|
||||||
OutputBestHypo(mbrBestHypo, translationId,
|
OutputBestHypo(mbrBestHypo, translationId,
|
||||||
options().output.ReportSegmentation,
|
options().output.ReportSegmentation,
|
||||||
options().output.ReportAllFactors, out);
|
options().output.ReportAllFactors, out);
|
||||||
OutputAlignment(m_alignmentOut, mbrBestHypo);
|
OutputAlignment(m_alignmentOut, mbrBestHypo);
|
||||||
IFVERBOSE(2) {
|
IFVERBOSE(2) {
|
||||||
@ -1650,8 +1650,8 @@ void Manager::OutputNBest(OutputCollector *collector) const
|
|||||||
ostringstream out;
|
ostringstream out;
|
||||||
CalcNBest(options().nbest.nbest_size, nBestList,
|
CalcNBest(options().nbest.nbest_size, nBestList,
|
||||||
options().nbest.only_distinct);
|
options().nbest.only_distinct);
|
||||||
OutputNBest(out, nBestList,
|
OutputNBest(out, nBestList,
|
||||||
staticData.GetOutputFactorOrder(),
|
staticData.GetOutputFactorOrder(),
|
||||||
m_source.GetTranslationId(),
|
m_source.GetTranslationId(),
|
||||||
options().output.ReportSegmentation);
|
options().output.ReportSegmentation);
|
||||||
collector->Write(m_source.GetTranslationId(), out.str());
|
collector->Write(m_source.GetTranslationId(), out.str());
|
||||||
@ -1659,12 +1659,12 @@ void Manager::OutputNBest(OutputCollector *collector) const
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputNBest(std::ostream& out,
|
OutputNBest(std::ostream& out,
|
||||||
const Moses::TrellisPathList &nBestList,
|
const Moses::TrellisPathList &nBestList,
|
||||||
const std::vector<Moses::FactorType>& outputFactorOrder,
|
const std::vector<Moses::FactorType>& outputFactorOrder,
|
||||||
long translationId, char reportSegmentation) const
|
long translationId, char reportSegmentation) const
|
||||||
{
|
{
|
||||||
const StaticData &staticData = StaticData::Instance();
|
const StaticData &staticData = StaticData::Instance();
|
||||||
NBestOptions const& nbo = options().nbest;
|
NBestOptions const& nbo = options().nbest;
|
||||||
@ -1681,8 +1681,8 @@ OutputNBest(std::ostream& out,
|
|||||||
out << translationId << " ||| ";
|
out << translationId << " ||| ";
|
||||||
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
|
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
|
||||||
const Hypothesis &edge = *edges[currEdge];
|
const Hypothesis &edge = *edges[currEdge];
|
||||||
OutputSurface(out, edge, outputFactorOrder, reportSegmentation,
|
OutputSurface(out, edge, outputFactorOrder, reportSegmentation,
|
||||||
reportAllFactors);
|
reportAllFactors);
|
||||||
}
|
}
|
||||||
out << " |||";
|
out << " |||";
|
||||||
|
|
||||||
@ -1741,11 +1741,11 @@ OutputNBest(std::ostream& out,
|
|||||||
/***
|
/***
|
||||||
* print surface factor only for the given phrase
|
* print surface factor only for the given phrase
|
||||||
*/
|
*/
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputSurface(std::ostream &out, const Hypothesis &edge,
|
OutputSurface(std::ostream &out, const Hypothesis &edge,
|
||||||
const std::vector<FactorType> &outputFactorOrder,
|
const std::vector<FactorType> &outputFactorOrder,
|
||||||
char reportSegmentation, bool reportAllFactors) const
|
char reportSegmentation, bool reportAllFactors) const
|
||||||
{
|
{
|
||||||
UTIL_THROW_IF2(outputFactorOrder.size() == 0,
|
UTIL_THROW_IF2(outputFactorOrder.size() == 0,
|
||||||
"Must specific at least 1 output factor");
|
"Must specific at least 1 output factor");
|
||||||
@ -1820,10 +1820,10 @@ OutputSurface(std::ostream &out, const Hypothesis &edge,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputAlignment(ostream &out, const AlignmentInfo &ai,
|
OutputAlignment(ostream &out, const AlignmentInfo &ai,
|
||||||
size_t sourceOffset, size_t targetOffset) const
|
size_t sourceOffset, size_t targetOffset) const
|
||||||
{
|
{
|
||||||
typedef std::vector< const std::pair<size_t,size_t>* > AlignVec;
|
typedef std::vector< const std::pair<size_t,size_t>* > AlignVec;
|
||||||
AlignVec alignments = ai.GetSortedAlignments(options().output.WA_SortOrder);
|
AlignVec alignments = ai.GetSortedAlignments(options().output.WA_SortOrder);
|
||||||
@ -1831,13 +1831,13 @@ OutputAlignment(ostream &out, const AlignmentInfo &ai,
|
|||||||
AlignVec::const_iterator it;
|
AlignVec::const_iterator it;
|
||||||
for (it = alignments.begin(); it != alignments.end(); ++it) {
|
for (it = alignments.begin(); it != alignments.end(); ++it) {
|
||||||
const std::pair<size_t,size_t> &alignment = **it;
|
const std::pair<size_t,size_t> &alignment = **it;
|
||||||
out << alignment.first + sourceOffset << "-"
|
out << alignment.first + sourceOffset << "-"
|
||||||
<< alignment.second + targetOffset << " ";
|
<< alignment.second + targetOffset << " ";
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputInput(std::ostream& os, const Hypothesis* hypo) const
|
OutputInput(std::ostream& os, const Hypothesis* hypo) const
|
||||||
{
|
{
|
||||||
@ -1883,9 +1883,9 @@ void Manager::OutputLatticeSamples(OutputCollector *collector) const
|
|||||||
TrellisPathList latticeSamples;
|
TrellisPathList latticeSamples;
|
||||||
ostringstream out;
|
ostringstream out;
|
||||||
CalcLatticeSamples(staticData.GetLatticeSamplesSize(), latticeSamples);
|
CalcLatticeSamples(staticData.GetLatticeSamplesSize(), latticeSamples);
|
||||||
OutputNBest(out,latticeSamples,
|
OutputNBest(out,latticeSamples,
|
||||||
staticData.GetOutputFactorOrder(),
|
staticData.GetOutputFactorOrder(),
|
||||||
m_source.GetTranslationId(),
|
m_source.GetTranslationId(),
|
||||||
options().output.ReportSegmentation);
|
options().output.ReportSegmentation);
|
||||||
collector->Write(m_source.GetTranslationId(), out.str());
|
collector->Write(m_source.GetTranslationId(), out.str());
|
||||||
}
|
}
|
||||||
@ -2075,11 +2075,11 @@ void Manager::OutputBestHypo(const std::vector<Word>& mbrBestHypo, long /*trans
|
|||||||
out << endl;
|
out << endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Manager::
|
Manager::
|
||||||
OutputBestHypo(const Moses::TrellisPath &path, long /*translationId*/,
|
OutputBestHypo(const Moses::TrellisPath &path, long /*translationId*/,
|
||||||
char reportSegmentation, bool reportAllFactors,
|
char reportSegmentation, bool reportAllFactors,
|
||||||
std::ostream &out) const
|
std::ostream &out) const
|
||||||
{
|
{
|
||||||
const std::vector<const Hypothesis *> &edges = path.GetEdges();
|
const std::vector<const Hypothesis *> &edges = path.GetEdges();
|
||||||
|
|
||||||
|
@ -131,7 +131,7 @@ protected:
|
|||||||
// nbest
|
// nbest
|
||||||
mutable std::ostringstream m_latticeNBestOut;
|
mutable std::ostringstream m_latticeNBestOut;
|
||||||
mutable std::ostringstream m_alignmentOut;
|
mutable std::ostringstream m_alignmentOut;
|
||||||
public:
|
public:
|
||||||
void OutputNBest(std::ostream& out
|
void OutputNBest(std::ostream& out
|
||||||
, const Moses::TrellisPathList &nBestList
|
, const Moses::TrellisPathList &nBestList
|
||||||
, const std::vector<Moses::FactorType>& outputFactorOrder
|
, const std::vector<Moses::FactorType>& outputFactorOrder
|
||||||
|
@ -45,13 +45,13 @@ MockHypothesisGuard
|
|||||||
m_manager.reset(new Manager(m_ttask));
|
m_manager.reset(new Manager(m_ttask));
|
||||||
|
|
||||||
//Initial empty hypothesis
|
//Initial empty hypothesis
|
||||||
Bitmaps bitmaps(m_sentence.get()->GetSize(),
|
Bitmaps bitmaps(m_sentence.get()->GetSize(),
|
||||||
m_sentence.get()->m_sourceCompleted);
|
m_sentence.get()->m_sourceCompleted);
|
||||||
m_manager->ResetSentenceStats(*m_sentence);
|
m_manager->ResetSentenceStats(*m_sentence);
|
||||||
|
|
||||||
const Bitmap &initBitmap = bitmaps.GetInitialBitmap();
|
const Bitmap &initBitmap = bitmaps.GetInitialBitmap();
|
||||||
m_hypothesis = new Hypothesis(*m_manager, *m_sentence, m_initialTransOpt,
|
m_hypothesis = new Hypothesis(*m_manager, *m_sentence, m_initialTransOpt,
|
||||||
initBitmap);
|
initBitmap);
|
||||||
|
|
||||||
//create the chain
|
//create the chain
|
||||||
vector<Alignment>::const_iterator ai = alignments.begin();
|
vector<Alignment>::const_iterator ai = alignments.begin();
|
||||||
@ -59,8 +59,8 @@ MockHypothesisGuard
|
|||||||
for (; ti != targetSegments.end() && ai != alignments.end(); ++ti,++ai) {
|
for (; ti != targetSegments.end() && ai != alignments.end(); ++ti,++ai) {
|
||||||
Hypothesis* prevHypo = m_hypothesis;
|
Hypothesis* prevHypo = m_hypothesis;
|
||||||
Range range(ai->first,ai->second);
|
Range range(ai->first,ai->second);
|
||||||
const Bitmap &newBitmap = bitmaps.GetBitmap(prevHypo->GetWordsBitmap(),
|
const Bitmap &newBitmap = bitmaps.GetBitmap(prevHypo->GetWordsBitmap(),
|
||||||
range);
|
range);
|
||||||
|
|
||||||
m_targetPhrases.push_back(TargetPhrase(NULL));
|
m_targetPhrases.push_back(TargetPhrase(NULL));
|
||||||
// m_targetPhrases.back().CreateFromString(Input, factors, *ti, "|", NULL);
|
// m_targetPhrases.back().CreateFromString(Input, factors, *ti, "|", NULL);
|
||||||
|
@ -1620,7 +1620,7 @@ SetParameter<bool>(bool ¶meter, std::string const& parameterName,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Parameter::
|
Parameter::
|
||||||
SetParameter(bool& var, std::string const& name)
|
SetParameter(bool& var, std::string const& name)
|
||||||
{
|
{
|
||||||
|
@ -155,14 +155,13 @@ public:
|
|||||||
// issues a warning if format is wrong
|
// issues a warning if format is wrong
|
||||||
const PARAM_VEC *params = GetParam(name);
|
const PARAM_VEC *params = GetParam(name);
|
||||||
val = (params && params->size());
|
val = (params && params->size());
|
||||||
if (val && params->size() != 1)
|
if (val && params->size() != 1) {
|
||||||
{
|
TRACE_ERR("ERROR: wrong format for switch -" << name);
|
||||||
TRACE_ERR("ERROR: wrong format for switch -" << name);
|
return false;
|
||||||
return false;
|
}
|
||||||
}
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
template<>
|
template<>
|
||||||
|
@ -280,7 +280,7 @@ void ScoreComponentCollection::ZeroDenseFeatures(const FeatureFunction* sp)
|
|||||||
}
|
}
|
||||||
|
|
||||||
//! get subset of scores that belong to a certain sparse ScoreProducer
|
//! get subset of scores that belong to a certain sparse ScoreProducer
|
||||||
FVector
|
FVector
|
||||||
ScoreComponentCollection::
|
ScoreComponentCollection::
|
||||||
GetVectorForProducer(const FeatureFunction* sp) const
|
GetVectorForProducer(const FeatureFunction* sp) const
|
||||||
{
|
{
|
||||||
@ -307,21 +307,21 @@ void ScoreComponentCollection::PlusEquals(const FeatureFunction* sp, const Score
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
ScoreComponentCollection::
|
ScoreComponentCollection::
|
||||||
OutputAllFeatureScores(std::ostream &out, bool with_labels) const
|
OutputAllFeatureScores(std::ostream &out, bool with_labels) const
|
||||||
{
|
{
|
||||||
std::string lastName = "";
|
std::string lastName = "";
|
||||||
const vector<const StatefulFeatureFunction*>& sff
|
const vector<const StatefulFeatureFunction*>& sff
|
||||||
= StatefulFeatureFunction::GetStatefulFeatureFunctions();
|
= StatefulFeatureFunction::GetStatefulFeatureFunctions();
|
||||||
for( size_t i=0; i<sff.size(); i++ ) {
|
for( size_t i=0; i<sff.size(); i++ ) {
|
||||||
const StatefulFeatureFunction *ff = sff[i];
|
const StatefulFeatureFunction *ff = sff[i];
|
||||||
if (ff->IsTuneable()) {
|
if (ff->IsTuneable()) {
|
||||||
OutputFeatureScores(out, ff, lastName, with_labels);
|
OutputFeatureScores(out, ff, lastName, with_labels);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const vector<const StatelessFeatureFunction*>& slf
|
const vector<const StatelessFeatureFunction*>& slf
|
||||||
= StatelessFeatureFunction::GetStatelessFeatureFunctions();
|
= StatelessFeatureFunction::GetStatelessFeatureFunctions();
|
||||||
for( size_t i=0; i<slf.size(); i++ ) {
|
for( size_t i=0; i<slf.size(); i++ ) {
|
||||||
const StatelessFeatureFunction *ff = slf[i];
|
const StatelessFeatureFunction *ff = slf[i];
|
||||||
if (ff->IsTuneable()) {
|
if (ff->IsTuneable()) {
|
||||||
@ -330,10 +330,10 @@ OutputAllFeatureScores(std::ostream &out, bool with_labels) const
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
ScoreComponentCollection::
|
ScoreComponentCollection::
|
||||||
OutputFeatureScores(std::ostream& out, FeatureFunction const* ff,
|
OutputFeatureScores(std::ostream& out, FeatureFunction const* ff,
|
||||||
std::string &lastName, bool with_labels) const
|
std::string &lastName, bool with_labels) const
|
||||||
{
|
{
|
||||||
// const StaticData &staticData = StaticData::Instance();
|
// const StaticData &staticData = StaticData::Instance();
|
||||||
// bool labeledOutput = staticData.options().nbest.include_feature_labels;
|
// bool labeledOutput = staticData.options().nbest.include_feature_labels;
|
||||||
|
@ -88,7 +88,7 @@ class ScoreComponentCollection
|
|||||||
{
|
{
|
||||||
friend std::ostream& operator<<(std::ostream& os, const ScoreComponentCollection& rhs);
|
friend std::ostream& operator<<(std::ostream& os, const ScoreComponentCollection& rhs);
|
||||||
friend void swap(ScoreComponentCollection &first, ScoreComponentCollection &second);
|
friend void swap(ScoreComponentCollection &first, ScoreComponentCollection &second);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
FVector m_scores;
|
FVector m_scores;
|
||||||
|
|
||||||
@ -433,7 +433,7 @@ public:
|
|||||||
|
|
||||||
void OutputAllFeatureScores(std::ostream &out, bool with_labels) const;
|
void OutputAllFeatureScores(std::ostream &out, bool with_labels) const;
|
||||||
void OutputFeatureScores(std::ostream& out, Moses::FeatureFunction const* ff,
|
void OutputFeatureScores(std::ostream& out, Moses::FeatureFunction const* ff,
|
||||||
std::string &lastName, bool with_labels) const;
|
std::string &lastName, bool with_labels) const;
|
||||||
|
|
||||||
#ifdef MPI_ENABLE
|
#ifdef MPI_ENABLE
|
||||||
public:
|
public:
|
||||||
|
@ -183,7 +183,7 @@ init(string line, std::vector<FactorType> const& factorOrder,
|
|||||||
aux_interpret_dlt(line); // some poorly documented cache-based stuff
|
aux_interpret_dlt(line); // some poorly documented cache-based stuff
|
||||||
|
|
||||||
// if sentences is specified as "<passthrough tag1=""/>"
|
// if sentences is specified as "<passthrough tag1=""/>"
|
||||||
if (SD.options().output.PrintPassThrough ||
|
if (SD.options().output.PrintPassThrough ||
|
||||||
SD.options().nbest.include_passthrough) {
|
SD.options().nbest.include_passthrough) {
|
||||||
string pthru = PassthroughSGML(line,"passthrough");
|
string pthru = PassthroughSGML(line,"passthrough");
|
||||||
this->SetPassthroughInformation(pthru);
|
this->SetPassthroughInformation(pthru);
|
||||||
@ -232,7 +232,7 @@ init(string line, std::vector<FactorType> const& factorOrder,
|
|||||||
|
|
||||||
int
|
int
|
||||||
Sentence::
|
Sentence::
|
||||||
Read(std::istream& in,
|
Read(std::istream& in,
|
||||||
const std::vector<FactorType>& factorOrder,
|
const std::vector<FactorType>& factorOrder,
|
||||||
AllOptions const& opts)
|
AllOptions const& opts)
|
||||||
{
|
{
|
||||||
@ -370,9 +370,9 @@ CreateFromString(vector<FactorType> const& FOrder, string const& phraseString)
|
|||||||
}
|
}
|
||||||
|
|
||||||
Sentence::
|
Sentence::
|
||||||
Sentence(size_t const transId,
|
Sentence(size_t const transId,
|
||||||
string const& stext,
|
string const& stext,
|
||||||
AllOptions const& opts,
|
AllOptions const& opts,
|
||||||
vector<FactorType> const* IFO)
|
vector<FactorType> const* IFO)
|
||||||
: InputType(transId)
|
: InputType(transId)
|
||||||
{
|
{
|
||||||
|
@ -65,8 +65,8 @@ protected:
|
|||||||
public:
|
public:
|
||||||
Sentence();
|
Sentence();
|
||||||
Sentence(size_t const transId, std::string const& stext,
|
Sentence(size_t const transId, std::string const& stext,
|
||||||
AllOptions const& opts,
|
AllOptions const& opts,
|
||||||
std::vector<FactorType> const* IFO = NULL);
|
std::vector<FactorType> const* IFO = NULL);
|
||||||
// Sentence(size_t const transId, std::string const& stext);
|
// Sentence(size_t const transId, std::string const& stext);
|
||||||
~Sentence();
|
~Sentence();
|
||||||
|
|
||||||
@ -97,7 +97,7 @@ public:
|
|||||||
void GetXmlTranslationOptions(std::vector<TranslationOption*> &list, size_t startPos, size_t endPos) const;
|
void GetXmlTranslationOptions(std::vector<TranslationOption*> &list, size_t startPos, size_t endPos) const;
|
||||||
std::vector<ChartTranslationOptions*> GetXmlChartTranslationOptions() const;
|
std::vector<ChartTranslationOptions*> GetXmlChartTranslationOptions() const;
|
||||||
|
|
||||||
virtual int
|
virtual int
|
||||||
Read(std::istream& in, const std::vector<FactorType>& factorOrder,
|
Read(std::istream& in, const std::vector<FactorType>& factorOrder,
|
||||||
AllOptions const& opts);
|
AllOptions const& opts);
|
||||||
|
|
||||||
|
@ -63,7 +63,7 @@ StaticData StaticData::s_instance;
|
|||||||
StaticData::StaticData()
|
StaticData::StaticData()
|
||||||
: m_sourceStartPosMattersForRecombination(false)
|
: m_sourceStartPosMattersForRecombination(false)
|
||||||
, m_requireSortingAfterSourceContext(false)
|
, m_requireSortingAfterSourceContext(false)
|
||||||
// , m_inputType(SentenceInput)
|
// , m_inputType(SentenceInput)
|
||||||
, m_lmEnableOOVFeature(false)
|
, m_lmEnableOOVFeature(false)
|
||||||
, m_isAlwaysCreateDirectTranslationOption(false)
|
, m_isAlwaysCreateDirectTranslationOption(false)
|
||||||
, m_currentWeightSetting("default")
|
, m_currentWeightSetting("default")
|
||||||
@ -169,21 +169,21 @@ StaticData
|
|||||||
m_parameter->SetParameter(m_verboseLevel, "verbose", (size_t) 1);
|
m_parameter->SetParameter(m_verboseLevel, "verbose", (size_t) 1);
|
||||||
|
|
||||||
|
|
||||||
m_parameter->SetParameter(m_includeLHSInSearchGraph,
|
m_parameter->SetParameter(m_includeLHSInSearchGraph,
|
||||||
"include-lhs-in-search-graph", false );
|
"include-lhs-in-search-graph", false );
|
||||||
|
|
||||||
m_parameter->SetParameter<string>(m_outputUnknownsFile,
|
m_parameter->SetParameter<string>(m_outputUnknownsFile,
|
||||||
"output-unknowns", "");
|
"output-unknowns", "");
|
||||||
|
|
||||||
//Print Translation Options
|
//Print Translation Options
|
||||||
m_parameter->SetParameter(m_printTranslationOptions,
|
m_parameter->SetParameter(m_printTranslationOptions,
|
||||||
"print-translation-option", false );
|
"print-translation-option", false );
|
||||||
|
|
||||||
//Print All Derivations
|
//Print All Derivations
|
||||||
m_parameter->SetParameter(m_printAllDerivations ,
|
m_parameter->SetParameter(m_printAllDerivations ,
|
||||||
"print-all-derivations", false );
|
"print-all-derivations", false );
|
||||||
|
|
||||||
m_parameter->SetParameter<long>(m_startTranslationId,
|
m_parameter->SetParameter<long>(m_startTranslationId,
|
||||||
"start-translation-id", 0);
|
"start-translation-id", 0);
|
||||||
|
|
||||||
//lattice samples
|
//lattice samples
|
||||||
@ -340,9 +340,9 @@ bool StaticData::LoadData(Parameter *parameter)
|
|||||||
|
|
||||||
ini_zombie_options(); // probably dead, or maybe not
|
ini_zombie_options(); // probably dead, or maybe not
|
||||||
|
|
||||||
m_parameter->SetParameter(m_placeHolderFactor, "placeholder-factor",
|
m_parameter->SetParameter(m_placeHolderFactor, "placeholder-factor",
|
||||||
NOT_FOUND);
|
NOT_FOUND);
|
||||||
|
|
||||||
// FEATURE FUNCTION INITIALIZATION HAPPENS HERE ===============================
|
// FEATURE FUNCTION INITIALIZATION HAPPENS HERE ===============================
|
||||||
initialize_features();
|
initialize_features();
|
||||||
|
|
||||||
@ -393,7 +393,7 @@ void StaticData::SetWeight(const FeatureFunction* sp, float weight)
|
|||||||
m_allWeights.Assign(sp,weight);
|
m_allWeights.Assign(sp,weight);
|
||||||
}
|
}
|
||||||
|
|
||||||
void StaticData::SetWeights(const FeatureFunction* sp,
|
void StaticData::SetWeights(const FeatureFunction* sp,
|
||||||
const std::vector<float>& weights)
|
const std::vector<float>& weights)
|
||||||
{
|
{
|
||||||
m_allWeights.Resize();
|
m_allWeights.Resize();
|
||||||
@ -444,9 +444,9 @@ void StaticData::LoadChartDecodingParameters()
|
|||||||
LoadNonTerminals();
|
LoadNonTerminals();
|
||||||
|
|
||||||
// source label overlap
|
// source label overlap
|
||||||
m_parameter->SetParameter(m_sourceLabelOverlap, "source-label-overlap",
|
m_parameter->SetParameter(m_sourceLabelOverlap, "source-label-overlap",
|
||||||
SourceLabelOverlapAdd);
|
SourceLabelOverlapAdd);
|
||||||
m_parameter->SetParameter(m_ruleLimit, "rule-limit",
|
m_parameter->SetParameter(m_ruleLimit, "rule-limit",
|
||||||
DEFAULT_MAX_TRANS_OPT_SIZE);
|
DEFAULT_MAX_TRANS_OPT_SIZE);
|
||||||
|
|
||||||
}
|
}
|
||||||
@ -485,16 +485,16 @@ void StaticData::LoadDecodeGraphs()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
StaticData::
|
StaticData::
|
||||||
LoadDecodeGraphsOld(const vector<string> &mappingVector,
|
LoadDecodeGraphsOld(const vector<string> &mappingVector,
|
||||||
const vector<size_t> &maxChartSpans)
|
const vector<size_t> &maxChartSpans)
|
||||||
{
|
{
|
||||||
const vector<PhraseDictionary*>& pts = PhraseDictionary::GetColl();
|
const vector<PhraseDictionary*>& pts = PhraseDictionary::GetColl();
|
||||||
const vector<GenerationDictionary*>& gens = GenerationDictionary::GetColl();
|
const vector<GenerationDictionary*>& gens = GenerationDictionary::GetColl();
|
||||||
|
|
||||||
const std::vector<FeatureFunction*> *featuresRemaining
|
const std::vector<FeatureFunction*> *featuresRemaining
|
||||||
= &FeatureFunction::GetFeatureFunctions();
|
= &FeatureFunction::GetFeatureFunctions();
|
||||||
DecodeStep *prev = 0;
|
DecodeStep *prev = 0;
|
||||||
size_t prevDecodeGraphInd = 0;
|
size_t prevDecodeGraphInd = 0;
|
||||||
|
|
||||||
@ -513,7 +513,7 @@ LoadDecodeGraphsOld(const vector<string> &mappingVector,
|
|||||||
// For specifying multiple translation model
|
// For specifying multiple translation model
|
||||||
decodeGraphInd = Scan<size_t>(token[0]);
|
decodeGraphInd = Scan<size_t>(token[0]);
|
||||||
//the vectorList index can only increment by one
|
//the vectorList index can only increment by one
|
||||||
UTIL_THROW_IF2(decodeGraphInd != prevDecodeGraphInd
|
UTIL_THROW_IF2(decodeGraphInd != prevDecodeGraphInd
|
||||||
&& decodeGraphInd != prevDecodeGraphInd + 1,
|
&& decodeGraphInd != prevDecodeGraphInd + 1,
|
||||||
"Malformed mapping");
|
"Malformed mapping");
|
||||||
if (decodeGraphInd > prevDecodeGraphInd) {
|
if (decodeGraphInd > prevDecodeGraphInd) {
|
||||||
@ -601,7 +601,7 @@ void StaticData::LoadDecodeGraphsNew(const std::vector<std::string> &mappingVect
|
|||||||
|
|
||||||
decodeGraphInd = Scan<size_t>(token[0]);
|
decodeGraphInd = Scan<size_t>(token[0]);
|
||||||
//the vectorList index can only increment by one
|
//the vectorList index can only increment by one
|
||||||
UTIL_THROW_IF2(decodeGraphInd != prevDecodeGraphInd
|
UTIL_THROW_IF2(decodeGraphInd != prevDecodeGraphInd
|
||||||
&& decodeGraphInd != prevDecodeGraphInd + 1,
|
&& decodeGraphInd != prevDecodeGraphInd + 1,
|
||||||
"Malformed mapping");
|
"Malformed mapping");
|
||||||
if (decodeGraphInd > prevDecodeGraphInd) {
|
if (decodeGraphInd > prevDecodeGraphInd) {
|
||||||
@ -702,14 +702,14 @@ StaticData::
|
|||||||
InitializeForInput(ttasksptr const& ttask) const
|
InitializeForInput(ttasksptr const& ttask) const
|
||||||
{
|
{
|
||||||
const std::vector<FeatureFunction*> &producers
|
const std::vector<FeatureFunction*> &producers
|
||||||
= FeatureFunction::GetFeatureFunctions();
|
= FeatureFunction::GetFeatureFunctions();
|
||||||
for(size_t i=0; i<producers.size(); ++i) {
|
for(size_t i=0; i<producers.size(); ++i) {
|
||||||
FeatureFunction &ff = *producers[i];
|
FeatureFunction &ff = *producers[i];
|
||||||
if (! IsFeatureFunctionIgnored(ff)) {
|
if (! IsFeatureFunctionIgnored(ff)) {
|
||||||
Timer iTime;
|
Timer iTime;
|
||||||
iTime.start();
|
iTime.start();
|
||||||
ff.InitializeForInput(ttask);
|
ff.InitializeForInput(ttask);
|
||||||
VERBOSE(3,"InitializeForInput( " << ff.GetScoreProducerDescription()
|
VERBOSE(3,"InitializeForInput( " << ff.GetScoreProducerDescription()
|
||||||
<< " )" << "= " << iTime << endl);
|
<< " )" << "= " << iTime << endl);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -720,7 +720,7 @@ StaticData::
|
|||||||
CleanUpAfterSentenceProcessing(ttasksptr const& ttask) const
|
CleanUpAfterSentenceProcessing(ttasksptr const& ttask) const
|
||||||
{
|
{
|
||||||
const std::vector<FeatureFunction*> &producers
|
const std::vector<FeatureFunction*> &producers
|
||||||
= FeatureFunction::GetFeatureFunctions();
|
= FeatureFunction::GetFeatureFunctions();
|
||||||
for(size_t i=0; i<producers.size(); ++i) {
|
for(size_t i=0; i<producers.size(); ++i) {
|
||||||
FeatureFunction &ff = *producers[i];
|
FeatureFunction &ff = *producers[i];
|
||||||
if (! IsFeatureFunctionIgnored(ff)) {
|
if (! IsFeatureFunctionIgnored(ff)) {
|
||||||
|
@ -470,15 +470,15 @@ public:
|
|||||||
// m_outputSearchGraph = outputSearchGraph;
|
// m_outputSearchGraph = outputSearchGraph;
|
||||||
// }
|
// }
|
||||||
|
|
||||||
// bool GetOutputSearchGraphExtended() const {
|
// bool GetOutputSearchGraphExtended() const {
|
||||||
// return m_outputSearchGraphExtended;
|
// return m_outputSearchGraphExtended;
|
||||||
// }
|
// }
|
||||||
// GetOutputSearchGraphSLF() const {
|
// GetOutputSearchGraphSLF() const {
|
||||||
// return m_outputSearchGraphSLF;
|
// return m_outputSearchGraphSLF;
|
||||||
// }
|
// }
|
||||||
// bool GetOutputSearchGraphHypergraph() const {
|
// bool GetOutputSearchGraphHypergraph() const {
|
||||||
// return m_outputSearchGraphHypergraph;
|
// return m_outputSearchGraphHypergraph;
|
||||||
// }
|
// }
|
||||||
|
|
||||||
// #ifdef HAVE_PROTOBUF
|
// #ifdef HAVE_PROTOBUF
|
||||||
// bool GetOutputSearchGraphPB() const {
|
// bool GetOutputSearchGraphPB() const {
|
||||||
|
@ -45,9 +45,9 @@ void TabbedSentence::CreateFromString(const std::vector<FactorType> &factorOrder
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
int
|
int
|
||||||
TabbedSentence::
|
TabbedSentence::
|
||||||
Read(std::istream& in,
|
Read(std::istream& in,
|
||||||
std::vector<FactorType> const& factorOrder,
|
std::vector<FactorType> const& factorOrder,
|
||||||
AllOptions const& opts)
|
AllOptions const& opts)
|
||||||
{
|
{
|
||||||
|
@ -67,7 +67,7 @@ public:
|
|||||||
virtual void CreateFromString(const std::vector<FactorType> &factorOrder
|
virtual void CreateFromString(const std::vector<FactorType> &factorOrder
|
||||||
, const std::string &tabbedString);
|
, const std::string &tabbedString);
|
||||||
|
|
||||||
virtual int
|
virtual int
|
||||||
Read(std::istream& in,const std::vector<FactorType>& factorOrder,
|
Read(std::istream& in,const std::vector<FactorType>& factorOrder,
|
||||||
AllOptions const& opts);
|
AllOptions const& opts);
|
||||||
|
|
||||||
|
@ -234,10 +234,10 @@ CreateTranslationOptionsForRangeLEGACY(const DecodeGraph &decodeGraph, size_t st
|
|||||||
list <const DecodeStep* >::const_iterator iterStep = decodeGraph.begin();
|
list <const DecodeStep* >::const_iterator iterStep = decodeGraph.begin();
|
||||||
const DecodeStep &decodeStep = **iterStep;
|
const DecodeStep &decodeStep = **iterStep;
|
||||||
|
|
||||||
DecodeStepTranslation const& dstep
|
DecodeStepTranslation const& dstep
|
||||||
= static_cast<const DecodeStepTranslation&>(decodeStep);
|
= static_cast<const DecodeStepTranslation&>(decodeStep);
|
||||||
dstep.ProcessInitialTransLEGACY(m_source, *oldPtoc, startPos, endPos,
|
dstep.ProcessInitialTransLEGACY(m_source, *oldPtoc, startPos, endPos,
|
||||||
adhereTableLimit, inputPathList);
|
adhereTableLimit, inputPathList);
|
||||||
|
|
||||||
// do rest of decode steps
|
// do rest of decode steps
|
||||||
int indexStep = 0;
|
int indexStep = 0;
|
||||||
|
@ -186,8 +186,8 @@ void TranslationTask::Run()
|
|||||||
|
|
||||||
// report thread number
|
// report thread number
|
||||||
#if defined(WITH_THREADS) && defined(BOOST_HAS_PTHREADS)
|
#if defined(WITH_THREADS) && defined(BOOST_HAS_PTHREADS)
|
||||||
VERBOSE(2, "Translating line " << translationId << " in thread id "
|
VERBOSE(2, "Translating line " << translationId << " in thread id "
|
||||||
<< pthread_self() << endl);
|
<< pthread_self() << endl);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
||||||
|
@ -237,7 +237,7 @@ bool TreeInput::ProcessAndStripXMLTags(string &line, std::vector<XMLParseOutput>
|
|||||||
}
|
}
|
||||||
|
|
||||||
//! populate this InputType with data from in stream
|
//! populate this InputType with data from in stream
|
||||||
int
|
int
|
||||||
TreeInput::
|
TreeInput::
|
||||||
Read(std::istream& in, const std::vector<FactorType>& factorOrder,
|
Read(std::istream& in, const std::vector<FactorType>& factorOrder,
|
||||||
AllOptions const& opts)
|
AllOptions const& opts)
|
||||||
@ -258,7 +258,7 @@ Read(std::istream& in, const std::vector<FactorType>& factorOrder,
|
|||||||
strme << line << endl;
|
strme << line << endl;
|
||||||
|
|
||||||
Sentence::Read(strme, factorOrder, opts);
|
Sentence::Read(strme, factorOrder, opts);
|
||||||
|
|
||||||
// size input chart
|
// size input chart
|
||||||
size_t sourceSize = GetSize();
|
size_t sourceSize = GetSize();
|
||||||
m_sourceChart.resize(sourceSize);
|
m_sourceChart.resize(sourceSize);
|
||||||
|
@ -53,8 +53,8 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
//! populate this InputType with data from in stream
|
//! populate this InputType with data from in stream
|
||||||
virtual int
|
virtual int
|
||||||
Read(std::istream& in,
|
Read(std::istream& in,
|
||||||
const std::vector<FactorType>& factorOrder,
|
const std::vector<FactorType>& factorOrder,
|
||||||
AllOptions const& opts);
|
AllOptions const& opts);
|
||||||
|
|
||||||
|
@ -147,9 +147,9 @@ InitializeFromPCNDataType
|
|||||||
return !cn.empty();
|
return !cn.empty();
|
||||||
}
|
}
|
||||||
|
|
||||||
int
|
int
|
||||||
WordLattice::
|
WordLattice::
|
||||||
Read(std::istream& in,
|
Read(std::istream& in,
|
||||||
std::vector<FactorType> const& factorOrder,
|
std::vector<FactorType> const& factorOrder,
|
||||||
AllOptions const& opts)
|
AllOptions const& opts)
|
||||||
{
|
{
|
||||||
|
@ -43,10 +43,10 @@ public:
|
|||||||
int InitializeFromPCNDataType(const PCN::CN& cn, const std::vector<FactorType>& factorOrder, const std::string& debug_line = "");
|
int InitializeFromPCNDataType(const PCN::CN& cn, const std::vector<FactorType>& factorOrder, const std::string& debug_line = "");
|
||||||
/** Read from PLF format (1 lattice per line)
|
/** Read from PLF format (1 lattice per line)
|
||||||
*/
|
*/
|
||||||
int Read(std::istream& in,
|
int Read(std::istream& in,
|
||||||
std::vector<FactorType> const& factorOrder,
|
std::vector<FactorType> const& factorOrder,
|
||||||
AllOptions const& opts);
|
AllOptions const& opts);
|
||||||
|
|
||||||
/** Convert internal representation into an edge matrix
|
/** Convert internal representation into an edge matrix
|
||||||
* @note edges[1][2] means there is an edge from 1 to 2
|
* @note edges[1][2] means there is an edge from 1 to 2
|
||||||
*/
|
*/
|
||||||
|
Loading…
Reference in New Issue
Block a user