mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
Work Master all the way up to main
This commit is contained in:
parent
e5c8cbd0c1
commit
6f9db664a5
2
lazy
2
lazy
@ -1 +1 @@
|
||||
Subproject commit 02625f416da85bad8641437d553f350f96ed4559
|
||||
Subproject commit a66f6cd85d3705649d713a51b6ed4ab946dd2d60
|
@ -89,6 +89,8 @@ public:
|
||||
void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
||||
void Backtrack(const Moses::ChartHypothesis *hypo);
|
||||
|
||||
Moses::OutputCollector *ExposeSingleBest() { return m_singleBestOutputCollector; }
|
||||
|
||||
void ResetTranslationId();
|
||||
|
||||
Moses::OutputCollector *GetSearchGraphOutputCollector() {
|
||||
|
@ -57,6 +57,7 @@ POSSIBILITY OF SUCH DAMAGE.
|
||||
#include "ChartHypothesis.h"
|
||||
#include "ChartTrellisPath.h"
|
||||
#include "ChartTrellisPathList.h"
|
||||
#include "Incremental/Manager.h"
|
||||
|
||||
#include "util/usage.hh"
|
||||
|
||||
@ -86,6 +87,15 @@ public:
|
||||
|
||||
VERBOSE(2,"\nTRANSLATING(" << lineNumber << "): " << *m_source);
|
||||
|
||||
if (staticData.GetSearchAlgorithm() == ChartIncremental) {
|
||||
Incremental::Manager manager(*m_source, system);
|
||||
manager.ProcessSentence();
|
||||
if (m_ioWrapper.ExposeSingleBest()) {
|
||||
m_ioWrapper.ExposeSingleBest()->Write(lineNumber, manager.String());
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
ChartManager manager(*m_source, &system);
|
||||
manager.ProcessSentence();
|
||||
|
||||
|
@ -56,15 +56,15 @@ class ChartCellCollectionBase {
|
||||
return *m_cells[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()];
|
||||
}
|
||||
|
||||
const ChartCellLabel &GetSourceWordLabel(size_t at) const {
|
||||
return m_source[at];
|
||||
}
|
||||
|
||||
// There's nothing mutable in ChartCellLabel base, so no public method.
|
||||
ChartCellBase &MutableBase(const WordsRange &coverage) {
|
||||
return *m_cells[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()];
|
||||
}
|
||||
|
||||
|
||||
const ChartCellLabel &GetSourceWordLabel(size_t at) const {
|
||||
return m_source[at];
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<std::vector<ChartCellBase*> > m_cells;
|
||||
|
||||
|
@ -43,7 +43,7 @@ template <class Model> void Fill<Model>::Add(const TargetPhraseCollection &targe
|
||||
for (; i < phrase.GetSize(); ++i) {
|
||||
const Word &word = phrase.GetWord(i);
|
||||
if (word.IsNonTerminal()) {
|
||||
words.push_back(search::Rule::kNonTerminal);
|
||||
words.push_back(search::kNonTerminal);
|
||||
} else {
|
||||
words.push_back(Convert(word));
|
||||
}
|
||||
@ -102,8 +102,12 @@ template <class Model> lm::WordIndex Fill<Model>::Convert(const Word &word) cons
|
||||
return (factor >= vocab_mapping_.size() ? 0 : vocab_mapping_[factor]);
|
||||
}
|
||||
|
||||
template class Fill<lm::ngram::RestProbingModel>;
|
||||
template class Fill<lm::ngram::ProbingModel>;
|
||||
template class Fill<lm::ngram::RestProbingModel>;
|
||||
template class Fill<lm::ngram::TrieModel>;
|
||||
template class Fill<lm::ngram::QuantTrieModel>;
|
||||
template class Fill<lm::ngram::ArrayTrieModel>;
|
||||
template class Fill<lm::ngram::QuantArrayTrieModel>;
|
||||
|
||||
} // namespace Incremental
|
||||
} // namespace Moses
|
||||
|
@ -1,5 +1,6 @@
|
||||
#include "Incremental/Manager.h"
|
||||
|
||||
#include "Incremental/Edge.h"
|
||||
#include "Incremental/Fill.h"
|
||||
|
||||
#include "ChartCell.h"
|
||||
@ -40,7 +41,7 @@ template <class Model> void Manager::LMCallback(const Model &model, const std::v
|
||||
abstract.GetWeight(),
|
||||
abstract.OOVFeatureEnabled() ? abstract.GetOOVWeight() : 0.0,
|
||||
system_.GetWeightWordPenalty());
|
||||
search::Config config(weights, StaticData::GetCubePruningPopLimit());
|
||||
search::Config config(weights, StaticData::Instance().GetCubePruningPopLimit());
|
||||
search::Context<Model> context(config, model);
|
||||
|
||||
size_t size = source_.GetSize();
|
||||
@ -49,16 +50,56 @@ template <class Model> void Manager::LMCallback(const Model &model, const std::v
|
||||
size_t endPos = startPos + width - 1;
|
||||
WordsRange range(startPos, endPos);
|
||||
Fill<Model> filler(context, words, owner_);
|
||||
parser_.Create(WordsRange(startPos, endPos), filler);
|
||||
parser_.Create(range, filler);
|
||||
filler.Search(cells_.MutableBase(range).MutableTargetLabelSet());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template void Manager::LMCallback<lm::ngram::ProbingModel>(const lm::ngram::ProbingModel &model, const std::vector<lm::WordIndex> &words);
|
||||
template void Manager::LMCallback<lm::ngram::RestProbingModel>(const lm::ngram::RestProbingModel &model, const std::vector<lm::WordIndex> &words);
|
||||
template void Manager::LMCallback<lm::ngram::TrieModel>(const lm::ngram::TrieModel &model, const std::vector<lm::WordIndex> &words);
|
||||
template void Manager::LMCallback<lm::ngram::QuantTrieModel>(const lm::ngram::QuantTrieModel &model, const std::vector<lm::WordIndex> &words);
|
||||
template void Manager::LMCallback<lm::ngram::ArrayTrieModel>(const lm::ngram::ArrayTrieModel &model, const std::vector<lm::WordIndex> &words);
|
||||
template void Manager::LMCallback<lm::ngram::QuantArrayTrieModel>(const lm::ngram::QuantArrayTrieModel &model, const std::vector<lm::WordIndex> &words);
|
||||
|
||||
namespace {
|
||||
|
||||
void ConstructString(const search::Final &final, std::ostringstream &stream) {
|
||||
const TargetPhrase &phrase = static_cast<const Edge&>(final.From()).GetMoses();
|
||||
size_t child = 0;
|
||||
for (std::size_t i = 0; i < phrase.GetSize(); ++i) {
|
||||
const Word &word = phrase.GetWord(i);
|
||||
if (word.IsNonTerminal()) {
|
||||
ConstructString(*final.Children()[child++], stream);
|
||||
} else {
|
||||
stream << word[0]->GetString() << ' ';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void Manager::ProcessSentence() {
|
||||
const LMList &lms = system_.GetLanguageModels();
|
||||
UTIL_THROW_IF(lms.size() != 1, util::Exception, "Incremental search only supports one language model.");
|
||||
(*lms.begin())->IncrementalCallback(*this);
|
||||
|
||||
const ChartCellLabelSet &labels = cells_.GetBase(WordsRange(0, source_.GetSize() - 1)).GetTargetLabelSet();
|
||||
const search::Final *best = NULL;
|
||||
for (ChartCellLabelSet::const_iterator i = labels.begin(); i != labels.end(); ++i) {
|
||||
const search::Final *child = i->second.GetStack().incr->BestChild();
|
||||
if (child && (!best || (child->Bound() > best->Bound()))) {
|
||||
best = child;
|
||||
}
|
||||
}
|
||||
if (!best) {
|
||||
output_.clear();
|
||||
return;
|
||||
}
|
||||
std::ostringstream stream;
|
||||
ConstructString(*best, stream);
|
||||
output_ = stream.str();
|
||||
}
|
||||
|
||||
} // namespace Incremental
|
||||
|
@ -21,12 +21,16 @@ class Manager {
|
||||
|
||||
void ProcessSentence();
|
||||
|
||||
const std::string &String() const { return output_; }
|
||||
|
||||
private:
|
||||
const InputType &source_;
|
||||
const TranslationSystem &system_;
|
||||
ChartCellCollectionBase cells_;
|
||||
ChartParser parser_;
|
||||
Owner owner_;
|
||||
|
||||
std::string output_;
|
||||
};
|
||||
} // namespace Incremental
|
||||
} // namespace Moses
|
||||
|
@ -88,7 +88,7 @@ template <class Model> class LanguageModelKen : public LanguageModel {
|
||||
|
||||
FFState *EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const;
|
||||
|
||||
void IncrementalCallback(Incremental::Manager &manager) {
|
||||
void IncrementalCallback(Incremental::Manager &manager) const {
|
||||
manager.LMCallback(*m_ngram, m_lmIdLookup);
|
||||
}
|
||||
|
||||
|
@ -173,6 +173,7 @@ enum SearchAlgorithm {
|
||||
,CubeGrowing = 2
|
||||
,ChartDecoding= 3
|
||||
,NormalBatch = 4
|
||||
,ChartIncremental = 5
|
||||
};
|
||||
|
||||
enum SourceLabelOverlap {
|
||||
|
Loading…
Reference in New Issue
Block a user