mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
Expose k-best lists from incremental search
This commit is contained in:
Kenneth Heafield
parent
6a3fdcd8c0
commit
2df57b6b09
@ -40,6 +40,7 @@ POSSIBILITY OF SUCH DAMAGE.
|
||||
#include "moses/StaticData.h"
|
||||
#include "moses/DummyScoreProducers.h"
|
||||
#include "moses/InputFileStream.h"
|
||||
#include "moses/Incremental.h"
|
||||
#include "moses/PhraseDictionary.h"
|
||||
#include "moses/ChartTrellisPathList.h"
|
||||
#include "moses/ChartTrellisPath.h"
|
||||
@ -366,8 +367,136 @@ void IOWrapper::OutputBestHypo(const ChartHypothesis *hypo, long translationId)
|
||||
m_singleBestOutputCollector->Write(translationId, out.str());
|
||||
}
|
||||
|
||||
void IOWrapper::OutputNBestList(const ChartTrellisPathList &nBestList, const ChartHypothesis *bestHypo, const TranslationSystem* system, long translationId)
|
||||
{
|
||||
void IOWrapper::OutputBestHypo(search::Applied applied, long translationId) {
|
||||
if (!m_singleBestOutputCollector) return;
|
||||
std::ostringstream out;
|
||||
IOWrapper::FixPrecision(out);
|
||||
if (StaticData::Instance().GetOutputHypoScore()) {
|
||||
out << applied.GetScore() << ' ';
|
||||
}
|
||||
Phrase outPhrase;
|
||||
Incremental::ToPhrase(applied, outPhrase);
|
||||
// delete 1st & last
|
||||
CHECK(outPhrase.GetSize() >= 2);
|
||||
outPhrase.RemoveWord(0);
|
||||
outPhrase.RemoveWord(outPhrase.GetSize() - 1);
|
||||
out << outPhrase.GetStringRep(StaticData::Instance().GetOutputFactorOrder());
|
||||
out << '\n';
|
||||
m_singleBestOutputCollector->Write(translationId, out.str());
|
||||
}
|
||||
|
||||
void IOWrapper::OutputBestNone(long translationId) {
|
||||
if (!m_singleBestOutputCollector) return;
|
||||
if (StaticData::Instance().GetOutputHypoScore()) {
|
||||
m_singleBestOutputCollector->Write(translationId, "0 \n");
|
||||
} else {
|
||||
m_singleBestOutputCollector->Write(translationId, "\n");
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
void OutputSparseFeatureScores(std::ostream& out, const ScoreComponentCollection &features, const FeatureFunction *ff, std::string &lastName) {
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
bool labeledOutput = staticData.IsLabeledNBestList();
|
||||
const FVector scores = features.GetVectorForProducer( ff );
|
||||
|
||||
// report weighted aggregate
|
||||
if (! ff->GetSparseFeatureReporting()) {
|
||||
const FVector &weights = staticData.GetAllWeights().GetScoresVector();
|
||||
if (labeledOutput && !boost::contains(ff->GetScoreProducerDescription(), ":"))
|
||||
out << " " << ff->GetScoreProducerWeightShortName() << ":";
|
||||
out << " " << scores.inner_product(weights);
|
||||
}
|
||||
|
||||
// report each feature
|
||||
else {
|
||||
for(FVector::FNVmap::const_iterator i = scores.cbegin(); i != scores.cend(); i++) {
|
||||
if (i->second != 0) { // do not report zero-valued features
|
||||
if (labeledOutput)
|
||||
out << " " << i->first << ":";
|
||||
out << " " << i->second;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void WriteFeatures(const TranslationSystem &system, const ScoreComponentCollection &features, std::ostream &out) {
|
||||
bool labeledOutput = StaticData::Instance().IsLabeledNBestList();
|
||||
// lm
|
||||
const LMList& lml = system.GetLanguageModels();
|
||||
if (lml.size() > 0) {
|
||||
if (labeledOutput)
|
||||
out << "lm:";
|
||||
LMList::const_iterator lmi = lml.begin();
|
||||
for (; lmi != lml.end(); ++lmi) {
|
||||
out << " " << features.GetScoreForProducer(*lmi);
|
||||
}
|
||||
}
|
||||
|
||||
std::string lastName = "";
|
||||
|
||||
// output stateful sparse features
|
||||
const vector<const StatefulFeatureFunction*>& sff = system.GetStatefulFeatureFunctions();
|
||||
for( size_t i=0; i<sff.size(); i++ )
|
||||
if (sff[i]->GetNumScoreComponents() == ScoreProducer::unlimited)
|
||||
OutputSparseFeatureScores(out, features, sff[i], lastName);
|
||||
|
||||
// translation components
|
||||
const vector<PhraseDictionaryFeature*>& pds = system.GetPhraseDictionaries();
|
||||
if (pds.size() > 0) {
|
||||
for( size_t i=0; i<pds.size(); i++ ) {
|
||||
size_t pd_numinputscore = pds[i]->GetNumInputScores();
|
||||
vector<float> scores = features.GetScoresForProducer( pds[i] );
|
||||
for (size_t j = 0; j<scores.size(); ++j){
|
||||
if (labeledOutput && (i == 0) ){
|
||||
if ((j == 0) || (j == pd_numinputscore)){
|
||||
lastName = pds[i]->GetScoreProducerWeightShortName(j);
|
||||
out << " " << lastName << ":";
|
||||
}
|
||||
}
|
||||
out << " " << scores[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// word penalty
|
||||
if (labeledOutput)
|
||||
out << " w:";
|
||||
out << " " << features.GetScoreForProducer(system.GetWordPenaltyProducer());
|
||||
|
||||
// generation
|
||||
const vector<GenerationDictionary*>& gds = system.GetGenerationDictionaries();
|
||||
if (gds.size() > 0) {
|
||||
for( size_t i=0; i<gds.size(); i++ ) {
|
||||
size_t pd_numinputscore = gds[i]->GetNumInputScores();
|
||||
vector<float> scores = features.GetScoresForProducer( gds[i] );
|
||||
for (size_t j = 0; j<scores.size(); ++j){
|
||||
if (labeledOutput && (i == 0) ){
|
||||
if ((j == 0) || (j == pd_numinputscore)){
|
||||
lastName = gds[i]->GetScoreProducerWeightShortName(j);
|
||||
out << " " << lastName << ":";
|
||||
}
|
||||
}
|
||||
out << " " << scores[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// output stateless sparse features
|
||||
lastName = "";
|
||||
|
||||
const vector<const StatelessFeatureFunction*>& slf = system.GetStatelessFeatureFunctions();
|
||||
for( size_t i=0; i<slf.size(); i++ ) {
|
||||
if (slf[i]->GetNumScoreComponents() == ScoreProducer::unlimited) {
|
||||
OutputSparseFeatureScores(out, features, slf[i], lastName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void IOWrapper::OutputNBestList(const ChartTrellisPathList &nBestList, const TranslationSystem* system, long translationId) {
|
||||
std::ostringstream out;
|
||||
|
||||
// Check if we're writing to std::cout.
|
||||
@ -376,17 +505,9 @@ void IOWrapper::OutputNBestList(const ChartTrellisPathList &nBestList, const Cha
|
||||
// preserve existing behaviour, but should probably be done either way.
|
||||
IOWrapper::FixPrecision(out);
|
||||
|
||||
// The output from -output-hypo-score is always written to std::cout.
|
||||
if (StaticData::Instance().GetOutputHypoScore()) {
|
||||
if (bestHypo != NULL) {
|
||||
out << bestHypo->GetTotalScore() << " ";
|
||||
} else {
|
||||
out << "0 ";
|
||||
}
|
||||
}
|
||||
// Used to check StaticData's GetOutputHypoScore(), but it makes no sense with nbest output.
|
||||
}
|
||||
|
||||
bool labeledOutput = StaticData::Instance().IsLabeledNBestList();
|
||||
//bool includeAlignment = StaticData::Instance().NBestIncludesAlignment();
|
||||
|
||||
ChartTrellisPathList::const_iterator iter;
|
||||
@ -410,75 +531,7 @@ void IOWrapper::OutputNBestList(const ChartTrellisPathList &nBestList, const Cha
|
||||
// before each model type, the corresponding command-line-like name must be emitted
|
||||
// MERT script relies on this
|
||||
|
||||
// lm
|
||||
const LMList& lml = system->GetLanguageModels();
|
||||
if (lml.size() > 0) {
|
||||
if (labeledOutput)
|
||||
out << "lm:";
|
||||
LMList::const_iterator lmi = lml.begin();
|
||||
for (; lmi != lml.end(); ++lmi) {
|
||||
out << " " << path.GetScoreBreakdown().GetScoreForProducer(*lmi);
|
||||
}
|
||||
}
|
||||
|
||||
std::string lastName = "";
|
||||
|
||||
// output stateful sparse features
|
||||
const vector<const StatefulFeatureFunction*>& sff = system->GetStatefulFeatureFunctions();
|
||||
for( size_t i=0; i<sff.size(); i++ )
|
||||
if (sff[i]->GetNumScoreComponents() == ScoreProducer::unlimited)
|
||||
OutputSparseFeatureScores( out, path, sff[i], lastName );
|
||||
|
||||
// translation components
|
||||
const vector<PhraseDictionaryFeature*>& pds = system->GetPhraseDictionaries();
|
||||
if (pds.size() > 0) {
|
||||
for( size_t i=0; i<pds.size(); i++ ) {
|
||||
size_t pd_numinputscore = pds[i]->GetNumInputScores();
|
||||
vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( pds[i] );
|
||||
for (size_t j = 0; j<scores.size(); ++j){
|
||||
if (labeledOutput && (i == 0) ){
|
||||
if ((j == 0) || (j == pd_numinputscore)){
|
||||
lastName = pds[i]->GetScoreProducerWeightShortName(j);
|
||||
out << " " << lastName << ":";
|
||||
}
|
||||
}
|
||||
out << " " << scores[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// word penalty
|
||||
if (labeledOutput)
|
||||
out << " w:";
|
||||
out << " " << path.GetScoreBreakdown().GetScoreForProducer(system->GetWordPenaltyProducer());
|
||||
|
||||
// generation
|
||||
const vector<GenerationDictionary*>& gds = system->GetGenerationDictionaries();
|
||||
if (gds.size() > 0) {
|
||||
for( size_t i=0; i<gds.size(); i++ ) {
|
||||
size_t pd_numinputscore = gds[i]->GetNumInputScores();
|
||||
vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( gds[i] );
|
||||
for (size_t j = 0; j<scores.size(); ++j){
|
||||
if (labeledOutput && (i == 0) ){
|
||||
if ((j == 0) || (j == pd_numinputscore)){
|
||||
lastName = gds[i]->GetScoreProducerWeightShortName(j);
|
||||
out << " " << lastName << ":";
|
||||
}
|
||||
}
|
||||
out << " " << scores[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// output stateless sparse features
|
||||
lastName = "";
|
||||
|
||||
const vector<const StatelessFeatureFunction*>& slf = system->GetStatelessFeatureFunctions();
|
||||
for( size_t i=0; i<slf.size(); i++ ) {
|
||||
if (slf[i]->GetNumScoreComponents() == ScoreProducer::unlimited) {
|
||||
OutputSparseFeatureScores( out, path, slf[i], lastName );
|
||||
}
|
||||
}
|
||||
WriteFeatures(*system, path.GetScoreBreakdown(), out);
|
||||
|
||||
// total
|
||||
out << " ||| " << path.GetTotalScore();
|
||||
@ -508,34 +561,33 @@ void IOWrapper::OutputNBestList(const ChartTrellisPathList &nBestList, const Cha
|
||||
|
||||
out <<std::flush;
|
||||
|
||||
CHECK(m_nBestOutputCollector);
|
||||
assert(m_nBestOutputCollector);
|
||||
m_nBestOutputCollector->Write(translationId, out.str());
|
||||
}
|
||||
|
||||
void IOWrapper::OutputSparseFeatureScores( std::ostream& out, const ChartTrellisPath &path, const FeatureFunction *ff, std::string &lastName )
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
bool labeledOutput = staticData.IsLabeledNBestList();
|
||||
const FVector scores = path.GetScoreBreakdown().GetVectorForProducer( ff );
|
||||
|
||||
// report weighted aggregate
|
||||
if (! ff->GetSparseFeatureReporting()) {
|
||||
const FVector &weights = staticData.GetAllWeights().GetScoresVector();
|
||||
if (labeledOutput && !boost::contains(ff->GetScoreProducerDescription(), ":"))
|
||||
out << " " << ff->GetScoreProducerWeightShortName() << ":";
|
||||
out << " " << scores.inner_product(weights);
|
||||
void IOWrapper::OutputNBestList(const std::vector<search::Applied> &nbest, const TranslationSystem &system, long translationId) {
|
||||
std::ostringstream out;
|
||||
// wtf? copied from the original OutputNBestList
|
||||
if (m_nBestOutputCollector->OutputIsCout()) {
|
||||
IOWrapper::FixPrecision(out);
|
||||
}
|
||||
|
||||
// report each feature
|
||||
else {
|
||||
for(FVector::FNVmap::const_iterator i = scores.cbegin(); i != scores.cend(); i++) {
|
||||
if (i->second != 0) { // do not report zero-valued features
|
||||
if (labeledOutput)
|
||||
out << " " << i->first << ":";
|
||||
out << " " << i->second;
|
||||
}
|
||||
}
|
||||
Phrase outputPhrase;
|
||||
ScoreComponentCollection features;
|
||||
for (std::vector<search::Applied>::const_iterator i = nbest.begin(); i != nbest.end(); ++i) {
|
||||
Incremental::PhraseAndFeatures(system, *i, outputPhrase, features);
|
||||
// <s> and </s>
|
||||
CHECK(outputPhrase.GetSize() >= 2);
|
||||
outputPhrase.RemoveWord(0);
|
||||
outputPhrase.RemoveWord(outputPhrase.GetSize() - 1);
|
||||
out << translationId << " ||| ";
|
||||
OutputSurface(out, outputPhrase, m_outputFactorOrder, false);
|
||||
out << " ||| ";
|
||||
WriteFeatures(system, features, out);
|
||||
out << " ||| " << i->GetScore() << '\n';
|
||||
}
|
||||
out << std::flush;
|
||||
assert(m_nBestOutputCollector);
|
||||
m_nBestOutputCollector->Write(translationId, out.str());
|
||||
}
|
||||
|
||||
void IOWrapper::FixPrecision(std::ostream &stream, size_t size)
|
||||
|
||||
@ -44,6 +44,7 @@ POSSIBILITY OF SUCH DAMAGE.
|
||||
#include "moses/OutputCollector.h"
|
||||
#include "moses/ChartHypothesis.h"
|
||||
#include "moses/ChartTrellisPath.h"
|
||||
#include "search/applied.hh"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
@ -85,14 +86,14 @@ public:
|
||||
|
||||
Moses::InputType* GetInput(Moses::InputType *inputType);
|
||||
void OutputBestHypo(const Moses::ChartHypothesis *hypo, long translationId);
|
||||
void OutputBestHypo(search::Applied applied, long translationId);
|
||||
void OutputBestHypo(const std::vector<const Moses::Factor*>& mbrBestHypo, long translationId);
|
||||
void OutputNBestList(const Moses::ChartTrellisPathList &nBestList, const Moses::ChartHypothesis *bestHypo, const Moses::TranslationSystem* system, long translationId);
|
||||
void OutputSparseFeatureScores(std::ostream& out, const Moses::ChartTrellisPath &path, const Moses::FeatureFunction *ff, std::string &lastName);
|
||||
void OutputBestNone(long translationId);
|
||||
void OutputNBestList(const Moses::ChartTrellisPathList &nBestList, const Moses::TranslationSystem* system, long translationId);
|
||||
void OutputNBestList(const std::vector<search::Applied> &nbest, const Moses::TranslationSystem &system, long translationId);
|
||||
void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
||||
void Backtrack(const Moses::ChartHypothesis *hypo);
|
||||
|
||||
Moses::OutputCollector *ExposeSingleBest() { return m_singleBestOutputCollector; }
|
||||
|
||||
void ResetTranslationId();
|
||||
|
||||
Moses::OutputCollector *GetSearchGraphOutputCollector() {
|
||||
|
||||
@ -91,10 +91,14 @@ public:
|
||||
|
||||
if (staticData.GetSearchAlgorithm() == ChartIncremental) {
|
||||
Incremental::Manager manager(*m_source, system);
|
||||
manager.ProcessSentence();
|
||||
if (m_ioWrapper.ExposeSingleBest()) {
|
||||
m_ioWrapper.ExposeSingleBest()->Write(lineNumber, manager.String() + '\n');
|
||||
const std::vector<search::Applied> &nbest = manager.ProcessSentence();
|
||||
if (!nbest.empty()) {
|
||||
m_ioWrapper.OutputBestHypo(nbest[0], lineNumber);
|
||||
} else {
|
||||
m_ioWrapper.OutputBestNone(lineNumber);
|
||||
}
|
||||
if (staticData.GetNBestSize() > 0)
|
||||
m_ioWrapper.OutputNBestList(nbest, system, lineNumber);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -121,7 +125,7 @@ public:
|
||||
VERBOSE(2,"WRITING " << nBestSize << " TRANSLATION ALTERNATIVES TO " << staticData.GetNBestFilePath() << endl);
|
||||
ChartTrellisPathList nBestList;
|
||||
manager.CalcNBest(nBestSize, nBestList,staticData.GetDistinctNBest());
|
||||
m_ioWrapper.OutputNBestList(nBestList, bestHypo, &system, lineNumber);
|
||||
m_ioWrapper.OutputNBestList(nBestList, &system, lineNumber);
|
||||
IFVERBOSE(2) {
|
||||
PrintUserTime("N-Best Hypotheses Generation Time:");
|
||||
}
|
||||
|
||||
@ -2,8 +2,10 @@
|
||||
|
||||
#include "moses/ChartCell.h"
|
||||
#include "moses/ChartParserCallback.h"
|
||||
#include "moses/TranslationSystem.h"
|
||||
#include "moses/FeatureVector.h"
|
||||
#include "moses/StaticData.h"
|
||||
#include "moses/TranslationSystem.h"
|
||||
#include "moses/Util.h"
|
||||
|
||||
#include "lm/model.hh"
|
||||
#include "search/applied.hh"
|
||||
@ -78,6 +80,14 @@ template <class Model> class Fill : public ChartParserCallback {
|
||||
edges_.Search(context_, callback);
|
||||
}
|
||||
|
||||
// Root: everything into one vertex.
|
||||
template <class Best> search::History RootSearch(Best &best) {
|
||||
search::Vertex vertex;
|
||||
search::RootVertexGenerator<Best> gen(vertex, best);
|
||||
edges_.Search(context_, gen);
|
||||
return vertex.BestChild();
|
||||
}
|
||||
|
||||
private:
|
||||
lm::WordIndex Convert(const Word &word) const;
|
||||
|
||||
@ -166,80 +176,58 @@ Manager::Manager(const InputType &source, const TranslationSystem &system) :
|
||||
source_(source),
|
||||
system_(system),
|
||||
cells_(source, ChartCellBaseFactory()),
|
||||
parser_(source, system, cells_) {}
|
||||
parser_(source, system, cells_),
|
||||
n_best_(search::NBestConfig(StaticData::Instance().GetNBestSize())) {}
|
||||
|
||||
Manager::~Manager() {
|
||||
system_.CleanUpAfterSentenceProcessing(source_);
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
void ConstructString(const search::Applied final, std::ostringstream &stream) {
|
||||
assert(final.Valid());
|
||||
const TargetPhrase &phrase = *static_cast<const TargetPhrase*>(final.GetNote().vp);
|
||||
size_t child = 0;
|
||||
for (std::size_t i = 0; i < phrase.GetSize(); ++i) {
|
||||
const Word &word = phrase.GetWord(i);
|
||||
if (word.IsNonTerminal()) {
|
||||
assert(child < final.GetArity());
|
||||
ConstructString(final.Children()[child++], stream);
|
||||
} else {
|
||||
stream << word[0]->GetString() << ' ';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void BestString(const ChartCellLabelSet &labels, std::string &out) {
|
||||
search::Applied best;
|
||||
for (ChartCellLabelSet::const_iterator i = labels.begin(); i != labels.end(); ++i) {
|
||||
const search::Applied child(i->second.GetStack().incr->BestChild());
|
||||
if (child.Valid() && (!best.Valid() || (child.GetScore() > best.GetScore()))) {
|
||||
best = child;
|
||||
}
|
||||
}
|
||||
if (!best.Valid()) {
|
||||
out.clear();
|
||||
return;
|
||||
}
|
||||
std::ostringstream stream;
|
||||
ConstructString(best, stream);
|
||||
out = stream.str();
|
||||
CHECK(out.size() > 9);
|
||||
// <s>
|
||||
out.erase(0, 4);
|
||||
// </s>
|
||||
out.erase(out.size() - 5);
|
||||
// Hack: include model score
|
||||
out += " ||| ";
|
||||
out += boost::lexical_cast<std::string>(best.GetScore());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
|
||||
template <class Model> void Manager::LMCallback(const Model &model, const std::vector<lm::WordIndex> &words) {
|
||||
template <class Model, class Best> search::History Manager::PopulateBest(const Model &model, const std::vector<lm::WordIndex> &words, Best &out) {
|
||||
const LanguageModel &abstract = **system_.GetLanguageModels().begin();
|
||||
const float oov_weight = abstract.OOVFeatureEnabled() ? abstract.GetOOVWeight() : 0.0;
|
||||
const StaticData &data = StaticData::Instance();
|
||||
search::Config config(abstract.GetWeight(), data.GetCubePruningPopLimit(), search::NBestConfig(data.GetNBestSize()));
|
||||
search::Context<Model> context(config, model);
|
||||
|
||||
search::SingleBest best;
|
||||
|
||||
size_t size = source_.GetSize();
|
||||
boost::object_pool<search::Vertex> vertex_pool(std::max<size_t>(size * size / 2, 32));
|
||||
|
||||
|
||||
for (size_t width = 1; width <= size; ++width) {
|
||||
for (size_t width = 1; width < size; ++width) {
|
||||
for (size_t startPos = 0; startPos <= size-width; ++startPos) {
|
||||
size_t endPos = startPos + width - 1;
|
||||
WordsRange range(startPos, endPos);
|
||||
WordsRange range(startPos, startPos + width - 1);
|
||||
Fill<Model> filler(context, words, oov_weight);
|
||||
parser_.Create(range, filler);
|
||||
filler.Search(best, cells_.MutableBase(range).MutableTargetLabelSet(), vertex_pool);
|
||||
filler.Search(out, cells_.MutableBase(range).MutableTargetLabelSet(), vertex_pool);
|
||||
}
|
||||
}
|
||||
|
||||
WordsRange range(0, size - 1);
|
||||
Fill<Model> filler(context, words, oov_weight);
|
||||
parser_.Create(range, filler);
|
||||
return filler.RootSearch(out);
|
||||
}
|
||||
|
||||
template <class Model> void Manager::LMCallback(const Model &model, const std::vector<lm::WordIndex> &words) {
|
||||
std::size_t nbest = StaticData::Instance().GetNBestSize();
|
||||
if (nbest <= 1) {
|
||||
search::History ret = PopulateBest(model, words, single_best_);
|
||||
if (ret) {
|
||||
backing_for_single_.resize(1);
|
||||
backing_for_single_[0] = search::Applied(ret);
|
||||
} else {
|
||||
backing_for_single_.clear();
|
||||
}
|
||||
completed_nbest_ = &backing_for_single_;
|
||||
} else {
|
||||
search::History ret = PopulateBest(model, words, n_best_);
|
||||
if (ret) {
|
||||
completed_nbest_ = &n_best_.Extract(ret);
|
||||
} else {
|
||||
backing_for_single_.clear();
|
||||
completed_nbest_ = &backing_for_single_;
|
||||
}
|
||||
}
|
||||
BestString(cells_.GetBase(WordsRange(0, source_.GetSize() - 1)).GetTargetLabelSet(), output_);
|
||||
}
|
||||
|
||||
template void Manager::LMCallback<lm::ngram::ProbingModel>(const lm::ngram::ProbingModel &model, const std::vector<lm::WordIndex> &words);
|
||||
@ -249,10 +237,59 @@ template void Manager::LMCallback<lm::ngram::QuantTrieModel>(const lm::ngram::Qu
|
||||
template void Manager::LMCallback<lm::ngram::ArrayTrieModel>(const lm::ngram::ArrayTrieModel &model, const std::vector<lm::WordIndex> &words);
|
||||
template void Manager::LMCallback<lm::ngram::QuantArrayTrieModel>(const lm::ngram::QuantArrayTrieModel &model, const std::vector<lm::WordIndex> &words);
|
||||
|
||||
void Manager::ProcessSentence() {
|
||||
const std::vector<search::Applied> &Manager::ProcessSentence() {
|
||||
const LMList &lms = system_.GetLanguageModels();
|
||||
UTIL_THROW_IF(lms.size() != 1, util::Exception, "Incremental search only supports one language model.");
|
||||
(*lms.begin())->IncrementalCallback(*this);
|
||||
return *completed_nbest_;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
struct NoOp {
|
||||
void operator()(const TargetPhrase &) const {}
|
||||
};
|
||||
struct AccumScore {
|
||||
AccumScore(ScoreComponentCollection &out) : out_(&out) {}
|
||||
void operator()(const TargetPhrase &phrase) {
|
||||
out_->PlusEquals(phrase.GetScoreBreakdown());
|
||||
}
|
||||
ScoreComponentCollection *out_;
|
||||
};
|
||||
template <class Action> void AppendToPhrase(const search::Applied final, Phrase &out, Action action) {
|
||||
assert(final.Valid());
|
||||
const TargetPhrase &phrase = *static_cast<const TargetPhrase*>(final.GetNote().vp);
|
||||
action(phrase);
|
||||
const search::Applied *child = final.Children();
|
||||
for (std::size_t i = 0; i < phrase.GetSize(); ++i) {
|
||||
const Word &word = phrase.GetWord(i);
|
||||
if (word.IsNonTerminal()) {
|
||||
AppendToPhrase(*child++, out, action);
|
||||
} else {
|
||||
out.AddWord(word);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void ToPhrase(const search::Applied final, Phrase &out) {
|
||||
out.Clear();
|
||||
AppendToPhrase(final, out, NoOp());
|
||||
}
|
||||
|
||||
void PhraseAndFeatures(const TranslationSystem &system, const search::Applied final, Phrase &phrase, ScoreComponentCollection &features) {
|
||||
phrase.Clear();
|
||||
features.ZeroAll();
|
||||
AppendToPhrase(final, phrase, AccumScore(features));
|
||||
|
||||
// If we made it this far, there is only one language model.
|
||||
float full, ignored_ngram;
|
||||
std::size_t ignored_oov;
|
||||
const LanguageModel &model = **system.GetLanguageModels().begin();
|
||||
model.CalcScore(phrase, full, ignored_ngram, ignored_oov);
|
||||
// CalcScore transforms, but EvaluateChart doesn't.
|
||||
features.Assign(&model, UntransformLMScore(full));
|
||||
}
|
||||
|
||||
} // namespace Incremental
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
#pragma once
|
||||
|
||||
#include "lm/word_index.hh"
|
||||
#include "search/applied.hh"
|
||||
#include "search/nbest.hh"
|
||||
|
||||
#include "moses/ChartCellCollection.h"
|
||||
#include "moses/ChartParser.h"
|
||||
@ -9,6 +11,7 @@
|
||||
#include <string>
|
||||
|
||||
namespace Moses {
|
||||
class ScoreComponentCollection;
|
||||
class InputType;
|
||||
class TranslationSystem;
|
||||
namespace Incremental {
|
||||
@ -21,18 +24,37 @@ class Manager {
|
||||
|
||||
template <class Model> void LMCallback(const Model &model, const std::vector<lm::WordIndex> &words);
|
||||
|
||||
void ProcessSentence();
|
||||
const std::vector<search::Applied> &ProcessSentence();
|
||||
|
||||
const std::string &String() const { return output_; }
|
||||
// Call to get the same value as ProcessSentence returned.
|
||||
const std::vector<search::Applied> &Completed() const {
|
||||
return *completed_nbest_;
|
||||
}
|
||||
|
||||
private:
|
||||
template <class Model, class Best> search::History PopulateBest(const Model &model, const std::vector<lm::WordIndex> &words, Best &out);
|
||||
|
||||
const InputType &source_;
|
||||
const TranslationSystem &system_;
|
||||
ChartCellCollectionBase cells_;
|
||||
ChartParser parser_;
|
||||
|
||||
std::string output_;
|
||||
// Only one of single_best_ or n_best_ will be used, but it was easier to do this than a template.
|
||||
search::SingleBest single_best_;
|
||||
// ProcessSentence returns a reference to a vector. ProcessSentence
|
||||
// doesn't have one, so this is populated and returned.
|
||||
std::vector<search::Applied> backing_for_single_;
|
||||
|
||||
search::NBest n_best_;
|
||||
|
||||
const std::vector<search::Applied> *completed_nbest_;
|
||||
};
|
||||
|
||||
// Just get the phrase.
|
||||
void ToPhrase(const search::Applied final, Phrase &out);
|
||||
// Get the phrase and the features.
|
||||
void PhraseAndFeatures(const TranslationSystem &system, const search::Applied final, Phrase &phrase, ScoreComponentCollection &features);
|
||||
|
||||
} // namespace Incremental
|
||||
} // namespace Moses
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user