diff --git a/mert/BleuScorer.cpp b/mert/BleuScorer.cpp index 381abeabf..04f4d75c9 100644 --- a/mert/BleuScorer.cpp +++ b/mert/BleuScorer.cpp @@ -289,23 +289,6 @@ float sentenceLevelBackgroundBleu(const std::vector& sent, const std::vec return exp(logbleu) * stats[kBleuNgramOrder*2]; } -float unsmoothedBleu(const std::vector& stats) -{ - UTIL_THROW_IF(stats.size() != kBleuNgramOrder * 2 + 1, util::Exception, "Error"); - - float logbleu = 0.0; - for (int j = 0; j < kBleuNgramOrder; j++) { - logbleu += log(stats[2 * j]) - log(stats[2 * j + 1]); - } - logbleu /= kBleuNgramOrder; - const float brevity = 1.0 - stats[(kBleuNgramOrder * 2)] / stats[1]; - - if (brevity < 0.0) { - logbleu += brevity; - } - return exp(logbleu); -} - vector BleuScorer::ScoreNbestList(const string& scoreFile, const string& featureFile) { vector scoreFiles; diff --git a/mert/BleuScorer.h b/mert/BleuScorer.h index 5fdce4a94..0594c8dca 100644 --- a/mert/BleuScorer.h +++ b/mert/BleuScorer.h @@ -55,6 +55,10 @@ public: return m_references.get(); } + virtual float getReferenceLength(const std::vector& totals) const { + return totals[kBleuNgramOrder*2]; + } + /** * Count the ngrams of each type, up to the given length in the input line. */ @@ -93,11 +97,6 @@ float smoothedSentenceBleu */ float sentenceLevelBackgroundBleu(const std::vector& sent, const std::vector& bg); -/** - * Computes plain old BLEU from a vector of stats - */ -float unsmoothedBleu(const std::vector& stats); - } #endif // MERT_BLEU_SCORER_H_ diff --git a/mert/HopeFearDecoder.cpp b/mert/HopeFearDecoder.cpp index 23ba1e099..d54d29936 100644 --- a/mert/HopeFearDecoder.cpp +++ b/mert/HopeFearDecoder.cpp @@ -28,7 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "util/exception.hh" #include "util/file_piece.hh" -#include "BleuScorer.h" +#include "Scorer.h" #include "HopeFearDecoder.h" using namespace std; @@ -39,7 +39,7 @@ namespace MosesTuning { static const ValType BLEU_RATIO = 5; ValType HopeFearDecoder::Evaluate(const AvgWeightVector& wv) { - vector stats(kBleuNgramOrder*2+1,0); + vector stats(scorer_->NumberOfScores(),0); for(reset(); !finished(); next()) { vector sent; MaxModel(wv,&sent); @@ -47,7 +47,7 @@ ValType HopeFearDecoder::Evaluate(const AvgWeightVector& wv) { stats[i]+=sent[i]; } } - return unsmoothedBleu(stats); + return scorer_->calculateScore(stats); } NbestHopeFearDecoder::NbestHopeFearDecoder( @@ -55,8 +55,10 @@ NbestHopeFearDecoder::NbestHopeFearDecoder( const vector& scoreFiles, bool streaming, bool no_shuffle, - bool safe_hope + bool safe_hope, + Scorer* scorer ) : safe_hope_(safe_hope) { + scorer_ = scorer; if (streaming) { train_.reset(new StreamingHypPackEnumerator(featureFiles, scoreFiles)); } else { @@ -93,7 +95,7 @@ void NbestHopeFearDecoder::HopeFear( for(size_t i=0; i< train_->cur_size(); i++) { const MiraFeatureVector& vec=train_->featuresAt(i); ValType score = wv.score(vec); - ValType bleu = sentenceLevelBackgroundBleu(train_->scoresAt(i),backgroundBleu); + ValType bleu = scorer_->calculateSentenceLevelBackgroundScore(train_->scoresAt(i),backgroundBleu); // Hope if(i==0 || (hope_scale*score + bleu) > hope_score) { hope_score = hope_scale*score + bleu; @@ -124,9 +126,9 @@ void NbestHopeFearDecoder::HopeFear( hopeFear->fearFeatures = train_->featuresAt(fear_index); hopeFear->hopeStats = train_->scoresAt(hope_index); - hopeFear->hopeBleu = sentenceLevelBackgroundBleu(hopeFear->hopeStats, backgroundBleu); + hopeFear->hopeBleu = scorer_->calculateSentenceLevelBackgroundScore(hopeFear->hopeStats, backgroundBleu); const vector& fear_stats = train_->scoresAt(fear_index); - hopeFear->fearBleu = sentenceLevelBackgroundBleu(fear_stats, backgroundBleu); + hopeFear->fearBleu = scorer_->calculateSentenceLevelBackgroundScore(fear_stats, backgroundBleu); hopeFear->modelStats = train_->scoresAt(model_index); hopeFear->hopeFearEqual = (hope_index == fear_index); @@ -158,7 +160,8 @@ HypergraphHopeFearDecoder::HypergraphHopeFearDecoder bool no_shuffle, bool safe_hope, size_t hg_pruning, - const MiraWeightVector& wv + const MiraWeightVector& wv, + Scorer* scorer ) : num_dense_(num_dense) { @@ -169,6 +172,7 @@ HypergraphHopeFearDecoder::HypergraphHopeFearDecoder SparseVector weights; wv.ToSparse(&weights); + scorer_ = scorer; static const string kWeights = "weights"; fs::directory_iterator dend; @@ -260,9 +264,9 @@ void HypergraphHopeFearDecoder::HopeFear( //Only C++11 //hopeFear->modelStats.assign(std::begin(modelHypo.bleuStats), std::end(modelHypo.bleuStats)); - vector fearStats(kBleuNgramOrder*2+1); - hopeFear->hopeStats.reserve(kBleuNgramOrder*2+1); - hopeFear->modelStats.reserve(kBleuNgramOrder*2+1); + vector fearStats(scorer_->NumberOfScores()); + hopeFear->hopeStats.reserve(scorer_->NumberOfScores()); + hopeFear->modelStats.reserve(scorer_->NumberOfScores()); for (size_t i = 0; i < fearStats.size(); ++i) { hopeFear->modelStats.push_back(modelHypo.bleuStats[i]); hopeFear->hopeStats.push_back(hopeHypo.bleuStats[i]); @@ -320,7 +324,7 @@ void HypergraphHopeFearDecoder::MaxModel(const AvgWeightVector& wv, vector bg(kBleuNgramOrder*2+1); + vector bg(scorer_->NumberOfScores()); Viterbi(*(graphs_[sentenceId]), weights, 0, references_, sentenceId, bg, &bestHypo); stats->resize(bestHypo.bleuStats.size()); /* diff --git a/mert/HopeFearDecoder.h b/mert/HopeFearDecoder.h index 694a3217e..d1881eeb2 100644 --- a/mert/HopeFearDecoder.h +++ b/mert/HopeFearDecoder.h @@ -37,6 +37,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA namespace MosesTuning { +class Scorer; + /** To be filled in by the decoder */ struct HopeFearData { MiraFeatureVector modelFeatures; @@ -60,6 +62,8 @@ public: virtual void next() = 0; virtual bool finished() = 0; + virtual ~HopeFearDecoder() {}; + /** * Calculate hope, fear and model hypotheses **/ @@ -76,6 +80,8 @@ public: /** Calculate bleu on training set */ ValType Evaluate(const AvgWeightVector& wv); +protected: + Scorer* scorer_; }; @@ -86,7 +92,8 @@ public: const std::vector& scoreFiles, bool streaming, bool no_shuffle, - bool safe_hope + bool safe_hope, + Scorer* scorer ); virtual void reset(); @@ -120,7 +127,8 @@ public: bool no_shuffle, bool safe_hope, size_t hg_pruning, - const MiraWeightVector& wv + const MiraWeightVector& wv, + Scorer* scorer_ ); virtual void reset(); diff --git a/mert/HwcmScorer.h b/mert/HwcmScorer.h index 1392143d5..16d563424 100644 --- a/mert/HwcmScorer.h +++ b/mert/HwcmScorer.h @@ -35,6 +35,10 @@ public: virtual float calculateScore(const std::vector& comps) const; + virtual float getReferenceLength(const std::vector& totals) const { + return totals[2]; + } + //TODO: actually, we use trees which we store in place of alignment. Maybe use something analogous to Phrase Properties to cleanly store trees? bool useAlignment() const { return true; diff --git a/mert/InterpolatedScorer.cpp b/mert/InterpolatedScorer.cpp index 87cec9211..ea4240472 100644 --- a/mert/InterpolatedScorer.cpp +++ b/mert/InterpolatedScorer.cpp @@ -153,6 +153,41 @@ void InterpolatedScorer::score(const candidates_t& candidates, const diffs_t& di } +/** Interpolated scorer gets a vector of sufficient statistics, calls all scorers with corresponding statistics, + and combines them with weights **/ +float InterpolatedScorer::calculateScore(const std::vector& totals) const +{ + size_t scorerNum = 0; + size_t last = 0; + float score = 0; + for (ScopedVector::const_iterator itsc = m_scorers.begin(); + itsc != m_scorers.end(); ++itsc) { + int numScoresScorer = (*itsc)->NumberOfScores(); + std::vector totals_scorer(totals.begin()+last, totals.begin()+last+numScoresScorer); + score += (*itsc)->calculateScore(totals_scorer) * m_scorer_weights[scorerNum]; + last += numScoresScorer; + scorerNum++; + } + return score; +} + + +float InterpolatedScorer::getReferenceLength(const std::vector& totals) const +{ + size_t scorerNum = 0; + size_t last = 0; + float refLen = 0; + for (ScopedVector::const_iterator itsc = m_scorers.begin(); + itsc != m_scorers.end(); ++itsc) { + int numScoresScorer = (*itsc)->NumberOfScores(); + std::vector totals_scorer(totals.begin()+last, totals.begin()+last+numScoresScorer); + refLen += (*itsc)->getReferenceLength(totals_scorer) * m_scorer_weights[scorerNum]; + last += numScoresScorer; + scorerNum++; + } + return refLen; + } + void InterpolatedScorer::setReferenceFiles(const vector& referenceFiles) { for (ScopedVector::iterator itsc = m_scorers.begin(); diff --git a/mert/InterpolatedScorer.h b/mert/InterpolatedScorer.h index d1078e9e1..159abf6d4 100644 --- a/mert/InterpolatedScorer.h +++ b/mert/InterpolatedScorer.h @@ -39,6 +39,10 @@ public: virtual void setScoreData(ScoreData* data); + virtual float calculateScore(const std::vector& totals) const; + + virtual float getReferenceLength(const std::vector& totals) const; + /** * Set the factors, which should be used for this metric */ diff --git a/mert/Scorer.h b/mert/Scorer.h index 0424398eb..8c468aff2 100644 --- a/mert/Scorer.h +++ b/mert/Scorer.h @@ -42,6 +42,19 @@ public: */ virtual std::size_t NumberOfScores() const = 0; + /** + * Calculate score based on a vector of sufficient statistics. + */ + virtual float calculateScore(const std::vector& totals) const = 0; + + float calculateSentenceLevelBackgroundScore(const std::vector& totals, const std::vector& bg) { + std::vector stats(totals.size()); + for(size_t i=0; i& totals) const = 0; + /** * Set the score data, prior to scoring. */ diff --git a/mert/StatisticsBasedScorer.h b/mert/StatisticsBasedScorer.h index 36108d639..f1c77e0ba 100644 --- a/mert/StatisticsBasedScorer.h +++ b/mert/StatisticsBasedScorer.h @@ -11,6 +11,8 @@ #include "Scorer.h" +#include "util/exception.hh" + namespace MosesTuning { @@ -21,6 +23,8 @@ namespace MosesTuning */ class StatisticsBasedScorer : public Scorer { +friend class HopeFearDecoder; + public: StatisticsBasedScorer(const std::string& name, const std::string& config); virtual ~StatisticsBasedScorer() {} @@ -40,6 +44,11 @@ protected: */ virtual statscore_t calculateScore(const std::vector& totals) const = 0; + virtual float getReferenceLength(const std::vector& totals) const { + UTIL_THROW(util::Exception, "getReferenceLength not implemented for this scorer type."); + return 0; + } + // regularisation RegularisationType m_regularization_type; std::size_t m_regularization_window; diff --git a/mert/kbmira.cpp b/mert/kbmira.cpp index 90f201a00..0be72e9f9 100644 --- a/mert/kbmira.cpp +++ b/mert/kbmira.cpp @@ -46,6 +46,9 @@ de recherches du Canada #include "MiraFeatureVector.h" #include "MiraWeightVector.h" +#include "Scorer.h" +#include "ScorerFactory.h" + using namespace std; using namespace MosesTuning; @@ -57,6 +60,8 @@ int main(int argc, char** argv) string denseInitFile; string sparseInitFile; string type = "nbest"; + string sctype = "BLEU"; + string scconfig = ""; vector scoreFiles; vector featureFiles; vector referenceFiles; //for hg mira @@ -78,6 +83,8 @@ int main(int argc, char** argv) desc.add_options() ("help,h", po::value(&help)->zero_tokens()->default_value(false), "Print this help message and exit") ("type,t", po::value(&type), "Either nbest or hypergraph") + ("sctype", po::value(&sctype), "the scorer type (default BLEU)") + ("scconfig,c", po::value(&scconfig), "configuration string passed to scorer") ("scfile,S", po::value >(&scoreFiles), "Scorer data files") ("ffile,F", po::value > (&featureFiles), "Feature data files") ("hgdir,H", po::value (&hgDir), "Directory containing hypergraphs") @@ -209,19 +216,20 @@ int main(int argc, char** argv) MiraWeightVector wv(initParams); - // Initialize background corpus - vector bg; - for(int j=0; j scorer(ScorerFactory::getScorer(sctype, scconfig)); + + // Initialize background corpus + vector bg(scorer->NumberOfScores(), 1); boost::scoped_ptr decoder; if (type == "nbest") { - decoder.reset(new NbestHopeFearDecoder(featureFiles, scoreFiles, streaming, no_shuffle, safe_hope)); + decoder.reset(new NbestHopeFearDecoder(featureFiles, scoreFiles, streaming, no_shuffle, safe_hope, scorer.get())); } else if (type == "hypergraph") { - decoder.reset(new HypergraphHopeFearDecoder(hgDir, referenceFiles, initDenseSize, streaming, no_shuffle, safe_hope, hgPruning, wv)); + decoder.reset(new HypergraphHopeFearDecoder(hgDir, referenceFiles, initDenseSize, streaming, no_shuffle, safe_hope, hgPruning, wv, scorer.get())); } else { UTIL_THROW(util::Exception, "Unknown batch mira type: '" << type << "'"); } diff --git a/scripts/training/mert-moses.pl b/scripts/training/mert-moses.pl index 21ae0b5e4..09f2b4912 100755 --- a/scripts/training/mert-moses.pl +++ b/scripts/training/mert-moses.pl @@ -386,7 +386,14 @@ if ($__PROMIX_TRAINING) { die "To use promix training, need to specify a filter and binarisation command" unless $filtercmd =~ /Binarizer/; } -$mertargs = "" if !defined $mertargs; +if (!defined $mertargs) { + if (defined $batch_mira_args) { + $mertargs = $batch_mira_args; + } + else { + $mertargs = ""; + } +} my $scconfig = undef; if ($mertargs =~ /\-\-scconfig\s+(.+?)(\s|$)/) {