moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-07-14 23:00:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

basic support for more metrics with kbmira

Browse Source 
metrics need getReferenceLength (for background smoothing) to work with kbmira
This commit is contained in:
Rico Sennrich 2014-09-17 14:14:11 +01:00
parent 6810b225cc
commit 3d00e5dc8c
11 changed files with 124 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
mert/BleuScorer.cpp
View File

@ -289,23 +289,6 @@ float sentenceLevelBackgroundBleu(const std::vector<float>& sent, const std::vec
return exp(logbleu) * stats[kBleuNgramOrder*2];
}
float unsmoothedBleu(const std::vector<float>& stats)
{
UTIL_THROW_IF(stats.size() != kBleuNgramOrder * 2 + 1, util::Exception, "Error");
float logbleu = 0.0;
for (int j = 0; j < kBleuNgramOrder; j++) {
logbleu += log(stats[2 * j]) - log(stats[2 * j + 1]);
}
logbleu /= kBleuNgramOrder;
const float brevity = 1.0 - stats[(kBleuNgramOrder * 2)] / stats[1];
if (brevity < 0.0) {
logbleu += brevity;
}
return exp(logbleu);
}
vector<float> BleuScorer::ScoreNbestList(const string& scoreFile, const string& featureFile)
{
vector<string> scoreFiles;

9
mert/BleuScorer.h
View File

@ -55,6 +55,10 @@ public:
return m_references.get();
}
virtual float getReferenceLength(const std::vector<ScoreStatsType>& totals) const {
return totals[kBleuNgramOrder*2];
}
/**
* Count the ngrams of each type, up to the given length in the input line.
*/
@ -93,11 +97,6 @@ float smoothedSentenceBleu
*/
float sentenceLevelBackgroundBleu(const std::vector<float>& sent, const std::vector<float>& bg);
/**
* Computes plain old BLEU from a vector of stats
*/
float unsmoothedBleu(const std::vector<float>& stats);
}
#endif // MERT_BLEU_SCORER_H_

28
mert/HopeFearDecoder.cpp
View File

@ -28,7 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "util/exception.hh"
#include "util/file_piece.hh"
#include "BleuScorer.h"
#include "Scorer.h"
#include "HopeFearDecoder.h"
using namespace std;
@ -39,7 +39,7 @@ namespace MosesTuning {
static const ValType BLEU_RATIO = 5;
ValType HopeFearDecoder::Evaluate(const AvgWeightVector& wv) {
vector<ValType> stats(kBleuNgramOrder*2+1,0);
vector<ValType> stats(scorer_->NumberOfScores(),0);
for(reset(); !finished(); next()) {
vector<ValType> sent;
MaxModel(wv,&sent);
@ -47,7 +47,7 @@ ValType HopeFearDecoder::Evaluate(const AvgWeightVector& wv) {
stats[i]+=sent[i];
}
}
return unsmoothedBleu(stats);
return scorer_->calculateScore(stats);
}
NbestHopeFearDecoder::NbestHopeFearDecoder(
@ -55,8 +55,10 @@ NbestHopeFearDecoder::NbestHopeFearDecoder(
const vector<string>& scoreFiles,
bool streaming,
bool no_shuffle,
bool safe_hope
bool safe_hope,
Scorer* scorer
) : safe_hope_(safe_hope) {
scorer_ = scorer;
if (streaming) {
train_.reset(new StreamingHypPackEnumerator(featureFiles, scoreFiles));
} else {
@ -93,7 +95,7 @@ void NbestHopeFearDecoder::HopeFear(
for(size_t i=0; i< train_->cur_size(); i++) {
const MiraFeatureVector& vec=train_->featuresAt(i);
ValType score = wv.score(vec);
ValType bleu = sentenceLevelBackgroundBleu(train_->scoresAt(i),backgroundBleu);
ValType bleu = scorer_->calculateSentenceLevelBackgroundScore(train_->scoresAt(i),backgroundBleu);
// Hope
if(i==0 || (hope_scale*score + bleu) > hope_score) {
hope_score = hope_scale*score + bleu;
@ -124,9 +126,9 @@ void NbestHopeFearDecoder::HopeFear(
hopeFear->fearFeatures = train_->featuresAt(fear_index);
hopeFear->hopeStats = train_->scoresAt(hope_index);
hopeFear->hopeBleu = sentenceLevelBackgroundBleu(hopeFear->hopeStats, backgroundBleu);
hopeFear->hopeBleu = scorer_->calculateSentenceLevelBackgroundScore(hopeFear->hopeStats, backgroundBleu);
const vector<float>& fear_stats = train_->scoresAt(fear_index);
hopeFear->fearBleu = sentenceLevelBackgroundBleu(fear_stats, backgroundBleu);
hopeFear->fearBleu = scorer_->calculateSentenceLevelBackgroundScore(fear_stats, backgroundBleu);
hopeFear->modelStats = train_->scoresAt(model_index);
hopeFear->hopeFearEqual = (hope_index == fear_index);
@ -158,7 +160,8 @@ HypergraphHopeFearDecoder::HypergraphHopeFearDecoder
bool no_shuffle,
bool safe_hope,
size_t hg_pruning,
const MiraWeightVector& wv
const MiraWeightVector& wv,
Scorer* scorer
) :
num_dense_(num_dense) {
@ -169,6 +172,7 @@ HypergraphHopeFearDecoder::HypergraphHopeFearDecoder
SparseVector weights;
wv.ToSparse(&weights);
scorer_ = scorer;
static const string kWeights = "weights";
fs::directory_iterator dend;
@ -260,9 +264,9 @@ void HypergraphHopeFearDecoder::HopeFear(
//Only C++11
//hopeFear->modelStats.assign(std::begin(modelHypo.bleuStats), std::end(modelHypo.bleuStats));
vector<ValType> fearStats(kBleuNgramOrder*2+1);
hopeFear->hopeStats.reserve(kBleuNgramOrder*2+1);
hopeFear->modelStats.reserve(kBleuNgramOrder*2+1);
vector<ValType> fearStats(scorer_->NumberOfScores());
hopeFear->hopeStats.reserve(scorer_->NumberOfScores());
hopeFear->modelStats.reserve(scorer_->NumberOfScores());
for (size_t i = 0; i < fearStats.size(); ++i) {
hopeFear->modelStats.push_back(modelHypo.bleuStats[i]);
hopeFear->hopeStats.push_back(hopeHypo.bleuStats[i]);
@ -320,7 +324,7 @@ void HypergraphHopeFearDecoder::MaxModel(const AvgWeightVector& wv, vector<ValTy
size_t sentenceId = *sentenceIdIter_;
SparseVector weights;
wv.ToSparse(&weights);
vector<ValType> bg(kBleuNgramOrder*2+1);
vector<ValType> bg(scorer_->NumberOfScores());
Viterbi(*(graphs_[sentenceId]), weights, 0, references_, sentenceId, bg, &bestHypo);
stats->resize(bestHypo.bleuStats.size());
/*

12
mert/HopeFearDecoder.h
View File

@ -37,6 +37,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
namespace MosesTuning {
class Scorer;
/** To be filled in by the decoder */
struct HopeFearData {
MiraFeatureVector modelFeatures;
@ -60,6 +62,8 @@ public:
virtual void next() = 0;
virtual bool finished() = 0;
virtual ~HopeFearDecoder() {};
/**
* Calculate hope, fear and model hypotheses
**/
@ -76,6 +80,8 @@ public:
/** Calculate bleu on training set */
ValType Evaluate(const AvgWeightVector& wv);
protected:
Scorer* scorer_;
};
@ -86,7 +92,8 @@ public:
const std::vector<std::string>& scoreFiles,
bool streaming,
bool no_shuffle,
bool safe_hope
bool safe_hope,
Scorer* scorer
);
virtual void reset();
@ -120,7 +127,8 @@ public:
bool no_shuffle,
bool safe_hope,
size_t hg_pruning,
const MiraWeightVector& wv
const MiraWeightVector& wv,
Scorer* scorer_
);
virtual void reset();

4
mert/HwcmScorer.h
View File

@ -35,6 +35,10 @@ public:
virtual float calculateScore(const std::vector<ScoreStatsType>& comps) const;
virtual float getReferenceLength(const std::vector<ScoreStatsType>& totals) const {
return totals[2];
}
//TODO: actually, we use trees which we store in place of alignment. Maybe use something analogous to Phrase Properties to cleanly store trees?
bool useAlignment() const {
return true;

35
mert/InterpolatedScorer.cpp
View File

@ -153,6 +153,41 @@ void InterpolatedScorer::score(const candidates_t& candidates, const diffs_t& di
}
/** Interpolated scorer gets a vector of sufficient statistics, calls all scorers with corresponding statistics,
and combines them with weights **/
float InterpolatedScorer::calculateScore(const std::vector<ScoreStatsType>& totals) const
{
size_t scorerNum = 0;
size_t last = 0;
float score = 0;
for (ScopedVector<Scorer>::const_iterator itsc = m_scorers.begin();
itsc != m_scorers.end(); ++itsc) {
int numScoresScorer = (*itsc)->NumberOfScores();
std::vector<ScoreStatsType> totals_scorer(totals.begin()+last, totals.begin()+last+numScoresScorer);
score += (*itsc)->calculateScore(totals_scorer) * m_scorer_weights[scorerNum];
last += numScoresScorer;
scorerNum++;
}
return score;
}
float InterpolatedScorer::getReferenceLength(const std::vector<ScoreStatsType>& totals) const
{
size_t scorerNum = 0;
size_t last = 0;
float refLen = 0;
for (ScopedVector<Scorer>::const_iterator itsc = m_scorers.begin();
itsc != m_scorers.end(); ++itsc) {
int numScoresScorer = (*itsc)->NumberOfScores();
std::vector<ScoreStatsType> totals_scorer(totals.begin()+last, totals.begin()+last+numScoresScorer);
refLen += (*itsc)->getReferenceLength(totals_scorer) * m_scorer_weights[scorerNum];
last += numScoresScorer;
scorerNum++;
}
return refLen;
}
void InterpolatedScorer::setReferenceFiles(const vector<string>& referenceFiles)
{
for (ScopedVector<Scorer>::iterator itsc = m_scorers.begin();

4
mert/InterpolatedScorer.h
View File

@ -39,6 +39,10 @@ public:
virtual void setScoreData(ScoreData* data);
virtual float calculateScore(const std::vector<ScoreStatsType>& totals) const;
virtual float getReferenceLength(const std::vector<ScoreStatsType>& totals) const;
/**
* Set the factors, which should be used for this metric
*/

18
mert/Scorer.h
View File

@ -42,6 +42,19 @@ public:
*/
virtual std::size_t NumberOfScores() const = 0;
/**
* Calculate score based on a vector of sufficient statistics.
*/
virtual float calculateScore(const std::vector<ScoreStatsType>& totals) const = 0;
float calculateSentenceLevelBackgroundScore(const std::vector<ScoreStatsType>& totals, const std::vector<ScoreStatsType>& bg) {
std::vector<ScoreStatsType> stats(totals.size());
for(size_t i=0; i<stats.size(); i++)
stats[i] = totals[i]+bg[i];
// Get score and scale by reference length (as per Chiang et al 08)
return calculateScore(stats) * getReferenceLength(stats);
}
/**
* Set the reference files. This must be called before prepareStats().
*/
@ -97,6 +110,11 @@ public:
return 0;
}
/**
* Based on vector of sufficient statistics, return length of reference.
*/
virtual float getReferenceLength(const std::vector<ScoreStatsType>& totals) const = 0;
/**
* Set the score data, prior to scoring.
*/

9
mert/StatisticsBasedScorer.h
View File

@ -11,6 +11,8 @@
#include "Scorer.h"
#include "util/exception.hh"
namespace MosesTuning
{
@ -21,6 +23,8 @@ namespace MosesTuning
*/
class StatisticsBasedScorer : public Scorer
{
friend class HopeFearDecoder;
public:
StatisticsBasedScorer(const std::string& name, const std::string& config);
virtual ~StatisticsBasedScorer() {}
@ -40,6 +44,11 @@ protected:
*/
virtual statscore_t calculateScore(const std::vector<ScoreStatsType>& totals) const = 0;
virtual float getReferenceLength(const std::vector<ScoreStatsType>& totals) const {
UTIL_THROW(util::Exception, "getReferenceLength not implemented for this scorer type.");
return 0;
}
// regularisation
RegularisationType m_regularization_type;
std::size_t m_regularization_window;

24
mert/kbmira.cpp
View File

@ -46,6 +46,9 @@ de recherches du Canada
#include "MiraFeatureVector.h"
#include "MiraWeightVector.h"
#include "Scorer.h"
#include "ScorerFactory.h"
using namespace std;
using namespace MosesTuning;
@ -57,6 +60,8 @@ int main(int argc, char** argv)
string denseInitFile;
string sparseInitFile;
string type = "nbest";
string sctype = "BLEU";
string scconfig = "";
vector<string> scoreFiles;
vector<string> featureFiles;
vector<string> referenceFiles; //for hg mira
@ -78,6 +83,8 @@ int main(int argc, char** argv)
desc.add_options()
("help,h", po::value(&help)->zero_tokens()->default_value(false), "Print this help message and exit")
("type,t", po::value<string>(&type), "Either nbest or hypergraph")
("sctype", po::value<string>(&sctype), "the scorer type (default BLEU)")
("scconfig,c", po::value<string>(&scconfig), "configuration string passed to scorer")
("scfile,S", po::value<vector<string> >(&scoreFiles), "Scorer data files")
("ffile,F", po::value<vector<string> > (&featureFiles), "Feature data files")
("hgdir,H", po::value<string> (&hgDir), "Directory containing hypergraphs")
@ -209,19 +216,20 @@ int main(int argc, char** argv)
MiraWeightVector wv(initParams);
// Initialize background corpus
vector<ValType> bg;
for(int j=0; j<kBleuNgramOrder; j++) {
bg.push_back(kBleuNgramOrder-j);
bg.push_back(kBleuNgramOrder-j);
// Initialize scorer
if(sctype != "BLEU" && type == "hypergraph") {
UTIL_THROW(util::Exception, "hypergraph mira only supports BLEU");
}
bg.push_back(kBleuNgramOrder);
boost::scoped_ptr<Scorer> scorer(ScorerFactory::getScorer(sctype, scconfig));
// Initialize background corpus
vector<ValType> bg(scorer->NumberOfScores(), 1);
boost::scoped_ptr<HopeFearDecoder> decoder;
if (type == "nbest") {
decoder.reset(new NbestHopeFearDecoder(featureFiles, scoreFiles, streaming, no_shuffle, safe_hope));
decoder.reset(new NbestHopeFearDecoder(featureFiles, scoreFiles, streaming, no_shuffle, safe_hope, scorer.get()));
} else if (type == "hypergraph") {
decoder.reset(new HypergraphHopeFearDecoder(hgDir, referenceFiles, initDenseSize, streaming, no_shuffle, safe_hope, hgPruning, wv));
decoder.reset(new HypergraphHopeFearDecoder(hgDir, referenceFiles, initDenseSize, streaming, no_shuffle, safe_hope, hgPruning, wv, scorer.get()));
} else {
UTIL_THROW(util::Exception, "Unknown batch mira type: '" << type << "'");
}

9
scripts/training/mert-moses.pl
View File

@ -386,7 +386,14 @@ if ($__PROMIX_TRAINING) {
die "To use promix training, need to specify a filter and binarisation command" unless $filtercmd =~ /Binarizer/;
}
$mertargs = "" if !defined $mertargs;
if (!defined $mertargs) {
if (defined $batch_mira_args) {
$mertargs = $batch_mira_args;
}
else {
$mertargs = "";
}
}
my $scconfig = undef;
if ($mertargs =~ /\-\-scconfig\s+(.+?)(\s|$)/) {