mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-08-17 07:20:48 +03:00
basic support for more metrics with kbmira
metrics need getReferenceLength (for background smoothing) to work with kbmira
This commit is contained in:
parent
6810b225cc
commit
3d00e5dc8c
@ -289,23 +289,6 @@ float sentenceLevelBackgroundBleu(const std::vector<float>& sent, const std::vec
|
|||||||
return exp(logbleu) * stats[kBleuNgramOrder*2];
|
return exp(logbleu) * stats[kBleuNgramOrder*2];
|
||||||
}
|
}
|
||||||
|
|
||||||
float unsmoothedBleu(const std::vector<float>& stats)
|
|
||||||
{
|
|
||||||
UTIL_THROW_IF(stats.size() != kBleuNgramOrder * 2 + 1, util::Exception, "Error");
|
|
||||||
|
|
||||||
float logbleu = 0.0;
|
|
||||||
for (int j = 0; j < kBleuNgramOrder; j++) {
|
|
||||||
logbleu += log(stats[2 * j]) - log(stats[2 * j + 1]);
|
|
||||||
}
|
|
||||||
logbleu /= kBleuNgramOrder;
|
|
||||||
const float brevity = 1.0 - stats[(kBleuNgramOrder * 2)] / stats[1];
|
|
||||||
|
|
||||||
if (brevity < 0.0) {
|
|
||||||
logbleu += brevity;
|
|
||||||
}
|
|
||||||
return exp(logbleu);
|
|
||||||
}
|
|
||||||
|
|
||||||
vector<float> BleuScorer::ScoreNbestList(const string& scoreFile, const string& featureFile)
|
vector<float> BleuScorer::ScoreNbestList(const string& scoreFile, const string& featureFile)
|
||||||
{
|
{
|
||||||
vector<string> scoreFiles;
|
vector<string> scoreFiles;
|
||||||
|
@ -55,6 +55,10 @@ public:
|
|||||||
return m_references.get();
|
return m_references.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
virtual float getReferenceLength(const std::vector<ScoreStatsType>& totals) const {
|
||||||
|
return totals[kBleuNgramOrder*2];
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Count the ngrams of each type, up to the given length in the input line.
|
* Count the ngrams of each type, up to the given length in the input line.
|
||||||
*/
|
*/
|
||||||
@ -93,11 +97,6 @@ float smoothedSentenceBleu
|
|||||||
*/
|
*/
|
||||||
float sentenceLevelBackgroundBleu(const std::vector<float>& sent, const std::vector<float>& bg);
|
float sentenceLevelBackgroundBleu(const std::vector<float>& sent, const std::vector<float>& bg);
|
||||||
|
|
||||||
/**
|
|
||||||
* Computes plain old BLEU from a vector of stats
|
|
||||||
*/
|
|
||||||
float unsmoothedBleu(const std::vector<float>& stats);
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#endif // MERT_BLEU_SCORER_H_
|
#endif // MERT_BLEU_SCORER_H_
|
||||||
|
@ -28,7 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
#include "util/exception.hh"
|
#include "util/exception.hh"
|
||||||
#include "util/file_piece.hh"
|
#include "util/file_piece.hh"
|
||||||
|
|
||||||
#include "BleuScorer.h"
|
#include "Scorer.h"
|
||||||
#include "HopeFearDecoder.h"
|
#include "HopeFearDecoder.h"
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
@ -39,7 +39,7 @@ namespace MosesTuning {
|
|||||||
static const ValType BLEU_RATIO = 5;
|
static const ValType BLEU_RATIO = 5;
|
||||||
|
|
||||||
ValType HopeFearDecoder::Evaluate(const AvgWeightVector& wv) {
|
ValType HopeFearDecoder::Evaluate(const AvgWeightVector& wv) {
|
||||||
vector<ValType> stats(kBleuNgramOrder*2+1,0);
|
vector<ValType> stats(scorer_->NumberOfScores(),0);
|
||||||
for(reset(); !finished(); next()) {
|
for(reset(); !finished(); next()) {
|
||||||
vector<ValType> sent;
|
vector<ValType> sent;
|
||||||
MaxModel(wv,&sent);
|
MaxModel(wv,&sent);
|
||||||
@ -47,7 +47,7 @@ ValType HopeFearDecoder::Evaluate(const AvgWeightVector& wv) {
|
|||||||
stats[i]+=sent[i];
|
stats[i]+=sent[i];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return unsmoothedBleu(stats);
|
return scorer_->calculateScore(stats);
|
||||||
}
|
}
|
||||||
|
|
||||||
NbestHopeFearDecoder::NbestHopeFearDecoder(
|
NbestHopeFearDecoder::NbestHopeFearDecoder(
|
||||||
@ -55,8 +55,10 @@ NbestHopeFearDecoder::NbestHopeFearDecoder(
|
|||||||
const vector<string>& scoreFiles,
|
const vector<string>& scoreFiles,
|
||||||
bool streaming,
|
bool streaming,
|
||||||
bool no_shuffle,
|
bool no_shuffle,
|
||||||
bool safe_hope
|
bool safe_hope,
|
||||||
|
Scorer* scorer
|
||||||
) : safe_hope_(safe_hope) {
|
) : safe_hope_(safe_hope) {
|
||||||
|
scorer_ = scorer;
|
||||||
if (streaming) {
|
if (streaming) {
|
||||||
train_.reset(new StreamingHypPackEnumerator(featureFiles, scoreFiles));
|
train_.reset(new StreamingHypPackEnumerator(featureFiles, scoreFiles));
|
||||||
} else {
|
} else {
|
||||||
@ -93,7 +95,7 @@ void NbestHopeFearDecoder::HopeFear(
|
|||||||
for(size_t i=0; i< train_->cur_size(); i++) {
|
for(size_t i=0; i< train_->cur_size(); i++) {
|
||||||
const MiraFeatureVector& vec=train_->featuresAt(i);
|
const MiraFeatureVector& vec=train_->featuresAt(i);
|
||||||
ValType score = wv.score(vec);
|
ValType score = wv.score(vec);
|
||||||
ValType bleu = sentenceLevelBackgroundBleu(train_->scoresAt(i),backgroundBleu);
|
ValType bleu = scorer_->calculateSentenceLevelBackgroundScore(train_->scoresAt(i),backgroundBleu);
|
||||||
// Hope
|
// Hope
|
||||||
if(i==0 || (hope_scale*score + bleu) > hope_score) {
|
if(i==0 || (hope_scale*score + bleu) > hope_score) {
|
||||||
hope_score = hope_scale*score + bleu;
|
hope_score = hope_scale*score + bleu;
|
||||||
@ -124,9 +126,9 @@ void NbestHopeFearDecoder::HopeFear(
|
|||||||
hopeFear->fearFeatures = train_->featuresAt(fear_index);
|
hopeFear->fearFeatures = train_->featuresAt(fear_index);
|
||||||
|
|
||||||
hopeFear->hopeStats = train_->scoresAt(hope_index);
|
hopeFear->hopeStats = train_->scoresAt(hope_index);
|
||||||
hopeFear->hopeBleu = sentenceLevelBackgroundBleu(hopeFear->hopeStats, backgroundBleu);
|
hopeFear->hopeBleu = scorer_->calculateSentenceLevelBackgroundScore(hopeFear->hopeStats, backgroundBleu);
|
||||||
const vector<float>& fear_stats = train_->scoresAt(fear_index);
|
const vector<float>& fear_stats = train_->scoresAt(fear_index);
|
||||||
hopeFear->fearBleu = sentenceLevelBackgroundBleu(fear_stats, backgroundBleu);
|
hopeFear->fearBleu = scorer_->calculateSentenceLevelBackgroundScore(fear_stats, backgroundBleu);
|
||||||
|
|
||||||
hopeFear->modelStats = train_->scoresAt(model_index);
|
hopeFear->modelStats = train_->scoresAt(model_index);
|
||||||
hopeFear->hopeFearEqual = (hope_index == fear_index);
|
hopeFear->hopeFearEqual = (hope_index == fear_index);
|
||||||
@ -158,7 +160,8 @@ HypergraphHopeFearDecoder::HypergraphHopeFearDecoder
|
|||||||
bool no_shuffle,
|
bool no_shuffle,
|
||||||
bool safe_hope,
|
bool safe_hope,
|
||||||
size_t hg_pruning,
|
size_t hg_pruning,
|
||||||
const MiraWeightVector& wv
|
const MiraWeightVector& wv,
|
||||||
|
Scorer* scorer
|
||||||
) :
|
) :
|
||||||
num_dense_(num_dense) {
|
num_dense_(num_dense) {
|
||||||
|
|
||||||
@ -169,6 +172,7 @@ HypergraphHopeFearDecoder::HypergraphHopeFearDecoder
|
|||||||
|
|
||||||
SparseVector weights;
|
SparseVector weights;
|
||||||
wv.ToSparse(&weights);
|
wv.ToSparse(&weights);
|
||||||
|
scorer_ = scorer;
|
||||||
|
|
||||||
static const string kWeights = "weights";
|
static const string kWeights = "weights";
|
||||||
fs::directory_iterator dend;
|
fs::directory_iterator dend;
|
||||||
@ -260,9 +264,9 @@ void HypergraphHopeFearDecoder::HopeFear(
|
|||||||
|
|
||||||
//Only C++11
|
//Only C++11
|
||||||
//hopeFear->modelStats.assign(std::begin(modelHypo.bleuStats), std::end(modelHypo.bleuStats));
|
//hopeFear->modelStats.assign(std::begin(modelHypo.bleuStats), std::end(modelHypo.bleuStats));
|
||||||
vector<ValType> fearStats(kBleuNgramOrder*2+1);
|
vector<ValType> fearStats(scorer_->NumberOfScores());
|
||||||
hopeFear->hopeStats.reserve(kBleuNgramOrder*2+1);
|
hopeFear->hopeStats.reserve(scorer_->NumberOfScores());
|
||||||
hopeFear->modelStats.reserve(kBleuNgramOrder*2+1);
|
hopeFear->modelStats.reserve(scorer_->NumberOfScores());
|
||||||
for (size_t i = 0; i < fearStats.size(); ++i) {
|
for (size_t i = 0; i < fearStats.size(); ++i) {
|
||||||
hopeFear->modelStats.push_back(modelHypo.bleuStats[i]);
|
hopeFear->modelStats.push_back(modelHypo.bleuStats[i]);
|
||||||
hopeFear->hopeStats.push_back(hopeHypo.bleuStats[i]);
|
hopeFear->hopeStats.push_back(hopeHypo.bleuStats[i]);
|
||||||
@ -320,7 +324,7 @@ void HypergraphHopeFearDecoder::MaxModel(const AvgWeightVector& wv, vector<ValTy
|
|||||||
size_t sentenceId = *sentenceIdIter_;
|
size_t sentenceId = *sentenceIdIter_;
|
||||||
SparseVector weights;
|
SparseVector weights;
|
||||||
wv.ToSparse(&weights);
|
wv.ToSparse(&weights);
|
||||||
vector<ValType> bg(kBleuNgramOrder*2+1);
|
vector<ValType> bg(scorer_->NumberOfScores());
|
||||||
Viterbi(*(graphs_[sentenceId]), weights, 0, references_, sentenceId, bg, &bestHypo);
|
Viterbi(*(graphs_[sentenceId]), weights, 0, references_, sentenceId, bg, &bestHypo);
|
||||||
stats->resize(bestHypo.bleuStats.size());
|
stats->resize(bestHypo.bleuStats.size());
|
||||||
/*
|
/*
|
||||||
|
@ -37,6 +37,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
|
|
||||||
namespace MosesTuning {
|
namespace MosesTuning {
|
||||||
|
|
||||||
|
class Scorer;
|
||||||
|
|
||||||
/** To be filled in by the decoder */
|
/** To be filled in by the decoder */
|
||||||
struct HopeFearData {
|
struct HopeFearData {
|
||||||
MiraFeatureVector modelFeatures;
|
MiraFeatureVector modelFeatures;
|
||||||
@ -60,6 +62,8 @@ public:
|
|||||||
virtual void next() = 0;
|
virtual void next() = 0;
|
||||||
virtual bool finished() = 0;
|
virtual bool finished() = 0;
|
||||||
|
|
||||||
|
virtual ~HopeFearDecoder() {};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Calculate hope, fear and model hypotheses
|
* Calculate hope, fear and model hypotheses
|
||||||
**/
|
**/
|
||||||
@ -76,6 +80,8 @@ public:
|
|||||||
/** Calculate bleu on training set */
|
/** Calculate bleu on training set */
|
||||||
ValType Evaluate(const AvgWeightVector& wv);
|
ValType Evaluate(const AvgWeightVector& wv);
|
||||||
|
|
||||||
|
protected:
|
||||||
|
Scorer* scorer_;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
@ -86,7 +92,8 @@ public:
|
|||||||
const std::vector<std::string>& scoreFiles,
|
const std::vector<std::string>& scoreFiles,
|
||||||
bool streaming,
|
bool streaming,
|
||||||
bool no_shuffle,
|
bool no_shuffle,
|
||||||
bool safe_hope
|
bool safe_hope,
|
||||||
|
Scorer* scorer
|
||||||
);
|
);
|
||||||
|
|
||||||
virtual void reset();
|
virtual void reset();
|
||||||
@ -120,7 +127,8 @@ public:
|
|||||||
bool no_shuffle,
|
bool no_shuffle,
|
||||||
bool safe_hope,
|
bool safe_hope,
|
||||||
size_t hg_pruning,
|
size_t hg_pruning,
|
||||||
const MiraWeightVector& wv
|
const MiraWeightVector& wv,
|
||||||
|
Scorer* scorer_
|
||||||
);
|
);
|
||||||
|
|
||||||
virtual void reset();
|
virtual void reset();
|
||||||
|
@ -35,6 +35,10 @@ public:
|
|||||||
|
|
||||||
virtual float calculateScore(const std::vector<ScoreStatsType>& comps) const;
|
virtual float calculateScore(const std::vector<ScoreStatsType>& comps) const;
|
||||||
|
|
||||||
|
virtual float getReferenceLength(const std::vector<ScoreStatsType>& totals) const {
|
||||||
|
return totals[2];
|
||||||
|
}
|
||||||
|
|
||||||
//TODO: actually, we use trees which we store in place of alignment. Maybe use something analogous to Phrase Properties to cleanly store trees?
|
//TODO: actually, we use trees which we store in place of alignment. Maybe use something analogous to Phrase Properties to cleanly store trees?
|
||||||
bool useAlignment() const {
|
bool useAlignment() const {
|
||||||
return true;
|
return true;
|
||||||
|
@ -153,6 +153,41 @@ void InterpolatedScorer::score(const candidates_t& candidates, const diffs_t& di
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Interpolated scorer gets a vector of sufficient statistics, calls all scorers with corresponding statistics,
|
||||||
|
and combines them with weights **/
|
||||||
|
float InterpolatedScorer::calculateScore(const std::vector<ScoreStatsType>& totals) const
|
||||||
|
{
|
||||||
|
size_t scorerNum = 0;
|
||||||
|
size_t last = 0;
|
||||||
|
float score = 0;
|
||||||
|
for (ScopedVector<Scorer>::const_iterator itsc = m_scorers.begin();
|
||||||
|
itsc != m_scorers.end(); ++itsc) {
|
||||||
|
int numScoresScorer = (*itsc)->NumberOfScores();
|
||||||
|
std::vector<ScoreStatsType> totals_scorer(totals.begin()+last, totals.begin()+last+numScoresScorer);
|
||||||
|
score += (*itsc)->calculateScore(totals_scorer) * m_scorer_weights[scorerNum];
|
||||||
|
last += numScoresScorer;
|
||||||
|
scorerNum++;
|
||||||
|
}
|
||||||
|
return score;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
float InterpolatedScorer::getReferenceLength(const std::vector<ScoreStatsType>& totals) const
|
||||||
|
{
|
||||||
|
size_t scorerNum = 0;
|
||||||
|
size_t last = 0;
|
||||||
|
float refLen = 0;
|
||||||
|
for (ScopedVector<Scorer>::const_iterator itsc = m_scorers.begin();
|
||||||
|
itsc != m_scorers.end(); ++itsc) {
|
||||||
|
int numScoresScorer = (*itsc)->NumberOfScores();
|
||||||
|
std::vector<ScoreStatsType> totals_scorer(totals.begin()+last, totals.begin()+last+numScoresScorer);
|
||||||
|
refLen += (*itsc)->getReferenceLength(totals_scorer) * m_scorer_weights[scorerNum];
|
||||||
|
last += numScoresScorer;
|
||||||
|
scorerNum++;
|
||||||
|
}
|
||||||
|
return refLen;
|
||||||
|
}
|
||||||
|
|
||||||
void InterpolatedScorer::setReferenceFiles(const vector<string>& referenceFiles)
|
void InterpolatedScorer::setReferenceFiles(const vector<string>& referenceFiles)
|
||||||
{
|
{
|
||||||
for (ScopedVector<Scorer>::iterator itsc = m_scorers.begin();
|
for (ScopedVector<Scorer>::iterator itsc = m_scorers.begin();
|
||||||
|
@ -39,6 +39,10 @@ public:
|
|||||||
|
|
||||||
virtual void setScoreData(ScoreData* data);
|
virtual void setScoreData(ScoreData* data);
|
||||||
|
|
||||||
|
virtual float calculateScore(const std::vector<ScoreStatsType>& totals) const;
|
||||||
|
|
||||||
|
virtual float getReferenceLength(const std::vector<ScoreStatsType>& totals) const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set the factors, which should be used for this metric
|
* Set the factors, which should be used for this metric
|
||||||
*/
|
*/
|
||||||
|
@ -42,6 +42,19 @@ public:
|
|||||||
*/
|
*/
|
||||||
virtual std::size_t NumberOfScores() const = 0;
|
virtual std::size_t NumberOfScores() const = 0;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate score based on a vector of sufficient statistics.
|
||||||
|
*/
|
||||||
|
virtual float calculateScore(const std::vector<ScoreStatsType>& totals) const = 0;
|
||||||
|
|
||||||
|
float calculateSentenceLevelBackgroundScore(const std::vector<ScoreStatsType>& totals, const std::vector<ScoreStatsType>& bg) {
|
||||||
|
std::vector<ScoreStatsType> stats(totals.size());
|
||||||
|
for(size_t i=0; i<stats.size(); i++)
|
||||||
|
stats[i] = totals[i]+bg[i];
|
||||||
|
// Get score and scale by reference length (as per Chiang et al 08)
|
||||||
|
return calculateScore(stats) * getReferenceLength(stats);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set the reference files. This must be called before prepareStats().
|
* Set the reference files. This must be called before prepareStats().
|
||||||
*/
|
*/
|
||||||
@ -97,6 +110,11 @@ public:
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Based on vector of sufficient statistics, return length of reference.
|
||||||
|
*/
|
||||||
|
virtual float getReferenceLength(const std::vector<ScoreStatsType>& totals) const = 0;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set the score data, prior to scoring.
|
* Set the score data, prior to scoring.
|
||||||
*/
|
*/
|
||||||
|
@ -11,6 +11,8 @@
|
|||||||
|
|
||||||
#include "Scorer.h"
|
#include "Scorer.h"
|
||||||
|
|
||||||
|
#include "util/exception.hh"
|
||||||
|
|
||||||
namespace MosesTuning
|
namespace MosesTuning
|
||||||
{
|
{
|
||||||
|
|
||||||
@ -21,6 +23,8 @@ namespace MosesTuning
|
|||||||
*/
|
*/
|
||||||
class StatisticsBasedScorer : public Scorer
|
class StatisticsBasedScorer : public Scorer
|
||||||
{
|
{
|
||||||
|
friend class HopeFearDecoder;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
StatisticsBasedScorer(const std::string& name, const std::string& config);
|
StatisticsBasedScorer(const std::string& name, const std::string& config);
|
||||||
virtual ~StatisticsBasedScorer() {}
|
virtual ~StatisticsBasedScorer() {}
|
||||||
@ -40,6 +44,11 @@ protected:
|
|||||||
*/
|
*/
|
||||||
virtual statscore_t calculateScore(const std::vector<ScoreStatsType>& totals) const = 0;
|
virtual statscore_t calculateScore(const std::vector<ScoreStatsType>& totals) const = 0;
|
||||||
|
|
||||||
|
virtual float getReferenceLength(const std::vector<ScoreStatsType>& totals) const {
|
||||||
|
UTIL_THROW(util::Exception, "getReferenceLength not implemented for this scorer type.");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
// regularisation
|
// regularisation
|
||||||
RegularisationType m_regularization_type;
|
RegularisationType m_regularization_type;
|
||||||
std::size_t m_regularization_window;
|
std::size_t m_regularization_window;
|
||||||
|
@ -46,6 +46,9 @@ de recherches du Canada
|
|||||||
#include "MiraFeatureVector.h"
|
#include "MiraFeatureVector.h"
|
||||||
#include "MiraWeightVector.h"
|
#include "MiraWeightVector.h"
|
||||||
|
|
||||||
|
#include "Scorer.h"
|
||||||
|
#include "ScorerFactory.h"
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
using namespace MosesTuning;
|
using namespace MosesTuning;
|
||||||
|
|
||||||
@ -57,6 +60,8 @@ int main(int argc, char** argv)
|
|||||||
string denseInitFile;
|
string denseInitFile;
|
||||||
string sparseInitFile;
|
string sparseInitFile;
|
||||||
string type = "nbest";
|
string type = "nbest";
|
||||||
|
string sctype = "BLEU";
|
||||||
|
string scconfig = "";
|
||||||
vector<string> scoreFiles;
|
vector<string> scoreFiles;
|
||||||
vector<string> featureFiles;
|
vector<string> featureFiles;
|
||||||
vector<string> referenceFiles; //for hg mira
|
vector<string> referenceFiles; //for hg mira
|
||||||
@ -78,6 +83,8 @@ int main(int argc, char** argv)
|
|||||||
desc.add_options()
|
desc.add_options()
|
||||||
("help,h", po::value(&help)->zero_tokens()->default_value(false), "Print this help message and exit")
|
("help,h", po::value(&help)->zero_tokens()->default_value(false), "Print this help message and exit")
|
||||||
("type,t", po::value<string>(&type), "Either nbest or hypergraph")
|
("type,t", po::value<string>(&type), "Either nbest or hypergraph")
|
||||||
|
("sctype", po::value<string>(&sctype), "the scorer type (default BLEU)")
|
||||||
|
("scconfig,c", po::value<string>(&scconfig), "configuration string passed to scorer")
|
||||||
("scfile,S", po::value<vector<string> >(&scoreFiles), "Scorer data files")
|
("scfile,S", po::value<vector<string> >(&scoreFiles), "Scorer data files")
|
||||||
("ffile,F", po::value<vector<string> > (&featureFiles), "Feature data files")
|
("ffile,F", po::value<vector<string> > (&featureFiles), "Feature data files")
|
||||||
("hgdir,H", po::value<string> (&hgDir), "Directory containing hypergraphs")
|
("hgdir,H", po::value<string> (&hgDir), "Directory containing hypergraphs")
|
||||||
@ -209,19 +216,20 @@ int main(int argc, char** argv)
|
|||||||
|
|
||||||
MiraWeightVector wv(initParams);
|
MiraWeightVector wv(initParams);
|
||||||
|
|
||||||
// Initialize background corpus
|
// Initialize scorer
|
||||||
vector<ValType> bg;
|
if(sctype != "BLEU" && type == "hypergraph") {
|
||||||
for(int j=0; j<kBleuNgramOrder; j++) {
|
UTIL_THROW(util::Exception, "hypergraph mira only supports BLEU");
|
||||||
bg.push_back(kBleuNgramOrder-j);
|
|
||||||
bg.push_back(kBleuNgramOrder-j);
|
|
||||||
}
|
}
|
||||||
bg.push_back(kBleuNgramOrder);
|
boost::scoped_ptr<Scorer> scorer(ScorerFactory::getScorer(sctype, scconfig));
|
||||||
|
|
||||||
|
// Initialize background corpus
|
||||||
|
vector<ValType> bg(scorer->NumberOfScores(), 1);
|
||||||
|
|
||||||
boost::scoped_ptr<HopeFearDecoder> decoder;
|
boost::scoped_ptr<HopeFearDecoder> decoder;
|
||||||
if (type == "nbest") {
|
if (type == "nbest") {
|
||||||
decoder.reset(new NbestHopeFearDecoder(featureFiles, scoreFiles, streaming, no_shuffle, safe_hope));
|
decoder.reset(new NbestHopeFearDecoder(featureFiles, scoreFiles, streaming, no_shuffle, safe_hope, scorer.get()));
|
||||||
} else if (type == "hypergraph") {
|
} else if (type == "hypergraph") {
|
||||||
decoder.reset(new HypergraphHopeFearDecoder(hgDir, referenceFiles, initDenseSize, streaming, no_shuffle, safe_hope, hgPruning, wv));
|
decoder.reset(new HypergraphHopeFearDecoder(hgDir, referenceFiles, initDenseSize, streaming, no_shuffle, safe_hope, hgPruning, wv, scorer.get()));
|
||||||
} else {
|
} else {
|
||||||
UTIL_THROW(util::Exception, "Unknown batch mira type: '" << type << "'");
|
UTIL_THROW(util::Exception, "Unknown batch mira type: '" << type << "'");
|
||||||
}
|
}
|
||||||
|
@ -386,7 +386,14 @@ if ($__PROMIX_TRAINING) {
|
|||||||
die "To use promix training, need to specify a filter and binarisation command" unless $filtercmd =~ /Binarizer/;
|
die "To use promix training, need to specify a filter and binarisation command" unless $filtercmd =~ /Binarizer/;
|
||||||
}
|
}
|
||||||
|
|
||||||
$mertargs = "" if !defined $mertargs;
|
if (!defined $mertargs) {
|
||||||
|
if (defined $batch_mira_args) {
|
||||||
|
$mertargs = $batch_mira_args;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
$mertargs = "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
my $scconfig = undef;
|
my $scconfig = undef;
|
||||||
if ($mertargs =~ /\-\-scconfig\s+(.+?)(\s|$)/) {
|
if ($mertargs =~ /\-\-scconfig\s+(.+?)(\s|$)/) {
|
||||||
|
Loading…
Reference in New Issue
Block a user