2015-05-01 00:26:30 +03:00
|
|
|
#pragma once
|
2008-05-14 12:46:15 +04:00
|
|
|
|
|
|
|
#include <iostream>
|
2008-05-15 20:03:49 +04:00
|
|
|
#include <sstream>
|
2008-05-14 16:23:58 +04:00
|
|
|
#include <stdexcept>
|
2008-05-14 12:46:15 +04:00
|
|
|
#include <string>
|
|
|
|
#include <vector>
|
2012-06-25 17:45:45 +04:00
|
|
|
#include <limits>
|
2008-05-15 23:09:01 +04:00
|
|
|
#include "Types.h"
|
2008-05-14 20:31:22 +04:00
|
|
|
#include "ScoreData.h"
|
2008-05-14 16:23:58 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
namespace mert
|
|
|
|
{
|
2012-03-19 22:43:04 +04:00
|
|
|
|
|
|
|
class Vocabulary;
|
|
|
|
|
|
|
|
} // namespace mert
|
2008-05-14 12:46:15 +04:00
|
|
|
|
2012-06-30 23:23:45 +04:00
|
|
|
namespace MosesTuning
|
|
|
|
{
|
2012-12-07 00:12:24 +04:00
|
|
|
|
2012-06-30 23:23:45 +04:00
|
|
|
class PreProcessFilter;
|
|
|
|
class ScoreStats;
|
|
|
|
|
2012-06-22 21:19:16 +04:00
|
|
|
enum ScorerRegularisationStrategy {REG_NONE, REG_AVERAGE, REG_MINIMUM};
|
|
|
|
|
2008-05-14 23:47:34 +04:00
|
|
|
/**
|
2011-11-12 03:58:23 +04:00
|
|
|
* Superclass of all scorers and dummy implementation.
|
|
|
|
*
|
|
|
|
* In order to add a new scorer it should be sufficient to override the members
|
|
|
|
* prepareStats(), setReferenceFiles() and score() (or calculateScore()).
|
|
|
|
*/
|
2011-02-24 15:42:19 +03:00
|
|
|
class Scorer
|
|
|
|
{
|
2013-05-29 21:16:15 +04:00
|
|
|
public:
|
2012-05-10 02:51:05 +04:00
|
|
|
Scorer(const std::string& name, const std::string& config);
|
2012-02-01 16:19:25 +04:00
|
|
|
virtual ~Scorer();
|
2008-06-24 23:27:18 +04:00
|
|
|
|
2011-02-24 15:42:19 +03:00
|
|
|
/**
|
2011-11-12 03:58:23 +04:00
|
|
|
* Return the number of statistics needed for the computation of the score.
|
|
|
|
*/
|
2012-05-10 02:51:05 +04:00
|
|
|
virtual std::size_t NumberOfScores() const = 0;
|
2011-02-24 15:42:19 +03:00
|
|
|
|
2014-09-17 17:14:11 +04:00
|
|
|
/**
|
|
|
|
* Calculate score based on a vector of sufficient statistics.
|
|
|
|
*/
|
|
|
|
virtual float calculateScore(const std::vector<ScoreStatsType>& totals) const = 0;
|
|
|
|
|
|
|
|
float calculateSentenceLevelBackgroundScore(const std::vector<ScoreStatsType>& totals, const std::vector<ScoreStatsType>& bg) {
|
2018-05-18 17:18:19 +03:00
|
|
|
if (m_name.compare("CHRF")!=0){
|
|
|
|
std::vector<ScoreStatsType> stats(totals.size());
|
|
|
|
for(size_t i=0; i<stats.size(); i++)
|
|
|
|
stats[i] = totals[i]+bg[i];
|
|
|
|
// Get score and scale by reference length (as per Chiang et al 08)
|
|
|
|
return calculateScore(stats) * getReferenceLength(stats);
|
|
|
|
}
|
|
|
|
else{
|
|
|
|
return calculateScore(totals);
|
|
|
|
}
|
2014-09-17 17:14:11 +04:00
|
|
|
}
|
|
|
|
|
2011-02-24 15:42:19 +03:00
|
|
|
/**
|
2011-11-12 03:58:23 +04:00
|
|
|
* Set the reference files. This must be called before prepareStats().
|
|
|
|
*/
|
2012-05-10 02:51:05 +04:00
|
|
|
virtual void setReferenceFiles(const std::vector<std::string>& referenceFiles) {
|
2012-02-01 16:26:47 +04:00
|
|
|
// do nothing
|
2011-02-24 15:42:19 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Process the given guessed text, corresponding to the given reference sindex
|
|
|
|
* and add the appropriate statistics to the entry.
|
2011-11-12 03:58:23 +04:00
|
|
|
*/
|
2012-05-10 02:51:05 +04:00
|
|
|
virtual void prepareStats(std::size_t sindex, const std::string& text, ScoreStats& entry) {
|
2012-02-01 16:26:47 +04:00
|
|
|
// do nothing.
|
|
|
|
}
|
2008-06-24 23:27:18 +04:00
|
|
|
|
2012-05-10 02:51:05 +04:00
|
|
|
virtual void prepareStats(const std::string& sindex, const std::string& text, ScoreStats& entry) {
|
|
|
|
this->prepareStats(static_cast<std::size_t>(atoi(sindex.c_str())), text, entry);
|
2011-02-24 15:42:19 +03:00
|
|
|
}
|
2008-05-14 23:47:34 +04:00
|
|
|
|
2011-02-24 15:42:19 +03:00
|
|
|
/**
|
2011-11-12 03:58:23 +04:00
|
|
|
* Score using each of the candidate index, then go through the diffs
|
|
|
|
* applying each in turn, and calculating a new score each time.
|
|
|
|
*/
|
2011-02-24 15:42:19 +03:00
|
|
|
virtual void score(const candidates_t& candidates, const diffs_t& diffs,
|
2012-02-26 21:53:08 +04:00
|
|
|
statscores_t& scores) const = 0;
|
|
|
|
/*
|
|
|
|
{
|
2011-02-24 15:42:19 +03:00
|
|
|
//dummy impl
|
2012-02-01 15:54:20 +04:00
|
|
|
if (!m_score_data) {
|
2011-02-24 15:42:19 +03:00
|
|
|
throw runtime_error("score data not loaded");
|
|
|
|
}
|
|
|
|
scores.push_back(0);
|
2012-05-10 02:51:05 +04:00
|
|
|
for (std::size_t i = 0; i < diffs.size(); ++i) {
|
2011-02-24 15:42:19 +03:00
|
|
|
scores.push_back(0);
|
|
|
|
}
|
|
|
|
}
|
2012-02-26 21:53:08 +04:00
|
|
|
*/
|
2008-05-14 23:47:34 +04:00
|
|
|
|
2011-02-24 15:42:19 +03:00
|
|
|
/**
|
2011-11-12 03:58:23 +04:00
|
|
|
* Calculate the score of the sentences corresponding to the list of candidate
|
|
|
|
* indices. Each index indicates the 1-best choice from the n-best list.
|
|
|
|
*/
|
2012-12-07 00:12:24 +04:00
|
|
|
float score(const candidates_t& candidates) const;
|
2008-05-14 23:47:34 +04:00
|
|
|
|
2012-05-10 02:51:05 +04:00
|
|
|
const std::string& getName() const {
|
2012-02-01 15:54:20 +04:00
|
|
|
return m_name;
|
2011-02-24 15:42:19 +03:00
|
|
|
}
|
2008-05-14 16:23:58 +04:00
|
|
|
|
2012-05-10 02:51:05 +04:00
|
|
|
std::size_t getReferenceSize() const {
|
2012-02-01 15:54:20 +04:00
|
|
|
if (m_score_data) {
|
|
|
|
return m_score_data->size();
|
2011-02-24 15:42:19 +03:00
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
2008-05-14 16:23:58 +04:00
|
|
|
|
2014-09-17 17:14:11 +04:00
|
|
|
/**
|
|
|
|
* Based on vector of sufficient statistics, return length of reference.
|
|
|
|
*/
|
|
|
|
virtual float getReferenceLength(const std::vector<ScoreStatsType>& totals) const = 0;
|
|
|
|
|
2011-02-24 15:42:19 +03:00
|
|
|
/**
|
2011-11-12 03:58:23 +04:00
|
|
|
* Set the score data, prior to scoring.
|
|
|
|
*/
|
2012-02-26 21:53:08 +04:00
|
|
|
virtual void setScoreData(ScoreData* data) {
|
2012-02-01 15:54:20 +04:00
|
|
|
m_score_data = data;
|
2011-02-24 15:42:19 +03:00
|
|
|
}
|
|
|
|
|
2012-07-10 12:25:00 +04:00
|
|
|
/**
|
|
|
|
* The scorer returns if it uses the reference alignment data
|
|
|
|
* for permutation distance scores
|
|
|
|
**/
|
|
|
|
virtual bool useAlignment() const {
|
|
|
|
//cout << "Scorer::useAlignment returning false " << endl;
|
|
|
|
return false;
|
|
|
|
};
|
|
|
|
|
2012-02-28 05:27:23 +04:00
|
|
|
/**
|
|
|
|
* Set the factors, which should be used for this metric
|
|
|
|
*/
|
2012-05-10 02:51:05 +04:00
|
|
|
virtual void setFactors(const std::string& factors);
|
2012-02-28 05:27:23 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
mert::Vocabulary* GetVocab() const {
|
|
|
|
return m_vocab;
|
|
|
|
}
|
2012-03-19 22:43:04 +04:00
|
|
|
|
2012-05-09 21:21:41 +04:00
|
|
|
/**
|
|
|
|
* Set unix filter, which will be used to preprocess the sentences
|
|
|
|
*/
|
2012-05-10 02:51:05 +04:00
|
|
|
virtual void setFilter(const std::string& filterCommand);
|
2012-05-10 01:57:44 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
private:
|
2012-05-10 02:51:05 +04:00
|
|
|
void InitConfig(const std::string& config);
|
2012-02-01 16:26:47 +04:00
|
|
|
|
2012-05-09 21:21:41 +04:00
|
|
|
/**
|
|
|
|
* Take the factored sentence and return the desired factors
|
|
|
|
*/
|
2012-05-10 02:51:05 +04:00
|
|
|
std::string applyFactors(const std::string& sentece) const;
|
2012-05-09 21:21:41 +04:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Preprocess the sentence with the filter (if given)
|
|
|
|
*/
|
2012-05-10 02:51:05 +04:00
|
|
|
std::string applyFilter(const std::string& sentence) const;
|
2012-05-09 21:21:41 +04:00
|
|
|
|
2012-05-10 02:51:05 +04:00
|
|
|
std::string m_name;
|
2012-03-19 22:43:04 +04:00
|
|
|
mert::Vocabulary* m_vocab;
|
2012-05-10 02:51:05 +04:00
|
|
|
std::map<std::string, std::string> m_config;
|
|
|
|
std::vector<int> m_factors;
|
2013-11-14 23:15:53 +04:00
|
|
|
|
|
|
|
#if defined(__GLIBCXX__) || defined(__GLIBCPP__)
|
2012-05-10 01:57:44 +04:00
|
|
|
PreProcessFilter* m_filter;
|
2013-11-14 23:15:53 +04:00
|
|
|
#endif
|
2012-02-01 16:19:25 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
protected:
|
2012-02-01 15:54:20 +04:00
|
|
|
ScoreData* m_score_data;
|
|
|
|
bool m_enable_preserve_case;
|
2011-02-24 15:42:19 +03:00
|
|
|
|
|
|
|
/**
|
2011-11-12 03:58:23 +04:00
|
|
|
* Get value of config variable. If not provided, return default.
|
|
|
|
*/
|
2012-05-10 02:51:05 +04:00
|
|
|
std::string getConfig(const std::string& key, const std::string& def="") const {
|
|
|
|
std::map<std::string,std::string>::const_iterator i = m_config.find(key);
|
2012-02-01 15:54:20 +04:00
|
|
|
if (i == m_config.end()) {
|
2011-02-24 15:42:19 +03:00
|
|
|
return def;
|
|
|
|
} else {
|
|
|
|
return i->second;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Tokenise line and encode.
|
2012-03-19 18:21:02 +04:00
|
|
|
* Note: We assume that all tokens are separated by whitespaces.
|
2011-11-12 03:58:23 +04:00
|
|
|
*/
|
2015-04-30 21:44:29 +03:00
|
|
|
void TokenizeAndEncode(const std::string& line, std::vector<int>& encoded) const;
|
2012-05-09 21:21:41 +04:00
|
|
|
|
2012-12-07 00:12:24 +04:00
|
|
|
/*
|
|
|
|
* Tokenize functions for testing only.
|
|
|
|
*/
|
2015-04-30 21:44:29 +03:00
|
|
|
void TokenizeAndEncodeTesting(const std::string& line, std::vector<int>& encoded) const;
|
2012-12-07 00:12:24 +04:00
|
|
|
|
2012-05-09 21:21:41 +04:00
|
|
|
/**
|
|
|
|
* Every inherited scorer should call this function for each sentence
|
|
|
|
*/
|
2013-05-29 21:16:15 +04:00
|
|
|
std::string preprocessSentence(const std::string& sentence) const {
|
2012-05-09 21:21:41 +04:00
|
|
|
return applyFactors(applyFilter(sentence));
|
|
|
|
}
|
|
|
|
|
2008-05-14 12:46:15 +04:00
|
|
|
};
|
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
namespace
|
|
|
|
{
|
2012-12-07 00:12:24 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
//regularisation strategies
|
|
|
|
inline float score_min(const statscores_t& scores, size_t start, size_t end)
|
|
|
|
{
|
|
|
|
float min = std::numeric_limits<float>::max();
|
|
|
|
for (size_t i = start; i < end; ++i) {
|
|
|
|
if (scores[i] < min) {
|
|
|
|
min = scores[i];
|
2012-06-22 21:19:16 +04:00
|
|
|
}
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
return min;
|
|
|
|
}
|
2012-12-07 00:12:24 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
inline float score_average(const statscores_t& scores, size_t start, size_t end)
|
|
|
|
{
|
|
|
|
if ((end - start) < 1) {
|
|
|
|
// this shouldn't happen
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
float total = 0;
|
|
|
|
for (size_t j = start; j < end; ++j) {
|
|
|
|
total += scores[j];
|
2012-06-22 21:19:16 +04:00
|
|
|
}
|
2012-12-07 00:12:24 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
return total / (end - start);
|
|
|
|
}
|
|
|
|
|
2012-06-22 21:19:16 +04:00
|
|
|
} // namespace
|
|
|
|
|
2012-06-30 23:23:45 +04:00
|
|
|
}
|
2012-06-24 06:51:48 +04:00
|
|
|
|