mosesdecoder/mert/HwcmScorer.cpp
2014-09-22 10:49:20 +01:00

165 lines
5.2 KiB
C++

#include "HwcmScorer.h"
#include <fstream>
#include "ScoreStats.h"
#include "Util.h"
#include "util/tokenize_piece.hh"
// HWCM score (Liu and Gildea, 2005). Implements F1 instead of precision for better modelling of hypothesis length.
// assumes dependency trees on target side (generated by scripts/training/wrappers/conll2mosesxml.py ; use with option --brackets for reference).
// reads reference trees from separate file {REFERENCE_FILE}.trees to support mix of string-based and tree-based metrics.
using namespace std;
namespace MosesTuning
{
HwcmScorer::HwcmScorer(const string& config)
: StatisticsBasedScorer("HWCM",config) {}
HwcmScorer::~HwcmScorer() {}
void HwcmScorer::setReferenceFiles(const vector<string>& referenceFiles)
{
// For each line in the reference file, create a tree object
if (referenceFiles.size() != 1) {
throw runtime_error("HWCM only supports a single reference");
}
m_ref_trees.clear();
m_ref_hwc.clear();
ifstream in((referenceFiles[0] + ".trees").c_str());
if (!in) {
throw runtime_error("Unable to open " + referenceFiles[0] + ".trees");
}
string line;
while (getline(in,line)) {
line = this->preprocessSentence(line);
TreePointer tree (boost::make_shared<InternalTree>(line));
m_ref_trees.push_back(tree);
vector<map<string, int> > hwc (kHwcmOrder);
vector<string> history(kHwcmOrder);
extractHeadWordChain(tree, history, hwc);
m_ref_hwc.push_back(hwc);
vector<int> totals(kHwcmOrder);
for (size_t i = 0; i < kHwcmOrder; i++) {
for (map<string, int>::const_iterator it = m_ref_hwc.back()[i].begin(); it != m_ref_hwc.back()[i].end(); it++) {
totals[i] += it->second;
}
}
m_ref_lengths.push_back(totals);
}
TRACE_ERR(endl);
}
void HwcmScorer::extractHeadWordChain(TreePointer tree, vector<string> & history, vector<map<string, int> > & hwc) {
if (tree->GetLength() > 0) {
string head = getHead(tree);
if (head.empty()) {
for (std::vector<TreePointer>::const_iterator it = tree->GetChildren().begin(); it != tree->GetChildren().end(); ++it) {
extractHeadWordChain(*it, history, hwc);
}
}
else {
vector<string> new_history(kHwcmOrder);
new_history[0] = head;
hwc[0][head]++;
for (size_t hist_idx = 0; hist_idx < kHwcmOrder-1; hist_idx++) {
if (!history[hist_idx].empty()) {
string chain = history[hist_idx] + " " + head;
hwc[hist_idx+1][chain]++;
if (hist_idx+2 < kHwcmOrder) {
new_history[hist_idx+1] = chain;
}
}
}
for (std::vector<TreePointer>::const_iterator it = tree->GetChildren().begin(); it != tree->GetChildren().end(); ++it) {
extractHeadWordChain(*it, new_history, hwc);
}
}
}
}
string HwcmScorer::getHead(TreePointer tree) {
// assumption (only true for dependency parse: each constituent has a preterminal label, and corresponding terminal is head)
// if constituent has multiple preterminals, first one is picked; if it has no preterminals, empty string is returned
for (std::vector<TreePointer>::const_iterator it = tree->GetChildren().begin(); it != tree->GetChildren().end(); ++it)
{
TreePointer child = *it;
if (child->GetLength() == 1 && child->GetChildren()[0]->IsTerminal()) {
return child->GetChildren()[0]->GetLabel();
}
}
return "";
}
void HwcmScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
{
if (sid >= m_ref_trees.size()) {
stringstream msg;
msg << "Sentence id (" << sid << ") not found in reference set";
throw runtime_error(msg.str());
}
string sentence = this->preprocessSentence(text);
// if sentence has '|||', assume that tree is in second position (n-best-list);
// otherwise, assume it is in first position (calling 'evaluate' with tree as reference)
util::TokenIter<util::MultiCharacter> it(sentence, util::MultiCharacter("|||"));
++it;
if (it) {
sentence = it->as_string();
}
TreePointer tree (boost::make_shared<InternalTree>(sentence));
vector<map<string, int> > hwc_test (kHwcmOrder);
vector<string> history(kHwcmOrder);
extractHeadWordChain(tree, history, hwc_test);
ostringstream stats;
for (size_t i = 0; i < kHwcmOrder; i++) {
int correct = 0;
int test_total = 0;
for (map<string, int>::const_iterator it = hwc_test[i].begin(); it != hwc_test[i].end(); it++) {
test_total += it->second;
map<string, int>::const_iterator it2 = m_ref_hwc[sid][i].find(it->first);
if (it2 != m_ref_hwc[sid][i].end()) {
correct += std::min(it->second, it2->second);
}
}
stats << correct << " " << test_total << " " << m_ref_lengths[sid][i] << " " ;
}
string stats_str = stats.str();
entry.set(stats_str);
}
float HwcmScorer::calculateScore(const vector<ScoreStatsType>& comps) const
{
float precision = 0;
float recall = 0;
for (size_t i = 0; i < kHwcmOrder; i++) {
float matches = comps[i*3];
float test_total = comps[1+(i*3)];
float ref_total = comps[2+(i*3)];
if (test_total > 0) {
precision += matches/test_total;
}
if (ref_total > 0) {
recall += matches/ref_total;
}
}
precision /= (float)kHwcmOrder;
recall /= (float)kHwcmOrder;
return (2*precision*recall)/(precision+recall); // f1-score
}
}