moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-11-09 16:04:41 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into weight-new

Browse Source
This commit is contained in:
Hieu Hoang 2012-12-06 14:48:20 +00:00
commit eb12e4c808
3 changed files with 11 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mert/BleuScorer.cpp
View File

@ -62,6 +62,7 @@ size_t BleuScorer::CountNgrams(const string& line, NgramCounts& counts,
}
for (size_t i = 0; i < encoded_tokens.size()-k+1; ++i) {
vector<int> ngram;
ngram.reserve(encoded_tokens.size());
for (size_t j = i; j < i+k && j < encoded_tokens.size(); ++j) {
ngram.push_back(encoded_tokens[j]);
}

9
mert/Data.cpp
View File

@ -153,9 +153,9 @@ void Data::loadNBest(const string &file)
getNextPound(line, temp, "|||"); //fourth field sentence score
if (line.length() > 0) {
getNextPound(line, alignment, "|||"); //fifth field (if present) is either phrase or word alignment
if (line.length() > 0) {
getNextPound(line, alignment, "|||"); //sixth field (if present) is word alignment
}
if (line.length() > 0) {
getNextPound(line, alignment, "|||"); //sixth field (if present) is word alignment
}
}
}
//TODO check alignment exists if scorers need it
@ -165,7 +165,7 @@ void Data::loadNBest(const string &file)
sentence += alignment;
}
m_scorer->prepareStats(sentence_index, sentence, scoreentry);
m_score_data->add(scoreentry, sentence_index);
// examine first line for name of features
@ -281,4 +281,3 @@ void Data::createShards(size_t shard_count, float shard_size, const string& scor
}
}

11
mert/Ngram.h
View File

@ -2,12 +2,13 @@
#define MERT_NGRAM_H_
#include <vector>
#include <map>
#include <string>
#include <boost/unordered_map.hpp>
namespace MosesTuning
{
/** A simple STL-std::map based n-gram counts. Basically, we provide
* typical accessors and mutaors, but we intentionally does not allow
* erasing elements.
@ -35,8 +36,8 @@ class NgramCounts {
typedef std::vector<int> Key;
typedef int Value;
typedef std::map<Key, Value, NgramComparator>::iterator iterator;
typedef std::map<Key, Value, NgramComparator>::const_iterator const_iterator;
typedef boost::unordered_map<Key, Value>::iterator iterator;
typedef boost::unordered_map<Key, Value>::const_iterator const_iterator;
NgramCounts() : kDefaultCount(1) { }
virtual ~NgramCounts() { }
@ -95,7 +96,7 @@ class NgramCounts {
private:
const int kDefaultCount;
std::map<Key, Value, NgramComparator> m_counts;
boost::unordered_map<Key, Value> m_counts;
};
}