moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

Avoid "using namespace std" in headers.

Browse Source
This commit is contained in:
Tetsuo Kiso 2012-05-10 07:51:05 +09:00
parent aa269bdb4d
commit 9c9d88a78a
35 changed files with 227 additions and 239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mert/BleuScorer.cpp
View File

@ -14,6 +14,8 @@
#include "Util.h"
#include "Vocabulary.h"
using namespace std;
namespace {
// configure regularisation

22
mert/BleuScorer.h
View File

@ -10,8 +10,6 @@
#include "Scorer.h"
#include "ScopedVector.h"
using namespace std;
const int kBleuNgramOrder = 4;
class NgramCounts;
@ -29,15 +27,15 @@ public:
SHORTEST
};
explicit BleuScorer(const string& config = "");
explicit BleuScorer(const std::string& config = "");
~BleuScorer();
virtual void setReferenceFiles(const vector<string>& referenceFiles);
virtual void prepareStats(size_t sid, const string& text, ScoreStats& entry);
virtual float calculateScore(const vector<int>& comps) const;
virtual size_t NumberOfScores() const { return 2 * kBleuNgramOrder + 1; }
virtual void setReferenceFiles(const std::vector<std::string>& referenceFiles);
virtual void prepareStats(std::size_t sid, const std::string& text, ScoreStats& entry);
virtual float calculateScore(const std::vector<int>& comps) const;
virtual std::size_t NumberOfScores() const { return 2 * kBleuNgramOrder + 1; }
int CalcReferenceLength(size_t sentence_id, size_t length);
int CalcReferenceLength(std::size_t sentence_id, std::size_t length);
ReferenceLengthType GetReferenceLengthType() const { return m_ref_length_type; }
void SetReferenceLengthType(ReferenceLengthType type) { m_ref_length_type = type; }
@ -47,14 +45,14 @@ public:
/**
* Count the ngrams of each type, up to the given length in the input line.
*/
size_t CountNgrams(const string& line, NgramCounts& counts, unsigned int n);
std::size_t CountNgrams(const std::string& line, NgramCounts& counts, unsigned int n);
void DumpCounts(std::ostream* os, const NgramCounts& counts) const;
bool OpenReference(const char* filename, size_t file_id);
bool OpenReference(const char* filename, std::size_t file_id);
// NOTE: this function is used for unit testing.
bool OpenReferenceStream(std::istream* is, size_t file_id);
bool OpenReferenceStream(std::istream* is, std::size_t file_id);
private:
ReferenceLengthType m_ref_length_type;
@ -70,6 +68,6 @@ private:
/** Computes sentence-level BLEU+1 score.
* This function is used in PRO.
*/
float sentenceLevelBleuPlusOne(const vector<float>& stats);
float sentenceLevelBleuPlusOne(const std::vector<float>& stats);
#endif // MERT_BLEU_SCORER_H_

4
mert/BleuScorerTest.cpp
View File

@ -220,7 +220,7 @@ BOOST_AUTO_TEST_CASE(bleu_clipped_counts) {
BOOST_AUTO_TEST_CASE(calculate_actual_score) {
BOOST_REQUIRE(4 == kBleuNgramOrder);
vector<int> stats(2 * kBleuNgramOrder + 1);
std::vector<int> stats(2 * kBleuNgramOrder + 1);
BleuScorer scorer;
// unigram
@ -247,7 +247,7 @@ BOOST_AUTO_TEST_CASE(calculate_actual_score) {
BOOST_AUTO_TEST_CASE(sentence_level_bleu) {
BOOST_REQUIRE(4 == kBleuNgramOrder);
vector<float> stats(2 * kBleuNgramOrder + 1);
std::vector<float> stats(2 * kBleuNgramOrder + 1);
// unigram
stats[0] = 6.0;

2
mert/CderScorer.cpp
View File

@ -4,6 +4,8 @@
#include <fstream>
#include <stdexcept>
using namespace std;
namespace {
inline int CalcDistance(int word1, int word2) {

20
mert/CderScorer.h
View File

@ -6,34 +6,32 @@
#include "Types.h"
#include "Scorer.h"
using namespace std;
/**
* CderScorer class can compute both CDER and WER metric.
*/
class CderScorer: public StatisticsBasedScorer {
public:
explicit CderScorer(const string& config, bool allowed_long_jumps = true);
explicit CderScorer(const std::string& config, bool allowed_long_jumps = true);
~CderScorer();
virtual void setReferenceFiles(const vector<string>& referenceFiles);
virtual void setReferenceFiles(const std::vector<std::string>& referenceFiles);
virtual void prepareStats(size_t sid, const string& text, ScoreStats& entry);
virtual void prepareStats(std::size_t sid, const std::string& text, ScoreStats& entry);
virtual void prepareStatsVector(size_t sid, const string& text, vector<int>& stats);
virtual void prepareStatsVector(std::size_t sid, const std::string& text, std::vector<int>& stats);
virtual size_t NumberOfScores() const { return 2; }
virtual std::size_t NumberOfScores() const { return 2; }
virtual float calculateScore(const vector<int>& comps) const;
virtual float calculateScore(const std::vector<int>& comps) const;
private:
bool m_allowed_long_jumps;
typedef vector<int> sent_t;
vector<vector<sent_t> > m_ref_sentences;
typedef std::vector<int> sent_t;
std::vector<std::vector<sent_t> > m_ref_sentences;
void computeCD(const sent_t& cand, const sent_t& ref,
vector<int>& stats) const;
std::vector<int>& stats) const;
// no copying allowed
CderScorer(const CderScorer&);

2
mert/DataTest.cpp
View File

@ -30,7 +30,7 @@ BOOST_AUTO_TEST_CASE(shard_basic) {
data.getScoreData()->add(sa3);
data.getScoreData()->add(sa4);
vector<Data> shards;
std::vector<Data> shards;
data.createShards(2,0,"",shards);
BOOST_CHECK_EQUAL(shards.size(),2);

3
mert/FeatureArray.cpp
View File

@ -6,11 +6,14 @@
*
*/
#include <iostream>
#include <fstream>
#include "FeatureArray.h"
#include "FileStream.h"
#include "Util.h"
using namespace std;
FeatureArray::FeatureArray()
: m_index(""), m_num_features(0), m_sparse_flag(false) {}

24
mert/FeatureArray.h
View File

@ -10,11 +10,9 @@
#define MERT_FEATURE_ARRAY_H_
#include <vector>
#include <iostream>
#include <iosfwd>
#include "FeatureStats.h"
using namespace std;
const char FEATURES_TXT_BEGIN[] = "FEATURES_TXT_BEGIN_0";
const char FEATURES_TXT_END[] = "FEATURES_TXT_END_0";
const char FEATURES_BIN_BEGIN[] = "FEATURES_BIN_BEGIN_0";
@ -27,7 +25,7 @@ private:
// the index inside the vector.
std::string m_index;
featarray_t m_array;
size_t m_num_features;
std::size_t m_num_features;
std::string m_features;
bool m_sparse_flag;
@ -42,27 +40,27 @@ public:
std::string getIndex() const { return m_index; }
void setIndex(const std::string& value) { m_index = value; }
FeatureStats& get(size_t i) { return m_array.at(i); }
const FeatureStats& get(size_t i) const { return m_array.at(i); }
FeatureStats& get(std::size_t i) { return m_array.at(i); }
const FeatureStats& get(std::size_t i) const { return m_array.at(i); }
void add(FeatureStats& e) { m_array.push_back(e); }
//ADDED BY TS
void swap(size_t i, size_t j) {
void swap(std::size_t i, std::size_t j) {
std::swap(m_array[i], m_array[j]);
}
void resize(size_t new_size) {
void resize(std::size_t new_size) {
m_array.resize(std::min(new_size, m_array.size()));
}
//END_ADDED
void merge(FeatureArray& e);
size_t size() const { return m_array.size(); }
std::size_t size() const { return m_array.size(); }
size_t NumberOfFeatures() const { return m_num_features; }
void NumberOfFeatures(size_t v) { m_num_features = v; }
std::size_t NumberOfFeatures() const { return m_num_features; }
void NumberOfFeatures(std::size_t v) { m_num_features = v; }
std::string Features() const { return m_features; }
void Features(const std::string& f) { m_features = f; }
@ -73,8 +71,8 @@ public:
void save(const std::string &file, bool bin=false);
void save(bool bin=false);
void loadtxt(std::istream* is, size_t n);
void loadbin(std::istream* is, size_t n);
void loadtxt(std::istream* is, std::size_t n);
void loadbin(std::istream* is, std::size_t n);
void load(std::istream* is);
void load(const std::string &file);

9
mert/FeatureData.h
View File

@ -100,10 +100,10 @@ public:
std::string getFeatureName(std::size_t idx) const {
if (idx >= m_index_to_feature_name.size())
throw runtime_error("Error: you required an too big index");
throw std::runtime_error("Error: you required an too big index");
std::map<std::size_t, std::string>::const_iterator it = m_index_to_feature_name.find(idx);
if (it == m_index_to_feature_name.end()) {
throw runtime_error("Error: specified id is unknown: " + idx);
throw std::runtime_error("Error: specified id is unknown: " + idx);
} else {
return it->second;
}
@ -113,8 +113,9 @@ public:
std::map<std::string, std::size_t>::const_iterator it = m_feature_name_to_index.find(name);
if (it == m_feature_name_to_index.end()) {
std::string msg = "Error: feature " + name + " is unknown. Known features: ";
for (std::map<std::string, std::size_t>::const_iterator it = m_feature_name_to_index.begin(); it != m_feature_name_to_index.end(); it++) {
msg += it->first;
for (std::map<std::string, std::size_t>::const_iterator cit = m_feature_name_to_index.begin();
cit != m_feature_name_to_index.end(); cit++) {
msg += cit->first;
msg += ", ";
}

2
mert/FeatureDataTest.cpp
View File

@ -12,7 +12,7 @@ void CheckFeatureMap(const FeatureData* feature_data,
for (int i = 0; i < num_feature; ++i) {
std::stringstream ss;
ss << str << "_" << i;
const string& s = ss.str();
const std::string& s = ss.str();
BOOST_CHECK_EQUAL(feature_data->getFeatureIndex(s), *cnt);
BOOST_CHECK_EQUAL(feature_data->getFeatureName(*cnt).c_str(), s);
++(*cnt);

2
mert/FeatureStats.cpp
View File

@ -12,6 +12,8 @@
#include <cmath>
#include "Util.h"
using namespace std;
namespace {
const int kAvailableSize = 8;
} // namespace

32
mert/FeatureStats.h
View File

@ -16,20 +16,18 @@
#include <vector>
#include "Types.h"
using namespace std;
// Minimal sparse vector
class SparseVector {
public:
typedef std::map<size_t,FeatureStatsType> fvector_t;
typedef std::map<std::string, size_t> name2id_t;
typedef std::map<std::size_t,FeatureStatsType> fvector_t;
typedef std::map<std::string, std::size_t> name2id_t;
typedef std::vector<std::string> id2name_t;
FeatureStatsType get(const std::string& name) const;
FeatureStatsType get(size_t id) const;
FeatureStatsType get(std::size_t id) const;
void set(const std::string& name, FeatureStatsType value);
void clear();
size_t size() const { return m_fvector.size(); }
std::size_t size() const { return m_fvector.size(); }
void write(std::ostream& out, const std::string& sep = " ") const;
@ -46,8 +44,8 @@ SparseVector operator-(const SparseVector& lhs, const SparseVector& rhs);
class FeatureStats
{
private:
size_t m_available_size;
size_t m_entries;
std::size_t m_available_size;
std::size_t m_entries;
// TODO: Use smart pointer for exceptional-safety.
featstats_t m_array;
@ -55,7 +53,7 @@ private:
public:
FeatureStats();
explicit FeatureStats(const size_t size);
explicit FeatureStats(const std::size_t size);
explicit FeatureStats(std::string &theString);
~FeatureStats();
@ -69,7 +67,7 @@ public:
bool isfull() const { return (m_entries < m_available_size) ? 0 : 1; }
void expand();
void add(FeatureStatsType v);
void addSparse(const string& name, FeatureStatsType v);
void addSparse(const std::string& name, FeatureStatsType v);
void clear() {
memset((void*)m_array, 0, GetArraySizeWithBytes());
@ -81,23 +79,23 @@ public:
clear();
}
FeatureStatsType get(size_t i) { return m_array[i]; }
FeatureStatsType get(size_t i)const { return m_array[i]; }
FeatureStatsType get(std::size_t i) { return m_array[i]; }
FeatureStatsType get(std::size_t i)const { return m_array[i]; }
featstats_t getArray() const { return m_array; }
const SparseVector& getSparse() const { return m_map; }
void set(std::string &theString);
inline size_t bytes() const { return GetArraySizeWithBytes(); }
inline std::size_t bytes() const { return GetArraySizeWithBytes(); }
size_t GetArraySizeWithBytes() const {
std::size_t GetArraySizeWithBytes() const {
return m_entries * sizeof(FeatureStatsType);
}
size_t size() const { return m_entries; }
std::size_t size() const { return m_entries; }
size_t available() const { return m_available_size; }
std::size_t available() const { return m_available_size; }
void savetxt(const std::string &file);
void savetxt(std::ostream* os);
@ -111,7 +109,7 @@ public:
/**
* Write the whole object to a stream.
*/
friend ostream& operator<<(ostream& o, const FeatureStats& e);
friend std::ostream& operator<<(std::ostream& o, const FeatureStats& e);
};
//ADEED_BY_TS

16
mert/InterpolatedScorer.h
View File

@ -15,17 +15,17 @@ class InterpolatedScorer : public Scorer
{
public:
// name would be: "HAMMING,BLEU" or similar
InterpolatedScorer(const string& name, const string& config);
InterpolatedScorer(const std::string& name, const std::string& config);
virtual ~InterpolatedScorer() {}
virtual void score(const candidates_t& candidates, const diffs_t& diffs,
statscores_t& scores) const;
virtual void setReferenceFiles(const vector<string>& referenceFiles);
virtual void prepareStats(size_t sid, const string& text, ScoreStats& entry);
virtual void setReferenceFiles(const std::vector<std::string>& referenceFiles);
virtual void prepareStats(std::size_t sid, const std::string& text, ScoreStats& entry);
virtual size_t NumberOfScores() const {
size_t sz = 0;
virtual std::size_t NumberOfScores() const {
std::size_t sz = 0;
for (ScopedVector<Scorer>::const_iterator itsc = m_scorers.begin();
itsc != m_scorers.end(); ++itsc) {
sz += (*itsc)->NumberOfScores();
@ -38,9 +38,9 @@ public:
/**
* Set the factors, which should be used for this metric
*/
virtual void setFactors(const string& factors);
virtual void setFactors(const std::string& factors);
virtual void setFilter(const string& filterCommand);
virtual void setFilter(const std::string& filterCommand);
protected:
ScopedVector<Scorer> m_scorers;
@ -49,7 +49,7 @@ protected:
// by Scorer objects.
ScopedVector<ScoreData> m_scorers_score_data;
vector<float> m_scorer_weights;
std::vector<float> m_scorer_weights;
};
#endif // MERT_INTERPOLATED_SCORER_H_

1
mert/MergeScorer.cpp
View File

@ -11,6 +11,7 @@
#include "TER/tercalc.h"
#include "TER/terAlignment.h"
using namespace std;
using namespace TERCpp;
MergeScorer::MergeScorer(const string& config)

14
mert/MergeScorer.h
View File

@ -1,15 +1,11 @@
#ifndef MERT_MERGE_SCORER_H_
#define MERT_MERGE_SCORER_H_
#include <iostream>
#include <set>
#include <string>
#include <vector>
#include "Scorer.h"
using namespace std;
class PerScorer;
class ScoreStats;
@ -20,16 +16,16 @@ const int kMergeScorerLength = 4;
*/
class MergeScorer: public StatisticsBasedScorer {
public:
explicit MergeScorer(const string& config = "");
explicit MergeScorer(const std::string& config = "");
~MergeScorer();
virtual void setReferenceFiles(const vector<string>& referenceFiles);
virtual void prepareStats(size_t sid, const string& text, ScoreStats& entry);
virtual size_t NumberOfScores() const { return 0; }
virtual void setReferenceFiles(const std::vector<std::string>& referenceFiles);
virtual void prepareStats(std::size_t sid, const std::string& text, ScoreStats& entry);
virtual std::size_t NumberOfScores() const { return 0; }
protected:
friend class PerScorer;
virtual float calculateScore(const vector<int>& comps) const;
virtual float calculateScore(const std::vector<int>& comps) const;
private:
// no copying allowed

2
mert/NgramTest.cpp
View File

@ -17,7 +17,7 @@ BOOST_AUTO_TEST_CASE(ngram_basic) {
NgramCounts::const_iterator it = counts.find(key);
BOOST_CHECK(it != counts.end());
BOOST_CHECK_EQUAL(it->first.size(), key.size());
for (size_t i = 0; i < key.size(); ++i) {
for (std::size_t i = 0; i < key.size(); ++i) {
BOOST_CHECK_EQUAL(it->first[i], key[i]);
}
BOOST_CHECK_EQUAL(it->second, 1);

30
mert/Optimizer.h
View File

@ -8,9 +8,7 @@
#include "Scorer.h"
#include "Types.h"
using namespace std;
static const float kMaxFloat = numeric_limits<float>::max();
static const float kMaxFloat = std::numeric_limits<float>::max();
class Point;
@ -24,10 +22,10 @@ protected:
FeatureDataHandle m_feature_data; // no accessor for them only child can use them
unsigned int m_num_random_directions;
const vector<bool>& m_positive;
const std::vector<bool>& m_positive;
public:
Optimizer(unsigned Pd, const vector<unsigned>& i2O, const vector<bool>& positive, const vector<parameter_t>& start, unsigned int nrandom);
Optimizer(unsigned Pd, const std::vector<unsigned>& i2O, const std::vector<bool>& positive, const std::vector<parameter_t>& start, unsigned int nrandom);
void SetScorer(Scorer *scorer) { m_scorer = scorer; }
void SetFeatureData(FeatureDataHandle feature_data) { m_feature_data = feature_data; }
@ -50,18 +48,18 @@ public:
/**
* Given a set of lambdas, get the nbest for each sentence.
*/
void Get1bests(const Point& param,vector<unsigned>& bests) const;
void Get1bests(const Point& param,std::vector<unsigned>& bests) const;
/**
* Given a set of nbests, get the Statistical score.
*/
statscore_t GetStatScore(const vector<unsigned>& nbests) const {
statscore_t GetStatScore(const std::vector<unsigned>& nbests) const {
return m_scorer->score(nbests);
}
statscore_t GetStatScore(const Point& param) const;
vector<statscore_t> GetIncStatScore(const vector<unsigned>& ref, const vector<vector<pair<unsigned,unsigned> > >& diffs) const;
std::vector<statscore_t> GetIncStatScore(const std::vector<unsigned>& ref, const std::vector<std::vector<std::pair<unsigned,unsigned> > >& diffs) const;
/**
* Get the optimal Lambda and the best score in a particular direction from a given Point.
@ -79,9 +77,9 @@ class SimpleOptimizer : public Optimizer
private:
const float kEPS;
public:
SimpleOptimizer(unsigned dim, const vector<unsigned>& i2O, const vector<bool>& positive,
const vector<parameter_t>& start, unsigned int nrandom)
: Optimizer(dim, i2O, positive, start,nrandom), kEPS(0.0001) {}
SimpleOptimizer(unsigned dim, const std::vector<unsigned>& i2O, const std::vector<bool>& positive,
const std::vector<parameter_t>& start, unsigned int nrandom)
: Optimizer(dim, i2O, positive, start,nrandom), kEPS(0.0001f) {}
virtual statscore_t TrueRun(Point&) const;
};
@ -93,9 +91,9 @@ class RandomDirectionOptimizer : public Optimizer
private:
const float kEPS;
public:
RandomDirectionOptimizer(unsigned dim, const vector<unsigned>& i2O, const vector<bool>& positive,
const vector<parameter_t>& start, unsigned int nrandom)
: Optimizer(dim, i2O, positive, start, nrandom), kEPS(0.0001) {}
RandomDirectionOptimizer(unsigned dim, const std::vector<unsigned>& i2O, const std::vector<bool>& positive,
const std::vector<parameter_t>& start, unsigned int nrandom)
: Optimizer(dim, i2O, positive, start, nrandom), kEPS(0.0001f) {}
virtual statscore_t TrueRun(Point&) const;
};
@ -105,8 +103,8 @@ public:
class RandomOptimizer : public Optimizer
{
public:
RandomOptimizer(unsigned dim, const vector<unsigned>& i2O, const vector<bool>& positive,
const vector<parameter_t>& start, unsigned int nrandom)
RandomOptimizer(unsigned dim, const std::vector<unsigned>& i2O, const std::vector<bool>& positive,
const std::vector<parameter_t>& start, unsigned int nrandom)
: Optimizer(dim, i2O, positive, start, nrandom) {}
virtual statscore_t TrueRun(Point&) const;
};

6
mert/OptimizerFactory.h
View File

@ -17,7 +17,7 @@ class OptimizerFactory
NOPTIMIZER
};
static std::vector<string> GetTypeNames();
static std::vector<std::string> GetTypeNames();
// Setup optimization types.
static void SetTypeNames();
@ -27,7 +27,7 @@ class OptimizerFactory
static Optimizer* BuildOptimizer(unsigned dim,
const std::vector<unsigned>& to_optimize,
const std::vector<bool>& positive,
const std::vector<bool>& positive,
const std::vector<parameter_t>& start,
const std::string& type,
unsigned int nrandom);
@ -36,7 +36,7 @@ class OptimizerFactory
OptimizerFactory() {}
~OptimizerFactory() {}
static vector<string> m_type_names;
static std::vector<std::string> m_type_names;
};
#endif // MERT_OPTIMIZER_FACTORY_H_

8
mert/OptimizerFactoryTest.cpp
View File

@ -8,10 +8,10 @@
namespace {
inline bool CheckBuildOptimizer(unsigned dim,
const vector<unsigned>& to_optimize,
const vector<bool>& positive,
const vector<parameter_t>& start,
const string& type,
const std::vector<unsigned>& to_optimize,
const std::vector<bool>& positive,
const std::vector<parameter_t>& start,
const std::string& type,
unsigned int num_random) {
boost::scoped_ptr<Optimizer> optimizer(OptimizerFactory::BuildOptimizer(dim, to_optimize, positive, start, type, num_random));
return optimizer.get() != NULL;

16
mert/PerScorer.h
View File

@ -7,8 +7,6 @@
#include "Types.h"
#include "Scorer.h"
using namespace std;
class ScoreStats;
/**
@ -20,13 +18,13 @@ class ScoreStats;
class PerScorer: public StatisticsBasedScorer
{
public:
explicit PerScorer(const string& config = "");
explicit PerScorer(const std::string& config = "");
~PerScorer();
virtual void setReferenceFiles(const vector<string>& referenceFiles);
virtual void prepareStats(size_t sid, const string& text, ScoreStats& entry);
virtual size_t NumberOfScores() const { return 3; }
virtual float calculateScore(const vector<int>& comps) const;
virtual void setReferenceFiles(const std::vector<std::string>& referenceFiles);
virtual void prepareStats(std::size_t sid, const std::string& text, ScoreStats& entry);
virtual std::size_t NumberOfScores() const { return 3; }
virtual float calculateScore(const std::vector<int>& comps) const;
private:
// no copying allowed
@ -34,8 +32,8 @@ private:
PerScorer& operator=(const PerScorer&);
// data extracted from reference files
vector<size_t> m_ref_lengths;
vector<multiset<int> > m_ref_tokens;
std::vector<std::size_t> m_ref_lengths;
std::vector<std::multiset<int> > m_ref_tokens;
};
#endif // MERT_PER_SCORER_H_

28
mert/Point.h
View File

@ -13,7 +13,7 @@ class Optimizer;
* A class that handles the subset of the Feature weight on which
* we run the optimization.
*/
class Point : public vector<parameter_t>
class Point : public std::vector<parameter_t>
{
friend class Optimizer;
@ -21,7 +21,7 @@ private:
/**
* The indices over which we optimize.
*/
static vector<unsigned int> m_opt_indices;
static std::vector<unsigned int> m_opt_indices;
/**
* Dimension of m_opt_indices and of the parent vector.
@ -31,7 +31,7 @@ private:
/**
* Fixed weights in case of partial optimzation.
*/
static map<unsigned int,parameter_t> m_fixed_weights;
static std::map<unsigned int,parameter_t> m_fixed_weights;
/**
* Total size of the parameter space; we have
@ -43,23 +43,23 @@ private:
/**
* The limits for randomization, both vectors are of full length, m_pdim.
*/
static vector<parameter_t> m_min;
static vector<parameter_t> m_max;
static std::vector<parameter_t> m_min;
static std::vector<parameter_t> m_max;
statscore_t m_score;
public:
static unsigned int getdim() { return m_dim; }
static void setdim(size_t d) { m_dim = d; }
static void setdim(std::size_t d) { m_dim = d; }
static unsigned int getpdim() { return m_pdim; }
static void setpdim(size_t pd) { m_pdim = pd; }
static void setpdim(std::size_t pd) { m_pdim = pd; }
static void set_optindices(const vector<unsigned int>& indices) {
static void set_optindices(const std::vector<unsigned int>& indices) {
m_opt_indices = indices;
}
static const vector<unsigned int>& get_optindices() {
static const std::vector<unsigned int>& get_optindices() {
return m_opt_indices;
}
@ -68,9 +68,9 @@ public:
}
Point();
Point(const vector<parameter_t>& init,
const vector<parameter_t>& min,
const vector<parameter_t>& max);
Point(const std::vector<parameter_t>& init,
const std::vector<parameter_t>& min,
const std::vector<parameter_t>& max);
~Point();
void Randomize();
@ -84,7 +84,7 @@ public:
/**
* Write the Whole featureweight to a stream (ie m_pdim float).
*/
friend ostream& operator<<(ostream& o,const Point& P);
friend std::ostream& operator<<(std::ostream& o,const Point& P);
void Normalize() { NormalizeL2(); }
void NormalizeL2();
@ -94,7 +94,7 @@ public:
* Return a vector of size m_pdim where all weights have been
* put (including fixed ones).
*/
void GetAllWeights(vector<parameter_t>& w) const;
void GetAllWeights(std::vector<parameter_t>& w) const;
statscore_t GetScore() const { return m_score; }
void SetScore(statscore_t score) { m_score = score; }

18
mert/Reference.h
View File

@ -14,8 +14,8 @@
class Reference {
public:
// for m_length
typedef std::vector<size_t>::iterator iterator;
typedef std::vector<size_t>::const_iterator const_iterator;
typedef std::vector<std::size_t>::iterator iterator;
typedef std::vector<std::size_t>::const_iterator const_iterator;
Reference() : m_counts(new NgramCounts) { }
~Reference() { delete m_counts; }
@ -28,36 +28,36 @@ class Reference {
iterator end() { return m_length.end(); }
const_iterator end() const { return m_length.end(); }
void push_back(size_t len) { m_length.push_back(len); }
void push_back(std::size_t len) { m_length.push_back(len); }
size_t num_references() const { return m_length.size(); }
std::size_t num_references() const { return m_length.size(); }
int CalcAverage() const;
int CalcClosest(size_t length) const;
int CalcClosest(std::size_t length) const;
int CalcShortest() const;
private:
NgramCounts* m_counts;
// multiple reference lengths
std::vector<size_t> m_length;
std::vector<std::size_t> m_length;
};
// TODO(tetsuok): fix this function and related stuff.
// "average" reference length should not be calculated at sentence-level unlike "closest".
inline int Reference::CalcAverage() const {
int total = 0;
for (size_t i = 0; i < m_length.size(); ++i) {
for (std::size_t i = 0; i < m_length.size(); ++i) {
total += m_length[i];
}
return static_cast<int>(
static_cast<float>(total) / m_length.size());
}
inline int Reference::CalcClosest(size_t length) const {
inline int Reference::CalcClosest(std::size_t length) const {
int min_diff = INT_MAX;
int closest_ref_id = 0; // an index of the closest reference translation
for (size_t i = 0; i < m_length.size(); ++i) {
for (std::size_t i = 0; i < m_length.size(); ++i) {
const int ref_length = m_length[i];
const int length_diff = abs(ref_length - static_cast<int>(length));
const int abs_min_diff = abs(min_diff);

10
mert/ScopedVector.h
View File

@ -23,10 +23,10 @@ class ScopedVector {
m_vec.clear();
}
void reserve(size_t capacity) { m_vec.reserve(capacity); }
void resize(size_t size) { m_vec.resize(size); }
void reserve(std::size_t capacity) { m_vec.reserve(capacity); }
void resize(std::size_t size) { m_vec.resize(size); }
size_t size() const {return m_vec.size(); }
std::size_t size() const {return m_vec.size(); }
iterator begin() { return m_vec.begin(); }
const_iterator begin() const { return m_vec.begin(); }
@ -40,8 +40,8 @@ class ScopedVector {
std::vector<T*>* operator->() { return &m_vec; }
const std::vector<T*>* operator->() const { return &m_vec; }
T*& operator[](size_t i) { return m_vec[i]; }
const T* operator[](size_t i) const { return m_vec[i]; }
T*& operator[](std::size_t i) { return m_vec[i]; }
const T* operator[](std::size_t i) const { return m_vec[i]; }
private:
std::vector<T*> m_vec;

3
mert/ScoreData.cpp
View File

@ -8,11 +8,14 @@
#include "ScoreData.h"
#include <iostream>
#include <fstream>
#include "Scorer.h"
#include "Util.h"
#include "FileStream.h"
using namespace std;
ScoreData::ScoreData(Scorer* scorer) :
m_scorer(scorer)
{

22
mert/ScoreData.h
View File

@ -9,15 +9,13 @@
#ifndef MERT_SCORE_DATA_H_
#define MERT_SCORE_DATA_H_
#include <iostream>
#include <iosfwd>
#include <vector>
#include <stdexcept>
#include <string>
#include "ScoreArray.h"
#include "ScoreStats.h"
using namespace std;
class Scorer;
class ScoreData
@ -32,7 +30,7 @@ private:
Scorer* m_scorer;
std::string m_score_type;
size_t m_num_scores;
std::size_t m_num_scores;
public:
ScoreData(Scorer* scorer);
@ -44,11 +42,11 @@ public:
return m_array.at(getIndex(idx));
}
inline ScoreArray& get(size_t idx) {
inline ScoreArray& get(std::size_t idx) {
return m_array.at(idx);
}
inline const ScoreArray& get(size_t idx) const {
inline const ScoreArray& get(std::size_t idx) const {
return m_array.at(idx);
}
@ -60,11 +58,11 @@ public:
return (sent_idx > -1 && sent_idx < static_cast<int>(m_array.size())) ? true : false;
}
inline ScoreStats& get(size_t i, size_t j) {
inline ScoreStats& get(std::size_t i, std::size_t j) {
return m_array.at(i).get(j);
}
inline const ScoreStats& get(size_t i, size_t j) const {
inline const ScoreStats& get(std::size_t i, std::size_t j) const {
return m_array.at(i).get(j);
}
@ -77,8 +75,8 @@ public:
void add(ScoreArray& e);
void add(const ScoreStats& e, const std::string& sent_idx);
size_t NumberOfScores() const { return m_num_scores; }
size_t size() const { return m_array.size(); }
std::size_t NumberOfScores() const { return m_num_scores; }
std::size_t size() const { return m_array.size(); }
void save(const std::string &file, bool bin=false);
void save(std::ostream* os, bool bin=false);
@ -99,10 +97,10 @@ public:
return -1;
}
inline std::string getIndex(size_t idx) const {
inline std::string getIndex(std::size_t idx) const {
idx2name::const_iterator i = m_index_to_array_name.find(idx);
if (i != m_index_to_array_name.end())
throw runtime_error("there is no entry at index " + idx);
throw std::runtime_error("there is no entry at index " + idx);
return i->second;
}
};

4
mert/ScoreStats.cpp
View File

@ -8,6 +8,10 @@
#include "Util.h"
#include "ScoreStats.h"
#include <fstream>
#include <iostream>
using namespace std;
namespace {
const int kAvailableSize = 8;

37
mert/ScoreStats.h
View File

@ -10,27 +10,24 @@
#define MERT_SCORE_STATS_H_
#include <vector>
#include <iostream>
#include <fstream>
#include <iosfwd>
#include <cstdlib>
#include <cstring>
#include "Types.h"
using namespace std;
class ScoreStats
{
private:
size_t m_available_size;
size_t m_entries;
std::size_t m_available_size;
std::size_t m_entries;
// TODO: Use smart pointer for exceptional-safety.
scorestats_t m_array;
public:
ScoreStats();
explicit ScoreStats(const size_t size);
explicit ScoreStats(const std::size_t size);
~ScoreStats();
@ -46,7 +43,7 @@ public:
void add(ScoreStatsType v);
void clear() {
memset((void*)m_array, 0, GetArraySizeWithBytes());
std::memset((void*)m_array, 0, GetArraySizeWithBytes());
}
void reset() {
@ -54,8 +51,8 @@ public:
clear();
}
ScoreStatsType get(size_t i) { return m_array[i]; }
ScoreStatsType get(size_t i) const { return m_array[i]; }
ScoreStatsType get(std::size_t i) { return m_array[i]; }
ScoreStatsType get(std::size_t i) const { return m_array[i]; }
scorestats_t getArray() const { return m_array; }
void set(const std::string& str);
@ -63,34 +60,34 @@ public:
// Much more efficient than the above.
void set(const std::vector<ScoreStatsType>& stats) {
reset();
for (size_t i = 0; i < stats.size(); ++i) {
for (std::size_t i = 0; i < stats.size(); ++i) {
add(stats[i]);
}
}
size_t bytes() const { return GetArraySizeWithBytes(); }
std::size_t bytes() const { return GetArraySizeWithBytes(); }
size_t GetArraySizeWithBytes() const {
std::size_t GetArraySizeWithBytes() const {
return m_entries * sizeof(ScoreStatsType);
}
size_t size() const { return m_entries; }
std::size_t size() const { return m_entries; }
size_t available() const { return m_available_size; }
std::size_t available() const { return m_available_size; }
void savetxt(const std::string &file);
void savetxt(ostream* os);
void savebin(ostream* os);
void savetxt(std::ostream* os);
void savebin(std::ostream* os);
void savetxt();
void loadtxt(const std::string &file);
void loadtxt(istream* is);
void loadbin(istream* is);
void loadtxt(std::istream* is);
void loadbin(std::istream* is);
/**
* Write the whole object to a stream.
*/
friend ostream& operator<<(ostream& o, const ScoreStats& e);
friend std::ostream& operator<<(std::ostream& o, const ScoreStats& e);
};
//ADDED_BY_TS

2
mert/Scorer.cpp
View File

@ -6,6 +6,8 @@
#include "Singleton.h"
#include "PreProcessFilter.h"
using namespace std;
namespace {
//regularisation strategies

50
mert/Scorer.h
View File

@ -9,8 +9,6 @@
#include "Types.h"
#include "ScoreData.h"
using namespace std;
class PreProcessFilter;
class ScoreStats;
@ -29,18 +27,18 @@ class Vocabulary;
class Scorer
{
public:
Scorer(const string& name, const string& config);
Scorer(const std::string& name, const std::string& config);
virtual ~Scorer();
/**
* Return the number of statistics needed for the computation of the score.
*/
virtual size_t NumberOfScores() const = 0;
virtual std::size_t NumberOfScores() const = 0;
/**
* Set the reference files. This must be called before prepareStats().
*/
virtual void setReferenceFiles(const vector<string>& referenceFiles) {
virtual void setReferenceFiles(const std::vector<std::string>& referenceFiles) {
// do nothing
}
@ -48,12 +46,12 @@ class Scorer
* Process the given guessed text, corresponding to the given reference sindex
* and add the appropriate statistics to the entry.
*/
virtual void prepareStats(size_t sindex, const string& text, ScoreStats& entry) {
virtual void prepareStats(std::size_t sindex, const std::string& text, ScoreStats& entry) {
// do nothing.
}
virtual void prepareStats(const string& sindex, const string& text, ScoreStats& entry) {
this->prepareStats(static_cast<size_t>(atoi(sindex.c_str())), text, entry);
virtual void prepareStats(const std::string& sindex, const std::string& text, ScoreStats& entry) {
this->prepareStats(static_cast<std::size_t>(atoi(sindex.c_str())), text, entry);
}
/**
@ -69,7 +67,7 @@ class Scorer
throw runtime_error("score data not loaded");
}
scores.push_back(0);
for (size_t i = 0; i < diffs.size(); ++i) {
for (std::size_t i = 0; i < diffs.size(); ++i) {
scores.push_back(0);
}
}
@ -86,11 +84,11 @@ class Scorer
return scores[0];
}
const string& getName() const {
const std::string& getName() const {
return m_name;
}
size_t getReferenceSize() const {
std::size_t getReferenceSize() const {
if (m_score_data) {
return m_score_data->size();
}
@ -107,32 +105,32 @@ class Scorer
/**
* Set the factors, which should be used for this metric
*/
virtual void setFactors(const string& factors);
virtual void setFactors(const std::string& factors);
mert::Vocabulary* GetVocab() const { return m_vocab; }
/**
* Set unix filter, which will be used to preprocess the sentences
*/
virtual void setFilter(const string& filterCommand);
virtual void setFilter(const std::string& filterCommand);
private:
void InitConfig(const string& config);
void InitConfig(const std::string& config);
/**
* Take the factored sentence and return the desired factors
*/
string applyFactors(const string& sentece) const;
std::string applyFactors(const std::string& sentece) const;
/**
* Preprocess the sentence with the filter (if given)
*/
string applyFilter(const string& sentence) const;
std::string applyFilter(const std::string& sentence) const;
string m_name;
std::string m_name;
mert::Vocabulary* m_vocab;
map<string, string> m_config;
vector<int> m_factors;
std::map<std::string, std::string> m_config;
std::vector<int> m_factors;
PreProcessFilter* m_filter;
protected:
@ -142,8 +140,8 @@ class Scorer
/**
* Get value of config variable. If not provided, return default.
*/
string getConfig(const string& key, const string& def="") const {
map<string,string>::const_iterator i = m_config.find(key);
std::string getConfig(const std::string& key, const std::string& def="") const {
std::map<std::string,std::string>::const_iterator i = m_config.find(key);
if (i == m_config.end()) {
return def;
} else {
@ -155,12 +153,12 @@ class Scorer
* Tokenise line and encode.
* Note: We assume that all tokens are separated by whitespaces.
*/
void TokenizeAndEncode(const string& line, vector<int>& encoded);
void TokenizeAndEncode(const std::string& line, std::vector<int>& encoded);
/**
* Every inherited scorer should call this function for each sentence
*/
string preprocessSentence(const string& sentence) const
std::string preprocessSentence(const std::string& sentence) const
{
return applyFactors(applyFilter(sentence));
}
@ -174,7 +172,7 @@ class Scorer
class StatisticsBasedScorer : public Scorer
{
public:
StatisticsBasedScorer(const string& name, const string& config);
StatisticsBasedScorer(const std::string& name, const std::string& config);
virtual ~StatisticsBasedScorer() {}
virtual void score(const candidates_t& candidates, const diffs_t& diffs,
statscores_t& scores) const;
@ -190,11 +188,11 @@ class StatisticsBasedScorer : public Scorer
/**
* Calculate the actual score.
*/
virtual statscore_t calculateScore(const vector<int>& totals) const = 0;
virtual statscore_t calculateScore(const std::vector<int>& totals) const = 0;
// regularisation
RegularisationType m_regularization_type;
size_t m_regularization_window;
std::size_t m_regularization_window;
};
#endif // MERT_SCORER_H_

2
mert/SemposScorer.h
View File

@ -54,7 +54,7 @@ private:
std::map<int, float> weightsMap;
void loadWeights(const string& weightsfile);
void loadWeights(const std::string& weightsfile);
// no copying allowed.
SemposScorer(const SemposScorer&);

1
mert/TerScorer.cpp
View File

@ -9,6 +9,7 @@
#include "TER/terAlignment.h"
#include "Util.h"
using namespace std;
using namespace TERCpp;
TerScorer::TerScorer(const string& config)

31
mert/TerScorer.h
View File

@ -1,7 +1,6 @@
#ifndef MERT_TER_SCORER_H_
#define MERT_TER_SCORER_H_
#include <iostream>
#include <set>
#include <string>
#include <vector>
@ -9,8 +8,6 @@
#include "Types.h"
#include "Scorer.h"
using namespace std;
class ScoreStats;
/**
@ -19,35 +16,31 @@ class ScoreStats;
class TerScorer: public StatisticsBasedScorer
{
public:
explicit TerScorer(const string& config = "");
explicit TerScorer(const std::string& config = "");
~TerScorer();
virtual void setReferenceFiles(const vector<string>& referenceFiles);
virtual void prepareStats(size_t sid, const string& text, ScoreStats& entry);
virtual void setReferenceFiles(const std::vector<std::string>& referenceFiles);
virtual void prepareStats(std::size_t sid, const std::string& text, ScoreStats& entry);
virtual size_t NumberOfScores() const {
virtual std::size_t NumberOfScores() const {
// cerr << "TerScorer: " << (LENGTH + 1) << endl;
return kLENGTH + 1;
}
virtual float calculateScore(const vector<int>& comps) const;
void whoami() const {
cerr << "I AM TerScorer" << std::endl;
}
virtual float calculateScore(const std::vector<int>& comps) const;
private:
const int kLENGTH;
string m_java_env;
string m_ter_com_env;
std::string m_java_env;
std::string m_ter_com_env;
// data extracted from reference files
vector<size_t> m_ref_lengths;
vector<multiset<int> > m_ref_tokens;
vector<vector<int> > m_references;
vector<vector<vector<int> > > m_multi_references;
string m_pid;
std::vector<std::size_t> m_ref_lengths;
std::vector<std::multiset<int> > m_ref_tokens;
std::vector<std::vector<int> > m_references;
std::vector<std::vector<std::vector<int> > > m_multi_references;
std::string m_pid;
// no copying allowed
TerScorer(const TerScorer&);

31
mert/Types.h
View File

@ -4,8 +4,7 @@
#include <vector>
#include <map>
#include <string>
using namespace std;
#include <utility>
class FeatureStats;
class FeatureArray;
@ -15,29 +14,29 @@ class ScoreArray;
class ScoreData;
typedef float parameter_t;
//typedef vector<parameter_t> parameters_t;confusing; use vector<parameter_t>
typedef vector<pair<unsigned int, unsigned int> > diff_t;
typedef pair<float,diff_t > threshold;
typedef vector<diff_t> diffs_t;
typedef vector<unsigned int> candidates_t;
//typedef std::vector<parameter_t> parameters_t;confusing; use std::vector<parameter_t>
typedef std::vector<std::pair<unsigned int, unsigned int> > diff_t;
typedef std::pair<float,diff_t > threshold;
typedef std::vector<diff_t> diffs_t;
typedef std::vector<unsigned int> candidates_t;
typedef float statscore_t;
typedef vector<statscore_t> statscores_t;
typedef std::vector<statscore_t> statscores_t;
typedef float FeatureStatsType;
typedef FeatureStatsType* featstats_t;
//typedef vector<FeatureStatsType> featstats_t;
typedef vector<FeatureStats> featarray_t;
typedef vector<FeatureArray> featdata_t;
//typedef std::vector<FeatureStatsType> featstats_t;
typedef std::vector<FeatureStats> featarray_t;
typedef std::vector<FeatureArray> featdata_t;
typedef int ScoreStatsType;
typedef ScoreStatsType* scorestats_t;
//typedef vector<ScoreStatsType> scorestats_t;
typedef vector<ScoreStats> scorearray_t;
typedef vector<ScoreArray> scoredata_t;
//typedef std::vector<ScoreStatsType> scorestats_t;
typedef std::vector<ScoreStats> scorearray_t;
typedef std::vector<ScoreArray> scoredata_t;
typedef map<size_t, std::string> idx2name;
typedef map<std::string, size_t> name2idx;
typedef std::map<std::size_t, std::string> idx2name;
typedef std::map<std::string, std::size_t> name2idx;
#endif // MERT_TYPE_H_

8
mert/Util.h
View File

@ -22,8 +22,6 @@
#include "Types.h"
using namespace std;
#ifdef TRACE_ENABLE
#define TRACE_ERR(str) { std::cerr << str; }
#else
@ -40,11 +38,11 @@ const float kEPS = 0.0001f;
template <typename T>
bool IsAlmostEqual(T expected, T actual, float round=kEPS) {
if (abs(expected - actual) < round) {
if (std::abs(expected - actual) < round) {
return true;
} else {
cerr << "Fail: expected = " << expected
<< " (actual = " << actual << ")" << endl;
std::cerr << "Fail: expected = " << expected
<< " (actual = " << actual << ")" << std::endl;
return false;
}
}

2
mert/Vocabulary.h
View File

@ -48,7 +48,7 @@ class Vocabulary {
bool empty() const { return m_vocab.empty(); }
size_t size() const { return m_vocab.size(); }
std::size_t size() const { return m_vocab.size(); }
iterator find(const std::string& str) { return m_vocab.find(str); }
const_iterator find(const std::string& str) const { return m_vocab.find(str); }