moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-08-16 15:00:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

beautify

Browse Source
This commit is contained in:
Hieu Hoang 2015-01-14 11:07:42 +00:00
parent 91cb549ccf
commit 05ead45e71
406 changed files with 19495 additions and 20485 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
OnDiskPt/TargetPhrase.h
View File

@ -113,13 +113,11 @@ public:
virtual void DebugPrint(std::ostream &out, const Vocab &vocab) const;
void SetProperty(const std::string &value)
{
void SetProperty(const std::string &value) {
m_property = value;
}
void SetSparseFeatures(const std::string &value)
{
void SetSparseFeatures(const std::string &value) {
m_sparseFeatures = value;
}
};

3
OnDiskPt/Word.cpp
View File

@ -107,8 +107,7 @@ void Word::ConvertToMoses(
if (m_isNonTerminal) {
const std::string &tok = vocab.GetString(m_vocabId);
overwrite.SetFactor(0, factorColl.AddFactor(tok, m_isNonTerminal));
}
else {
} else {
// TODO: this conversion should have been done at load time.
util::TokenIter<util::SingleCharacter> tok(vocab.GetString(m_vocabId), '|');

9
biconcor/phrase-lookup.cpp
View File

@ -7,7 +7,8 @@ size_t lookup( string );
vector<string> tokenize( const char input[] );
SuffixArray suffixArray;
int main(int argc, char* argv[]) {
int main(int argc, char* argv[])
{
// handle parameters
string query;
string fileNameSuffix;
@ -95,14 +96,14 @@ int main(int argc, char* argv[]) {
}
cout << lookup( query ) << endl;
}
}
else if (queryFlag) {
} else if (queryFlag) {
cout << lookup( query ) << endl;
}
return 0;
}
size_t lookup( string query ) {
size_t lookup( string query )
{
cerr << "query is " << query << endl;
vector< string > queryString = tokenize( query.c_str() );
return suffixArray.Count( queryString );

3
mert/FeatureStats.cpp
View File

@ -61,7 +61,8 @@ void SparseVector::set(const string& name, FeatureStatsType value)
m_fvector[id] = value;
}
void SparseVector::set(size_t id, FeatureStatsType value) {
void SparseVector::set(size_t id, FeatureStatsType value)
{
assert(m_id_to_name.size() > id);
m_fvector[id] = value;
}

30
mert/ForestRescore.cpp
View File

@ -31,9 +31,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
using namespace std;
namespace MosesTuning {
namespace MosesTuning
{
std::ostream& operator<<(std::ostream& out, const WordVec& wordVec) {
std::ostream& operator<<(std::ostream& out, const WordVec& wordVec)
{
out << "[";
for (size_t i = 0; i < wordVec.size(); ++i) {
out << wordVec[i]->first;
@ -44,7 +46,8 @@ std::ostream& operator<<(std::ostream& out, const WordVec& wordVec) {
}
void ReferenceSet::Load(const vector<string>& files, Vocab& vocab) {
void ReferenceSet::Load(const vector<string>& files, Vocab& vocab)
{
for (size_t i = 0; i < files.size(); ++i) {
util::FilePiece fh(files[i].c_str());
size_t sentenceId = 0;
@ -62,7 +65,8 @@ void ReferenceSet::Load(const vector<string>& files, Vocab& vocab) {
}
void ReferenceSet::AddLine(size_t sentenceId, const StringPiece& line, Vocab& vocab) {
void ReferenceSet::AddLine(size_t sentenceId, const StringPiece& line, Vocab& vocab)
{
//cerr << line << endl;
NgramCounter ngramCounts;
list<WordVec> openNgrams;
@ -105,7 +109,8 @@ void ReferenceSet::AddLine(size_t sentenceId, const StringPiece& line, Vocab& vo
}
size_t ReferenceSet::NgramMatches(size_t sentenceId, const WordVec& ngram, bool clip) const {
size_t ReferenceSet::NgramMatches(size_t sentenceId, const WordVec& ngram, bool clip) const
{
const NgramMap& ngramCounts = ngramCounts_.at(sentenceId);
NgramMap::const_iterator ngi = ngramCounts.find(ngram);
if (ngi == ngramCounts.end()) return 0;
@ -114,7 +119,8 @@ size_t ReferenceSet::NgramMatches(size_t sentenceId, const WordVec& ngram, bool
VertexState::VertexState(): bleuStats(kBleuNgramOrder), targetLength(0) {}
void HgBleuScorer::UpdateMatches(const NgramCounter& counts, vector<FeatureStatsType>& bleuStats ) const {
void HgBleuScorer::UpdateMatches(const NgramCounter& counts, vector<FeatureStatsType>& bleuStats ) const
{
for (NgramCounter::const_iterator ngi = counts.begin(); ngi != counts.end(); ++ngi) {
//cerr << "Checking: " << *ngi << " matches " << references_.NgramMatches(sentenceId_,*ngi,false) << endl;
size_t order = ngi->first.size();
@ -124,7 +130,8 @@ void HgBleuScorer::UpdateMatches(const NgramCounter& counts, vector<FeatureStats
}
}
size_t HgBleuScorer::GetTargetLength(const Edge& edge) const {
size_t HgBleuScorer::GetTargetLength(const Edge& edge) const
{
size_t targetLength = 0;
for (size_t i = 0; i < edge.Words().size(); ++i) {
const Vocab::Entry* word = edge.Words()[i];
@ -137,7 +144,8 @@ size_t HgBleuScorer::GetTargetLength(const Edge& edge) const {
return targetLength;
}
FeatureStatsType HgBleuScorer::Score(const Edge& edge, const Vertex& head, vector<FeatureStatsType>& bleuStats) {
FeatureStatsType HgBleuScorer::Score(const Edge& edge, const Vertex& head, vector<FeatureStatsType>& bleuStats)
{
NgramCounter ngramCounts;
size_t childId = 0;
size_t wordId = 0;
@ -242,7 +250,8 @@ FeatureStatsType HgBleuScorer::Score(const Edge& edge, const Vertex& head, vecto
return bleu;
}
void HgBleuScorer::UpdateState(const Edge& winnerEdge, size_t vertexId, const vector<FeatureStatsType>& bleuStats) {
void HgBleuScorer::UpdateState(const Edge& winnerEdge, size_t vertexId, const vector<FeatureStatsType>& bleuStats)
{
//TODO: Maybe more efficient to absorb into the Score() method
VertexState& vertexState = vertexStates_[vertexId];
//cerr << "Updating state for " << vertexId << endl;
@ -314,7 +323,8 @@ typedef pair<const Edge*,FeatureStatsType> BackPointer;
* Recurse through back pointers
**/
static void GetBestHypothesis(size_t vertexId, const Graph& graph, const vector<BackPointer>& bps,
HgHypothesis* bestHypo) {
HgHypothesis* bestHypo)
{
//cerr << "Expanding " << vertexId << " Score: " << bps[vertexId].second << endl;
//UTIL_THROW_IF(bps[vertexId].second == kMinScore+1, HypergraphException, "Landed at vertex " << vertexId << " which is a dead end");
if (!bps[vertexId].first) return;

17
mert/ForestRescore.h
View File

@ -27,7 +27,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "BleuScorer.h"
#include "Hypergraph.h"
namespace MosesTuning {
namespace MosesTuning
{
std::ostream& operator<<(std::ostream& out, const WordVec& wordVec);
@ -47,7 +48,8 @@ struct NgramEquals : public std::binary_function<const WordVec&, const WordVec&,
typedef boost::unordered_map<WordVec, size_t, NgramHash, NgramEquals> NgramCounter;
class ReferenceSet {
class ReferenceSet
{
public:
@ -58,7 +60,9 @@ public:
size_t NgramMatches(size_t sentenceId, const WordVec&, bool clip) const;
size_t Length(size_t sentenceId) const {return lengths_[sentenceId];}
size_t Length(size_t sentenceId) const {
return lengths_[sentenceId];
}
private:
//ngrams to (clipped,unclipped) counts
@ -80,8 +84,9 @@ struct VertexState {
/**
* Used to score an rule (ie edge) when we are applying it.
**/
class HgBleuScorer {
public:
class HgBleuScorer
{
public:
HgBleuScorer(const ReferenceSet& references, const Graph& graph, size_t sentenceId, const std::vector<FeatureStatsType>& backgroundBleu):
references_(references), sentenceId_(sentenceId), graph_(graph), backgroundBleu_(backgroundBleu),
backgroundRefLength_(backgroundBleu[kBleuNgramOrder*2]) {
@ -94,7 +99,7 @@ class HgBleuScorer {
void UpdateState(const Edge& winnerEdge, size_t vertexId, const std::vector<FeatureStatsType>& bleuStats);
private:
private:
const ReferenceSet& references_;
std::vector<VertexState> vertexStates_;
size_t sentenceId_;

46
mert/HopeFearDecoder.cpp
View File

@ -34,11 +34,13 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
using namespace std;
namespace fs = boost::filesystem;
namespace MosesTuning {
namespace MosesTuning
{
static const ValType BLEU_RATIO = 5;
ValType HopeFearDecoder::Evaluate(const AvgWeightVector& wv) {
ValType HopeFearDecoder::Evaluate(const AvgWeightVector& wv)
{
vector<ValType> stats(scorer_->NumberOfScores(),0);
for(reset(); !finished(); next()) {
vector<ValType> sent;
@ -57,7 +59,8 @@ NbestHopeFearDecoder::NbestHopeFearDecoder(
bool no_shuffle,
bool safe_hope,
Scorer* scorer
) : safe_hope_(safe_hope) {
) : safe_hope_(safe_hope)
{
scorer_ = scorer;
if (streaming) {
train_.reset(new StreamingHypPackEnumerator(featureFiles, scoreFiles));
@ -67,15 +70,18 @@ NbestHopeFearDecoder::NbestHopeFearDecoder(
}
void NbestHopeFearDecoder::next() {
void NbestHopeFearDecoder::next()
{
train_->next();
}
bool NbestHopeFearDecoder::finished() {
bool NbestHopeFearDecoder::finished()
{
return train_->finished();
}
void NbestHopeFearDecoder::reset() {
void NbestHopeFearDecoder::reset()
{
train_->reset();
}
@ -83,7 +89,8 @@ void NbestHopeFearDecoder::HopeFear(
const std::vector<ValType>& backgroundBleu,
const MiraWeightVector& wv,
HopeFearData* hopeFear
) {
)
{
// Hope / fear decode
@ -134,7 +141,8 @@ void NbestHopeFearDecoder::HopeFear(
hopeFear->hopeFearEqual = (hope_index == fear_index);
}
void NbestHopeFearDecoder::MaxModel(const AvgWeightVector& wv, std::vector<ValType>* stats) {
void NbestHopeFearDecoder::MaxModel(const AvgWeightVector& wv, std::vector<ValType>* stats)
{
// Find max model
size_t max_index=0;
ValType max_score=0;
@ -152,7 +160,7 @@ void NbestHopeFearDecoder::MaxModel(const AvgWeightVector& wv, std::vector<ValTy
HypergraphHopeFearDecoder::HypergraphHopeFearDecoder
(
(
const string& hypergraphDir,
const vector<string>& referenceFiles,
size_t num_dense,
@ -162,8 +170,9 @@ HypergraphHopeFearDecoder::HypergraphHopeFearDecoder
size_t hg_pruning,
const MiraWeightVector& wv,
Scorer* scorer
) :
num_dense_(num_dense) {
) :
num_dense_(num_dense)
{
UTIL_THROW_IF(streaming, util::Exception, "Streaming not currently supported for hypergraphs");
UTIL_THROW_IF(!fs::exists(hypergraphDir), HypergraphException, "Directory '" << hypergraphDir << "' does not exist");
@ -211,15 +220,18 @@ HypergraphHopeFearDecoder::HypergraphHopeFearDecoder
}
void HypergraphHopeFearDecoder::reset() {
void HypergraphHopeFearDecoder::reset()
{
sentenceIdIter_ = sentenceIds_.begin();
}
void HypergraphHopeFearDecoder::next() {
void HypergraphHopeFearDecoder::next()
{
sentenceIdIter_++;
}
bool HypergraphHopeFearDecoder::finished() {
bool HypergraphHopeFearDecoder::finished()
{
return sentenceIdIter_ == sentenceIds_.end();
}
@ -227,7 +239,8 @@ void HypergraphHopeFearDecoder::HopeFear(
const vector<ValType>& backgroundBleu,
const MiraWeightVector& wv,
HopeFearData* hopeFear
) {
)
{
size_t sentenceId = *sentenceIdIter_;
SparseVector weights;
wv.ToSparse(&weights);
@ -319,7 +332,8 @@ void HypergraphHopeFearDecoder::HopeFear(
hopeFear->hopeFearEqual = hopeFear->hopeFearEqual && (hopeFear->fearFeatures == hopeFear->hopeFeatures);
}
void HypergraphHopeFearDecoder::MaxModel(const AvgWeightVector& wv, vector<ValType>* stats) {
void HypergraphHopeFearDecoder::MaxModel(const AvgWeightVector& wv, vector<ValType>* stats)
{
assert(!finished());
HgHypothesis bestHypo;
size_t sentenceId = *sentenceIdIter_;

12
mert/HopeFearDecoder.h
View File

@ -35,7 +35,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
// the n-best list and lattice/hypergraph implementations
//
namespace MosesTuning {
namespace MosesTuning
{
class Scorer;
@ -55,7 +56,8 @@ struct HopeFearData {
};
//Abstract base class
class HopeFearDecoder {
class HopeFearDecoder
{
public:
//iterator methods
virtual void reset() = 0;
@ -86,7 +88,8 @@ protected:
/** Gets hope-fear from nbest lists */
class NbestHopeFearDecoder : public virtual HopeFearDecoder {
class NbestHopeFearDecoder : public virtual HopeFearDecoder
{
public:
NbestHopeFearDecoder(const std::vector<std::string>& featureFiles,
const std::vector<std::string>& scoreFiles,
@ -117,7 +120,8 @@ private:
/** Gets hope-fear from hypergraphs */
class HypergraphHopeFearDecoder : public virtual HopeFearDecoder {
class HypergraphHopeFearDecoder : public virtual HopeFearDecoder
{
public:
HypergraphHopeFearDecoder(
const std::string& hypergraphDir,

12
mert/HwcmScorer.cpp
View File

@ -55,7 +55,8 @@ void HwcmScorer::setReferenceFiles(const vector<string>& referenceFiles)
}
void HwcmScorer::extractHeadWordChain(TreePointer tree, vector<string> & history, vector<map<string, int> > & hwc) {
void HwcmScorer::extractHeadWordChain(TreePointer tree, vector<string> & history, vector<map<string, int> > & hwc)
{
if (tree->GetLength() > 0) {
string head = getHead(tree);
@ -64,8 +65,7 @@ void HwcmScorer::extractHeadWordChain(TreePointer tree, vector<string> & history
for (std::vector<TreePointer>::const_iterator it = tree->GetChildren().begin(); it != tree->GetChildren().end(); ++it) {
extractHeadWordChain(*it, history, hwc);
}
}
else {
} else {
vector<string> new_history(kHwcmOrder);
new_history[0] = head;
hwc[0][head]++;
@ -85,11 +85,11 @@ void HwcmScorer::extractHeadWordChain(TreePointer tree, vector<string> & history
}
}
string HwcmScorer::getHead(TreePointer tree) {
string HwcmScorer::getHead(TreePointer tree)
{
// assumption (only true for dependency parse: each constituent has a preterminal label, and corresponding terminal is head)
// if constituent has multiple preterminals, first one is picked; if it has no preterminals, empty string is returned
for (std::vector<TreePointer>::const_iterator it = tree->GetChildren().begin(); it != tree->GetChildren().end(); ++it)
{
for (std::vector<TreePointer>::const_iterator it = tree->GetChildren().begin(); it != tree->GetChildren().end(); ++it) {
TreePointer child = *it;
if (child->GetLength() == 1 && child->GetChildren()[0]->IsTerminal()) {

29
mert/Hypergraph.cpp
View File

@ -31,18 +31,22 @@ using namespace std;
static const string kBOS = "<s>";
static const string kEOS = "</s>";
namespace MosesTuning {
namespace MosesTuning
{
StringPiece NextLine(util::FilePiece& from) {
StringPiece NextLine(util::FilePiece& from)
{
StringPiece line;
while ((line = from.ReadLine()).starts_with("#"));
return line;
}
Vocab::Vocab() : eos_( FindOrAdd(kEOS)), bos_(FindOrAdd(kBOS)){
Vocab::Vocab() : eos_( FindOrAdd(kEOS)), bos_(FindOrAdd(kBOS))
{
}
const Vocab::Entry &Vocab::FindOrAdd(const StringPiece &str) {
const Vocab::Entry &Vocab::FindOrAdd(const StringPiece &str)
{
#if BOOST_VERSION >= 104200
Map::const_iterator i= map_.find(str, Hash(), Equals());
#else
@ -62,7 +66,8 @@ double_conversion::StringToDoubleConverter converter(double_conversion::StringTo
/**
* Reads an incoming edge. Returns edge and source words covered.
**/
static pair<Edge*,size_t> ReadEdge(util::FilePiece &from, Graph &graph) {
static pair<Edge*,size_t> ReadEdge(util::FilePiece &from, Graph &graph)
{
Edge* edge = graph.NewEdge();
StringPiece line = from.ReadLine(); //Don't allow comments within edge lists
util::TokenIter<util::MultiCharacter> pipes(line, util::MultiCharacter(" ||| "));
@ -103,7 +108,8 @@ static pair<Edge*,size_t> ReadEdge(util::FilePiece &from, Graph &graph) {
return pair<Edge*,size_t>(edge,sourceCovered);
}
void Graph::Prune(Graph* pNewGraph, const SparseVector& weights, size_t minEdgeCount) const {
void Graph::Prune(Graph* pNewGraph, const SparseVector& weights, size_t minEdgeCount) const
{
Graph& newGraph = *pNewGraph;
//TODO: Optimise case where no pruning required
@ -235,7 +241,7 @@ void Graph::Prune(Graph* pNewGraph, const SparseVector& weights, size_t minEdgeC
map<size_t,size_t> oldIdToNew;
size_t vi = 0;
for (set<size_t>::const_iterator i = retainedVertices.begin(); i != retainedVertices.end(); ++i, ++vi) {
// cerr << *i << " New: " << vi << endl;
// cerr << *i << " New: " << vi << endl;
oldIdToNew[*i] = vi;
Vertex* vertex = newGraph.NewVertex();
vertex->SetSourceCovered(vertices_[*i].SourceCovered());
@ -276,14 +282,15 @@ void Graph::Prune(Graph* pNewGraph, const SparseVector& weights, size_t minEdgeC
cerr << endl;
}
*/
*/
}
/**
* Read from "Kenneth's hypergraph" aka cdec target_graph format (with comments)
**/
void ReadGraph(util::FilePiece &from, Graph &graph) {
void ReadGraph(util::FilePiece &from, Graph &graph)
{
//First line should contain field names
StringPiece line = from.ReadLine();
@ -306,7 +313,9 @@ void ReadGraph(util::FilePiece &from, Graph &graph) {
vertex->AddEdge(edge.first);
//Note: the file format attaches this to the edge, but it's really a property
//of the vertex.
if (!e) {vertex->SetSourceCovered(edge.second);}
if (!e) {
vertex->SetSourceCovered(edge.second);
}
}
}
}

83
mert/Hypergraph.h
View File

@ -37,14 +37,16 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "FeatureStats.h"
namespace MosesTuning {
namespace MosesTuning
{
typedef unsigned int WordIndex;
const WordIndex kMaxWordIndex = UINT_MAX;
const FeatureStatsType kMinScore = -1e10;
template <class T> class FixedAllocator : boost::noncopyable {
public:
template <class T> class FixedAllocator : boost::noncopyable
{
public:
FixedAllocator() : current_(NULL), end_(NULL) {}
void Init(std::size_t count) {
@ -75,25 +77,30 @@ template <class T> class FixedAllocator : boost::noncopyable {
return current_ - array_.get();
}
private:
private:
boost::scoped_array<T> array_;
T *current_, *end_;
};
class Vocab {
public:
class Vocab
{
public:
Vocab();
typedef std::pair<const char *const, WordIndex> Entry;
const Entry &FindOrAdd(const StringPiece &str);
const Entry& Bos() const {return bos_;}
const Entry& Bos() const {
return bos_;
}
const Entry& Eos() const {return eos_;}
const Entry& Eos() const {
return eos_;
}
private:
private:
util::Pool piece_backing_;
struct Hash : public std::unary_function<const char *, std::size_t> {
@ -125,9 +132,12 @@ typedef boost::shared_ptr<SparseVector> FeaturePtr;
/**
* An edge has 1 head vertex, 0..n child (tail) vertices, a list of words and a feature vector.
**/
class Edge {
public:
Edge() {features_.reset(new SparseVector());}
class Edge
{
public:
Edge() {
features_.reset(new SparseVector());
}
void AddWord(const Vocab::Entry *word) {
words_.push_back(word);
@ -163,7 +173,7 @@ class Edge {
return inner_product(*(features_.get()), weights);
}
private:
private:
// NULL for non-terminals.
std::vector<const Vocab::Entry*> words_;
std::vector<size_t> children_;
@ -173,26 +183,36 @@ class Edge {
/*
* A vertex has 0..n incoming edges
**/
class Vertex {
public:
class Vertex
{
public:
Vertex() : sourceCovered_(0) {}
void AddEdge(const Edge* edge) {incoming_.push_back(edge);}
void AddEdge(const Edge* edge) {
incoming_.push_back(edge);
}
void SetSourceCovered(size_t sourceCovered) {sourceCovered_ = sourceCovered;}
void SetSourceCovered(size_t sourceCovered) {
sourceCovered_ = sourceCovered;
}
const std::vector<const Edge*>& GetIncoming() const {return incoming_;}
const std::vector<const Edge*>& GetIncoming() const {
return incoming_;
}
size_t SourceCovered() const {return sourceCovered_;}
size_t SourceCovered() const {
return sourceCovered_;
}
private:
private:
std::vector<const Edge*> incoming_;
size_t sourceCovered_;
};
class Graph : boost::noncopyable {
public:
class Graph : boost::noncopyable
{
public:
Graph(Vocab& vocab) : vocab_(vocab) {}
void SetCounts(std::size_t vertices, std::size_t edges) {
@ -200,7 +220,9 @@ class Graph : boost::noncopyable {
edges_.Init(edges);
}
Vocab &MutableVocab() { return vocab_; }
Vocab &MutableVocab() {
return vocab_;
}
Edge *NewEdge() {
return edges_.New();
@ -223,21 +245,26 @@ class Graph : boost::noncopyable {
Colin Cherry */
void Prune(Graph* newGraph, const SparseVector& weights, size_t minEdgeCount) const;
std::size_t VertexSize() const { return vertices_.Size(); }
std::size_t EdgeSize() const { return edges_.Size(); }
std::size_t VertexSize() const {
return vertices_.Size();
}
std::size_t EdgeSize() const {
return edges_.Size();
}
bool IsBoundary(const Vocab::Entry* word) const {
return word->second == vocab_.Bos().second || word->second == vocab_.Eos().second;
}
private:
private:
FixedAllocator<Edge> edges_;
FixedAllocator<Vertex> vertices_;
Vocab& vocab_;
};
class HypergraphException : public util::Exception {
public:
class HypergraphException : public util::Exception
{
public:
HypergraphException() {}
~HypergraphException() throw() {}
};

2
mert/InterpolatedScorer.cpp
View File

@ -186,7 +186,7 @@ float InterpolatedScorer::getReferenceLength(const std::vector<ScoreStatsType>&
scorerNum++;
}
return refLen;
}
}
void InterpolatedScorer::setReferenceFiles(const vector<string>& referenceFiles)
{

9
mert/MiraFeatureVector.cpp
View File

@ -9,7 +9,8 @@ namespace MosesTuning
{
void MiraFeatureVector::InitSparse(const SparseVector& sparse, size_t ignoreLimit) {
void MiraFeatureVector::InitSparse(const SparseVector& sparse, size_t ignoreLimit)
{
vector<size_t> sparseFeats = sparse.feats();
bool bFirst = true;
size_t lastFeat = 0;
@ -40,7 +41,8 @@ MiraFeatureVector::MiraFeatureVector(const FeatureDataItem& vec)
InitSparse(vec.sparse);
}
MiraFeatureVector::MiraFeatureVector(const SparseVector& sparse, size_t num_dense) {
MiraFeatureVector::MiraFeatureVector(const SparseVector& sparse, size_t num_dense)
{
m_dense.resize(num_dense);
//Assume that features with id [0,num_dense) are the dense features
for (size_t id = 0; id < num_dense; ++id) {
@ -162,7 +164,8 @@ MiraFeatureVector operator-(const MiraFeatureVector& a, const MiraFeatureVector&
return MiraFeatureVector(dense,sparseFeats,sparseVals);
}
bool operator==(const MiraFeatureVector& a,const MiraFeatureVector& b) {
bool operator==(const MiraFeatureVector& a,const MiraFeatureVector& b)
{
ValType eps = 1e-8;
//dense features
if (a.m_dense.size() != b.m_dense.size()) return false;

6
mert/MiraWeightVector.cpp
View File

@ -93,7 +93,8 @@ void MiraWeightVector::update(size_t index, ValType delta)
m_lastUpdated[index] = m_numUpdates;
}
void MiraWeightVector::ToSparse(SparseVector* sparse) const {
void MiraWeightVector::ToSparse(SparseVector* sparse) const
{
for (size_t i = 0; i < m_weights.size(); ++i) {
if(abs(m_weights[i])>1e-8) {
sparse->set(i,m_weights[i]);
@ -171,7 +172,8 @@ size_t AvgWeightVector::size() const
return m_wv.m_weights.size();
}
void AvgWeightVector::ToSparse(SparseVector* sparse) const {
void AvgWeightVector::ToSparse(SparseVector* sparse) const
{
for (size_t i = 0; i < size(); ++i) {
ValType w = weight(i);
if(abs(w)>1e-8) {

2
mert/StatisticsBasedScorer.h
View File

@ -23,7 +23,7 @@ namespace MosesTuning
*/
class StatisticsBasedScorer : public Scorer
{
friend class HopeFearDecoder;
friend class HopeFearDecoder;
public:
StatisticsBasedScorer(const std::string& name, const std::string& config);

8
mert/TER/alignmentStruct.cpp
View File

@ -23,15 +23,15 @@ Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
using namespace std;
namespace TERCpp
{
string alignmentStruct::toString()
{
string alignmentStruct::toString()
{
stringstream s;
// s << "nword : " << vectorToString(nwords)<<endl;
// s << "alignment" << vectorToString(alignment)<<endl;
// s << "afterShift" << vectorToString(alignment)<<endl;
s << "Nothing to be printed" <<endl;
return s.str();
}
}
// alignmentStruct::alignmentStruct()
// {
@ -99,7 +99,7 @@ namespace TERCpp
// return s.str();
// }
/* The distance of the shift. */
/* The distance of the shift. */
// int alignmentStruct::distance()
// {
// if (moveto < start)

10
mert/TER/alignmentStruct.h
View File

@ -34,10 +34,10 @@ using namespace Tools;
namespace TERCpp
{
class alignmentStruct
{
private:
public:
class alignmentStruct
{
private:
public:
// alignmentStruct();
// alignmentStruct (int _start, int _end, int _moveto, int _newloc);
@ -60,7 +60,7 @@ namespace TERCpp
// calculate it multiple times.
double cost;
string toString();
};
};
}
#endif

10
mert/TER/bestShiftStruct.h
View File

@ -36,10 +36,10 @@ using namespace Tools;
namespace TERCpp
{
class bestShiftStruct
{
private:
public:
class bestShiftStruct
{
private:
public:
// alignmentStruct();
// alignmentStruct (int _start, int _end, int _moveto, int _newloc);
@ -64,7 +64,7 @@ namespace TERCpp
// This is used to store the cost of a shift, so we don't have to
// calculate it multiple times.
// double cost;
};
};
}
#endif

108
mert/TER/hashMap.cpp
View File

@ -28,70 +28,65 @@ using namespace std;
namespace HashMapSpace
{
// hashMap::hashMap();
/* hashMap::~hashMap()
/* hashMap::~hashMap()
{
// vector<stringHasher>::const_iterator del = m_hasher.begin();
// vector<stringHasher>::const_iterator del = m_hasher.begin();
for ( vector<stringHasher>::const_iterator del=m_hasher.begin(); del != m_hasher.end(); del++ )
{
delete(*del);
}
}*/
/**
/**
* int hashMap::trouve ( long searchKey )
* @param searchKey
* @return
*/
int hashMap::trouve ( long searchKey )
{
int hashMap::trouve ( long searchKey )
{
long foundKey;
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
{
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
foundKey= ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
return 1;
}
}
return 0;
}
int hashMap::trouve ( string key )
{
}
int hashMap::trouve ( string key )
{
long searchKey=hashValue ( key );
long foundKey;;
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
{
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
foundKey= ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
return 1;
}
}
return 0;
}
/**
}
/**
* long hashMap::hashValue ( string key )
* @param key
* @return
*/
long hashMap::hashValue ( string key )
{
long hashMap::hashValue ( string key )
{
locale loc; // the "C" locale
const collate<char>& coll = use_facet<collate<char> >(loc);
return coll.hash(key.data(),key.data()+key.length());
// boost::hash<string> hasher;
// return hasher ( key );
}
/**
}
/**
* void hashMap::addHasher ( string key, string value )
* @param key
* @param value
*/
void hashMap::addHasher ( string key, string value )
{
if ( trouve ( hashValue ( key ) ) ==0 )
{
void hashMap::addHasher ( string key, string value )
{
if ( trouve ( hashValue ( key ) ) ==0 ) {
// cerr << "ICI1" <<endl;
stringHasher H ( hashValue ( key ),key,value );
// cerr <<" "<< hashValue ( key )<<" "<< key<<" "<<value <<endl;
@ -99,85 +94,76 @@ namespace HashMapSpace
m_hasher.push_back ( H );
}
}
stringHasher hashMap::getHasher ( string key )
{
}
stringHasher hashMap::getHasher ( string key )
{
long searchKey=hashValue ( key );
long foundKey;
stringHasher defaut(0,"","");
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
{
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
foundKey= ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
return ( *l_hasher );
}
}
return defaut;
}
string hashMap::getValue ( string key )
{
}
string hashMap::getValue ( string key )
{
long searchKey=hashValue ( key );
long foundKey;
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
{
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
foundKey= ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
// cerr <<"value found : " << key<<"|"<< ( *l_hasher ).getValue()<<endl;
return ( *l_hasher ).getValue();
}
}
return "";
}
string hashMap::searchValue ( string value )
{
}
string hashMap::searchValue ( string value )
{
// long searchKey=hashValue ( key );
// long foundKey;
string foundValue;
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
{
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
foundValue= ( *l_hasher ).getValue();
if ( foundValue.compare ( value ) == 0 )
{
if ( foundValue.compare ( value ) == 0 ) {
return ( *l_hasher ).getKey();
}
}
return "";
}
}
void hashMap::setValue ( string key , string value )
{
void hashMap::setValue ( string key , string value )
{
long searchKey=hashValue ( key );
long foundKey;
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
{
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
foundKey= ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
( *l_hasher ).setValue ( value );
// return ( *l_hasher ).getValue();
}
}
}
}
/**
/**
*
*/
void hashMap::printHash()
{
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
{
void hashMap::printHash()
{
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
}
}
}

10
mert/TER/hashMap.h
View File

@ -35,12 +35,12 @@ using namespace std;
namespace HashMapSpace
{
class hashMap
{
private:
class hashMap
{
private:
vector<stringHasher> m_hasher;
public:
public:
// ~hashMap();
long hashValue ( string key );
int trouve ( long searchKey );
@ -55,7 +55,7 @@ namespace HashMapSpace
string printStringHash();
string printStringHash2();
string printStringHashForLexicon();
};
};
}

111
mert/TER/hashMapInfos.cpp
View File

@ -28,71 +28,66 @@ using namespace std;
namespace HashMapSpace
{
// hashMapInfos::hashMap();
/* hashMapInfos::~hashMap()
/* hashMapInfos::~hashMap()
{
// vector<infosHasher>::const_iterator del = m_hasher.begin();
// vector<infosHasher>::const_iterator del = m_hasher.begin();
for ( vector<infosHasher>::const_iterator del=m_hasher.begin(); del != m_hasher.end(); del++ )
{
delete(*del);
}
}*/
/**
/**
* int hashMapInfos::trouve ( long searchKey )
* @param searchKey
* @return
*/
int hashMapInfos::trouve ( long searchKey )
{
int hashMapInfos::trouve ( long searchKey )
{
long foundKey;
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
{
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
foundKey= ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
return 1;
}
}
return 0;
}
int hashMapInfos::trouve ( string key )
{
}
int hashMapInfos::trouve ( string key )
{
long searchKey=hashValue ( key );
long foundKey;;
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
{
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
foundKey= ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
return 1;
}
}
return 0;
}
}
/**
/**
* long hashMapInfos::hashValue ( string key )
* @param key
* @return
*/
long hashMapInfos::hashValue ( string key )
{
long hashMapInfos::hashValue ( string key )
{
locale loc; // the "C" locale
const collate<char>& coll = use_facet<collate<char> >(loc);
return coll.hash(key.data(),key.data()+key.length());
// boost::hash<string> hasher;
// return hasher ( key );
}
/**
}
/**
* void hashMapInfos::addHasher ( string key, string value )
* @param key
* @param value
*/
void hashMapInfos::addHasher ( string key, vector<int> value )
{
if ( trouve ( hashValue ( key ) ) ==0 )
{
void hashMapInfos::addHasher ( string key, vector<int> value )
{
if ( trouve ( hashValue ( key ) ) ==0 ) {
// cerr << "ICI1" <<endl;
infosHasher H ( hashValue ( key ),key,value );
// cerr <<" "<< hashValue ( key )<<" "<< key<<" "<<value <<endl;
@ -100,45 +95,41 @@ namespace HashMapSpace
m_hasher.push_back ( H );
}
}
void hashMapInfos::addValue ( string key, vector<int> value )
{
}
void hashMapInfos::addValue ( string key, vector<int> value )
{
addHasher ( key, value );
}
infosHasher hashMapInfos::getHasher ( string key )
{
}
infosHasher hashMapInfos::getHasher ( string key )
{
long searchKey=hashValue ( key );
long foundKey;
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
{
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
foundKey= ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
return ( *l_hasher );
}
}
vector<int> temp;
infosHasher defaut(0,"",temp);
return defaut;
}
vector<int> hashMapInfos::getValue ( string key )
{
}
vector<int> hashMapInfos::getValue ( string key )
{
long searchKey=hashValue ( key );
long foundKey;
vector<int> retour;
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
{
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
foundKey= ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
// cerr <<"value found : " << key<<"|"<< ( *l_hasher ).getValue()<<endl;
return ( *l_hasher ).getValue();
}
}
return retour;
}
}
// string hashMapInfos::searchValue ( string value )
// {
// // long searchKey=hashValue ( key );
@ -158,42 +149,38 @@ namespace HashMapSpace
// }
//
void hashMapInfos::setValue ( string key , vector<int> value )
{
void hashMapInfos::setValue ( string key , vector<int> value )
{
long searchKey=hashValue ( key );
long foundKey;
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
{
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
foundKey= ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
( *l_hasher ).setValue ( value );
// return ( *l_hasher ).getValue();
}
}
}
string hashMapInfos::toString ()
{
}
string hashMapInfos::toString ()
{
stringstream to_return;
for ( vector<infosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
{
for ( vector<infosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
to_return << (*l_hasher).toString();
// cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
}
return to_return.str();
}
}
/**
/**
*
*/
void hashMapInfos::printHash()
{
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
{
void hashMapInfos::printHash()
{
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
// cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
}
}
}

10
mert/TER/hashMapInfos.h
View File

@ -34,12 +34,12 @@ using namespace std;
namespace HashMapSpace
{
class hashMapInfos
{
private:
class hashMapInfos
{
private:
vector<infosHasher> m_hasher;
public:
public:
// ~hashMap();
long hashValue ( string key );
int trouve ( long searchKey );
@ -56,7 +56,7 @@ namespace HashMapSpace
string printStringHash();
string printStringHash2();
string printStringHashForLexicon();
};
};
}

187
mert/TER/hashMapStringInfos.cpp
View File

@ -27,73 +27,68 @@ using namespace std;
namespace HashMapSpace
{
// hashMapStringInfos::hashMap();
/* hashMapStringInfos::~hashMap()
{
// vector<stringInfosHasher>::const_iterator del = m_hasher.begin();
// hashMapStringInfos::hashMap();
/* hashMapStringInfos::~hashMap()
{
// vector<stringInfosHasher>::const_iterator del = m_hasher.begin();
for ( vector<stringInfosHasher>::const_iterator del=m_hasher.begin(); del != m_hasher.end(); del++ )
{
delete(*del);
}
}*/
/**
* int hashMapStringInfos::trouve ( long searchKey )
* @param searchKey
* @return
*/
int hashMapStringInfos::trouve ( long searchKey )
{
}*/
/**
* int hashMapStringInfos::trouve ( long searchKey )
* @param searchKey
* @return
*/
int hashMapStringInfos::trouve ( long searchKey )
{
long foundKey;
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
{
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
foundKey = ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
return 1;
}
}
return 0;
}
}
int hashMapStringInfos::trouve ( string key )
{
int hashMapStringInfos::trouve ( string key )
{
long searchKey = hashValue ( key );
long foundKey;;
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
{
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
foundKey = ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
return 1;
}
}
return 0;
}
}
/**
* long hashMapStringInfos::hashValue ( string key )
* @param key
* @return
*/
long hashMapStringInfos::hashValue ( string key )
{
/**
* long hashMapStringInfos::hashValue ( string key )
* @param key
* @return
*/
long hashMapStringInfos::hashValue ( string key )
{
locale loc; // the "C" locale
const collate<char>& coll = use_facet<collate<char> > ( loc );
return coll.hash ( key.data(), key.data() + key.length() );
// boost::hash<string> hasher;
// return hasher ( key );
}
/**
* void hashMapStringInfos::addHasher ( string key, string value )
* @param key
* @param value
*/
void hashMapStringInfos::addHasher ( string key, vector<string> value )
{
if ( trouve ( hashValue ( key ) ) == 0 )
{
}
/**
* void hashMapStringInfos::addHasher ( string key, string value )
* @param key
* @param value
*/
void hashMapStringInfos::addHasher ( string key, vector<string> value )
{
if ( trouve ( hashValue ( key ) ) == 0 ) {
// cerr << "ICI1" <<endl;
stringInfosHasher H ( hashValue ( key ), key, value );
// cerr <<" "<< hashValue ( key )<<" "<< key<<" "<<value <<endl;
@ -101,105 +96,97 @@ namespace HashMapSpace
m_hasher.push_back ( H );
}
}
void hashMapStringInfos::addValue ( string key, vector<string> value )
{
}
void hashMapStringInfos::addValue ( string key, vector<string> value )
{
addHasher ( key, value );
}
stringInfosHasher hashMapStringInfos::getHasher ( string key )
{
}
stringInfosHasher hashMapStringInfos::getHasher ( string key )
{
long searchKey = hashValue ( key );
long foundKey;
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
{
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
foundKey = ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
return ( *l_hasher );
}
}
vector<string> tmp;
stringInfosHasher defaut ( 0, "", tmp );
return defaut;
}
vector<string> hashMapStringInfos::getValue ( string key )
{
}
vector<string> hashMapStringInfos::getValue ( string key )
{
long searchKey = hashValue ( key );
long foundKey;
vector<string> retour;
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
{
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
foundKey = ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
// cerr <<"value found : " << key<<"|"<< ( *l_hasher ).getValue()<<endl;
return ( *l_hasher ).getValue();
}
}
return retour;
}
// string hashMapStringInfos::searchValue ( string value )
// {
// // long searchKey=hashValue ( key );
// // long foundKey;
// vector<int> foundValue;
//
// // vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
// for ( vector<stringInfosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
// {
// foundValue= ( *l_hasher ).getValue();
// /* if ( foundValue.compare ( value ) == 0 )
// {
// return ( *l_hasher ).getKey();
// }*/
// }
// return "";
// }
//
}
// string hashMapStringInfos::searchValue ( string value )
// {
// // long searchKey=hashValue ( key );
// // long foundKey;
// vector<int> foundValue;
//
// // vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
// for ( vector<stringInfosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
// {
// foundValue= ( *l_hasher ).getValue();
// /* if ( foundValue.compare ( value ) == 0 )
// {
// return ( *l_hasher ).getKey();
// }*/
// }
// return "";
// }
//
void hashMapStringInfos::setValue ( string key , vector<string> value )
{
void hashMapStringInfos::setValue ( string key , vector<string> value )
{
long searchKey = hashValue ( key );
long foundKey;
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
{
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
foundKey = ( *l_hasher ).getHashKey();
if ( searchKey == foundKey )
{
if ( searchKey == foundKey ) {
( *l_hasher ).setValue ( value );
// return ( *l_hasher ).getValue();
}
}
}
}
string hashMapStringInfos::toString ()
{
string hashMapStringInfos::toString ()
{
stringstream to_return;
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
{
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
to_return << (*l_hasher).toString();
// cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
}
return to_return.str();
}
}
/**
*
*/
void hashMapStringInfos::printHash()
{
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
{
/**
*
*/
void hashMapStringInfos::printHash()
{
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
// cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
}
}
vector< stringInfosHasher > hashMapStringInfos::getHashMap()
{
}
vector< stringInfosHasher > hashMapStringInfos::getHashMap()
{
return m_hasher;
}
}

10
mert/TER/hashMapStringInfos.h
View File

@ -34,12 +34,12 @@ using namespace std;
namespace HashMapSpace
{
class hashMapStringInfos
{
private:
class hashMapStringInfos
{
private:
vector<stringInfosHasher> m_hasher;
public:
public:
// ~hashMap();
long hashValue ( string key );
int trouve ( long searchKey );
@ -56,7 +56,7 @@ namespace HashMapSpace
string printStringHash();
string printStringHash2();
string printStringHashForLexicon();
};
};
}

36
mert/TER/infosHasher.cpp
View File

@ -27,35 +27,35 @@ using namespace Tools;
namespace HashMapSpace
{
infosHasher::infosHasher (long cle,string cleTxt, vector<int> valueVecInt )
{
infosHasher::infosHasher (long cle,string cleTxt, vector<int> valueVecInt )
{
m_hashKey=cle;
m_key=cleTxt;
m_value=valueVecInt;
}
}
// infosHasher::~infosHasher(){};*/
long infosHasher::getHashKey()
{
long infosHasher::getHashKey()
{
return m_hashKey;
}
string infosHasher::getKey()
{
}
string infosHasher::getKey()
{
return m_key;
}
vector<int> infosHasher::getValue()
{
}
vector<int> infosHasher::getValue()
{
return m_value;
}
void infosHasher::setValue ( vector<int> value )
{
}
void infosHasher::setValue ( vector<int> value )
{
m_value=value;
}
string infosHasher::toString()
{
}
string infosHasher::toString()
{
stringstream to_return;
to_return << m_hashKey << "\t" << m_key << "\t" << vectorToString(m_value,"\t") << endl;
return to_return.str();
}
}
// typedef stdext::hash_map<std::string,string, stringhasher> HASH_S_S;

10
mert/TER/infosHasher.h
View File

@ -31,14 +31,14 @@ Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
using namespace std;
namespace HashMapSpace
{
class infosHasher
{
private:
class infosHasher
{
private:
long m_hashKey;
string m_key;
vector<int> m_value;
public:
public:
infosHasher ( long cle, string cleTxt, vector<int> valueVecInt );
long getHashKey();
string getKey();
@ -47,7 +47,7 @@ namespace HashMapSpace
string toString();
};
};
}

30
mert/TER/stringHasher.cpp
View File

@ -26,29 +26,29 @@ using namespace std;
namespace HashMapSpace
{
stringHasher::stringHasher ( long cle, string cleTxt, string valueTxt )
{
stringHasher::stringHasher ( long cle, string cleTxt, string valueTxt )
{
m_hashKey=cle;
m_key=cleTxt;
m_value=valueTxt;
}
}
// stringHasher::~stringHasher(){};*/
long stringHasher::getHashKey()
{
long stringHasher::getHashKey()
{
return m_hashKey;
}
string stringHasher::getKey()
{
}
string stringHasher::getKey()
{
return m_key;
}
string stringHasher::getValue()
{
}
string stringHasher::getValue()
{
return m_value;
}
void stringHasher::setValue ( string value )
{
}
void stringHasher::setValue ( string value )
{
m_value=value;
}
}
// typedef stdext::hash_map<string, string, stringhasher> HASH_S_S;

10
mert/TER/stringHasher.h
View File

@ -28,14 +28,14 @@ using namespace std;
namespace HashMapSpace
{
class stringHasher
{
private:
class stringHasher
{
private:
long m_hashKey;
string m_key;
string m_value;
public:
public:
stringHasher ( long cle, string cleTxt, string valueTxt );
long getHashKey();
string getKey();
@ -43,7 +43,7 @@ namespace HashMapSpace
void setValue ( string value );
};
};
}

36
mert/TER/stringInfosHasher.cpp
View File

@ -27,35 +27,35 @@ using namespace Tools;
namespace HashMapSpace
{
stringInfosHasher::stringInfosHasher ( long cle, string cleTxt, vector<string> valueVecInt )
{
stringInfosHasher::stringInfosHasher ( long cle, string cleTxt, vector<string> valueVecInt )
{
m_hashKey=cle;
m_key=cleTxt;
m_value=valueVecInt;
}
}
// stringInfosHasher::~stringInfosHasher(){};*/
long stringInfosHasher::getHashKey()
{
long stringInfosHasher::getHashKey()
{
return m_hashKey;
}
string stringInfosHasher::getKey()
{
}
string stringInfosHasher::getKey()
{
return m_key;
}
vector<string> stringInfosHasher::getValue()
{
}
vector<string> stringInfosHasher::getValue()
{
return m_value;
}
void stringInfosHasher::setValue ( vector<string> value )
{
}
void stringInfosHasher::setValue ( vector<string> value )
{
m_value=value;
}
string stringInfosHasher::toString()
{
}
string stringInfosHasher::toString()
{
stringstream to_return;
to_return << m_hashKey << "\t" << m_key << "\t" << vectorToString(m_value,"\t") << endl;
return to_return.str();
}
}
// typedef stdext::hash_map<string, string, stringhasher> HASH_S_S;

10
mert/TER/stringInfosHasher.h
View File

@ -29,14 +29,14 @@ Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
using namespace std;
namespace HashMapSpace
{
class stringInfosHasher
{
private:
class stringInfosHasher
{
private:
long m_hashKey;
string m_key;
vector<string> m_value;
public:
public:
stringInfosHasher ( long cle, string cleTxt, vector<string> valueVecInt );
long getHashKey();
string getKey();
@ -45,7 +45,7 @@ namespace HashMapSpace
string toString();
};
};
}

118
mert/TER/terAlignment.cpp
View File

@ -24,8 +24,8 @@ using namespace std;
namespace TERCpp
{
terAlignment::terAlignment()
{
terAlignment::terAlignment()
{
// vector<string> ref;
// vector<string> hyp;
// vector<string> aftershift;
@ -41,9 +41,9 @@ namespace TERCpp
numSub=0;
numSft=0;
numWsf=0;
}
string terAlignment::toString()
{
}
string terAlignment::toString()
{
stringstream s;
s.str ( "" );
s << "Original Ref: \t" << join ( " ", ref ) << endl;
@ -52,12 +52,10 @@ namespace TERCpp
// s << "Hyp After Shift: " << join ( " ", aftershift );
s << endl;
// string s = "Original Ref: " + join(" ", ref) + "\nOriginal Hyp: " + join(" ", hyp) + "\nHyp After Shift: " + join(" ", aftershift);
if ( ( int ) sizeof ( alignment ) >0 )
{
if ( ( int ) sizeof ( alignment ) >0 ) {
s << "Alignment: (";
// s += "\nAlignment: (";
for ( int i = 0; i < ( int ) ( alignment.size() ); i++ )
{
for ( int i = 0; i < ( int ) ( alignment.size() ); i++ ) {
s << alignment[i];
// s+=alignment[i];
}
@ -65,17 +63,13 @@ namespace TERCpp
s << ")";
}
s << endl;
if ( ( int ) allshifts.size() == 0 )
{
if ( ( int ) allshifts.size() == 0 ) {
// s += "\nNumShifts: 0";
s << "NumShifts: 0";
}
else
{
} else {
// s += "\nNumShifts: " + (int)allshifts.size();
s << "NumShifts: "<< ( int ) allshifts.size();
for ( int i = 0; i < ( int ) allshifts.size(); i++ )
{
for ( int i = 0; i < ( int ) allshifts.size(); i++ ) {
s << endl << " " ;
s << ( ( terShift ) allshifts[i] ).toString();
// s += "\n " + allshifts[i];
@ -85,71 +79,58 @@ namespace TERCpp
// s += "\nScore: " + score() + " (" + numEdits + "/" + numWords + ")";
return s.str();
}
string terAlignment::join ( string delim, vector<string> arr )
{
}
string terAlignment::join ( string delim, vector<string> arr )
{
if ( ( int ) arr.size() == 0 ) return "";
// if ((int)delim.compare("") == 0) delim = new String("");
// String s = new String("");
stringstream s;
s.str ( "" );
for ( int i = 0; i < ( int ) arr.size(); i++ )
{
if ( i == 0 )
{
for ( int i = 0; i < ( int ) arr.size(); i++ ) {
if ( i == 0 ) {
s << arr.at ( i );
}
else
{
} else {
s << delim << arr.at ( i );
}
}
return s.str();
// return "";
}
double terAlignment::score()
{
if ( ( numWords <= 0.0 ) && ( numEdits > 0.0 ) )
{
}
double terAlignment::score()
{
if ( ( numWords <= 0.0 ) && ( numEdits > 0.0 ) ) {
return 1.0;
}
if ( numWords <= 0.0 )
{
if ( numWords <= 0.0 ) {
return 0.0;
}
return ( double ) numEdits / numWords;
}
double terAlignment::scoreAv()
{
if ( ( averageWords <= 0.0 ) && ( numEdits > 0.0 ) )
{
}
double terAlignment::scoreAv()
{
if ( ( averageWords <= 0.0 ) && ( numEdits > 0.0 ) ) {
return 1.0;
}
if ( averageWords <= 0.0 )
{
if ( averageWords <= 0.0 ) {
return 0.0;
}
return ( double ) numEdits / averageWords;
}
}
void terAlignment::scoreDetails()
{
void terAlignment::scoreDetails()
{
numIns = numDel = numSub = numWsf = numSft = 0;
if((int)allshifts.size()>0)
{
for(int i = 0; i < (int)allshifts.size(); ++i)
{
if((int)allshifts.size()>0) {
for(int i = 0; i < (int)allshifts.size(); ++i) {
numWsf += allshifts[i].size();
}
numSft = allshifts.size();
}
if((int)alignment.size()>0 )
{
for(int i = 0; i < (int)alignment.size(); ++i)
{
switch (alignment[i])
{
if((int)alignment.size()>0 ) {
for(int i = 0; i < (int)alignment.size(); ++i) {
switch (alignment[i]) {
case 'S':
case 'T':
numSub++;
@ -166,43 +147,34 @@ namespace TERCpp
// if(numEdits != numSft + numDel + numIns + numSub)
// System.out.println("** Error, unmatch edit erros " + numEdits +
// " vs " + (numSft + numDel + numIns + numSub));
}
string terAlignment::printAlignments()
{
}
string terAlignment::printAlignments()
{
stringstream to_return;
for(int i = 0; i < (int)alignment.size(); ++i)
{
for(int i = 0; i < (int)alignment.size(); ++i) {
char alignInfo=alignment.at(i);
if (alignInfo == 'A' )
{
if (alignInfo == 'A' ) {
alignInfo='A';
}
if (i==0)
{
if (i==0) {
to_return << alignInfo;
}
else
{
} else {
to_return << " " << alignInfo;
}
}
return to_return.str();
}
}
string terAlignment::printAllShifts()
{
stringstream to_return;
if ( ( int ) allshifts.size() == 0 )
{
if ( ( int ) allshifts.size() == 0 ) {
// s += "\nNumShifts: 0";
to_return << "NbrShifts: 0";
}
else
{
} else {
// s += "\nNumShifts: " + (int)allshifts.size();
to_return << "NbrShifts: "<< ( int ) allshifts.size();
for ( int i = 0; i < ( int ) allshifts.size(); i++ )
{
for ( int i = 0; i < ( int ) allshifts.size(); i++ ) {
to_return << "\t" ;
to_return << ( ( terShift ) allshifts[i] ).toString();
// s += "\n " + allshifts[i];

10
mert/TER/terAlignment.h
View File

@ -34,10 +34,10 @@ using namespace std;
namespace TERCpp
{
class terAlignment
{
private:
public:
class terAlignment
{
private:
public:
terAlignment();
string toString();
@ -68,7 +68,7 @@ namespace TERCpp
double scoreAv();
string printAlignments();
string printAllShifts();
};
};
}
#endif

58
mert/TER/terShift.cpp
View File

@ -42,32 +42,32 @@ namespace TERCpp
// numSft=0;
// numWsf=0;
// }
terShift::terShift ()
{
terShift::terShift ()
{
start = 0;
end = 0;
moveto = 0;
newloc = 0;
cost=1.0;
}
terShift::terShift ( int _start, int _end, int _moveto, int _newloc )
{
}
terShift::terShift ( int _start, int _end, int _moveto, int _newloc )
{
start = _start;
end = _end;
moveto = _moveto;
newloc = _newloc;
cost=1.0;
}
}
terShift::terShift ( int _start, int _end, int _moveto, int _newloc, vector<string> _shifted )
{
terShift::terShift ( int _start, int _end, int _moveto, int _newloc, vector<string> _shifted )
{
start = _start;
end = _end;
moveto = _moveto;
newloc = _newloc;
shifted = _shifted;
cost=1.0;
}
}
// string terShift::vectorToString(vector<string> vec)
// {
// string retour("");
@ -78,44 +78,38 @@ namespace TERCpp
// return retour;
// }
string terShift::toString()
{
string terShift::toString()
{
stringstream s;
s.str ( "" );
s << "[" << start << ", " << end << ", " << moveto << "/" << newloc << "]";
if ( ( int ) shifted.size() > 0 )
{
if ( ( int ) shifted.size() > 0 ) {
s << " (" << vectorToString ( shifted ) << ")";
}
return s.str();
}
}
/* The distance of the shift. */
int terShift::distance()
{
if ( moveto < start )
{
/* The distance of the shift. */
int terShift::distance()
{
if ( moveto < start ) {
return start - moveto;
}
else if ( moveto > end )
{
} else if ( moveto > end ) {
return moveto - end;
}
else
{
} else {
return moveto - start;
}
}
}
bool terShift::leftShift()
{
bool terShift::leftShift()
{
return ( moveto < start );
}
}
int terShift::size()
{
int terShift::size()
{
return ( end - start ) + 1;
}
}
// terShift terShift::operator=(terShift t)
// {
//

10
mert/TER/terShift.h
View File

@ -34,10 +34,10 @@ using namespace Tools;
namespace TERCpp
{
class terShift
{
private:
public:
class terShift
{
private:
public:
terShift();
terShift ( int _start, int _end, int _moveto, int _newloc );
@ -59,7 +59,7 @@ namespace TERCpp
// This is used to store the cost of a shift, so we don't have to
// calculate it multiple times.
double cost;
};
};
}
#endif

548
mert/TER/tercalc.cpp
View File

@ -35,8 +35,8 @@ using namespace Tools;
namespace TERCpp
{
terCalc::terCalc()
{
terCalc::terCalc()
{
TAILLE_PERMUT_MAX = 50;
infinite = 999999.0;
shift_cost = 1.0;
@ -52,108 +52,88 @@ namespace TERCpp
PRINT_DEBUG = false;
hypSpans.clear();
refSpans.clear();
}
}
terAlignment terCalc::WERCalculation ( vector< string > hyp , vector< string > ref )
{
terAlignment terCalc::WERCalculation ( vector< string > hyp , vector< string > ref )
{
return minimizeDistanceEdition ( hyp, ref, hypSpans );
}
}
terAlignment terCalc::TER ( std::vector< int > hyp, std::vector< int > ref )
{
terAlignment terCalc::TER ( std::vector< int > hyp, std::vector< int > ref )
{
stringstream s;
s.str ( "" );
string stringRef ( "" );
string stringHyp ( "" );
for ( vector<int>::iterator l_it = ref.begin(); l_it != ref.end(); l_it++ )
{
if ( l_it == ref.begin() )
{
for ( vector<int>::iterator l_it = ref.begin(); l_it != ref.end(); l_it++ ) {
if ( l_it == ref.begin() ) {
s << ( *l_it );
}
else
{
} else {
s << " " << ( *l_it );
}
}
stringRef = s.str();
s.str ( "" );
for ( vector<int>::iterator l_itHyp = hyp.begin(); l_itHyp != hyp.end(); l_itHyp++ )
{
if ( l_itHyp == hyp.begin() )
{
for ( vector<int>::iterator l_itHyp = hyp.begin(); l_itHyp != hyp.end(); l_itHyp++ ) {
if ( l_itHyp == hyp.begin() ) {
s << ( *l_itHyp );
}
else
{
} else {
s << " " << ( *l_itHyp );
}
}
stringHyp = s.str();
s.str ( "" );
return TER ( stringToVector ( stringRef , " " ), stringToVector ( stringHyp , " " ) );
}
}
hashMapInfos terCalc::createConcordMots ( vector<string> hyp, vector<string> ref )
{
hashMapInfos terCalc::createConcordMots ( vector<string> hyp, vector<string> ref )
{
hashMap tempHash;
hashMapInfos retour;
for ( int i = 0; i < ( int ) hyp.size(); i++ )
{
for ( int i = 0; i < ( int ) hyp.size(); i++ ) {
tempHash.addHasher ( hyp.at ( i ), "" );
}
bool cor[ref.size() ];
for ( int i = 0; i < ( int ) ref.size(); i++ )
{
if ( tempHash.trouve ( ( string ) ref.at ( i ) ) )
{
for ( int i = 0; i < ( int ) ref.size(); i++ ) {
if ( tempHash.trouve ( ( string ) ref.at ( i ) ) ) {
cor[i] = true;
}
else
{
} else {
cor[i] = false;
}
}
for ( int start = 0; start < ( int ) ref.size(); start++ )
{
if ( cor[start] )
{
for ( int end = start; ( ( end < ( int ) ref.size() ) && ( end - start <= TAILLE_PERMUT_MAX ) && ( cor[end] ) );end++ )
{
for ( int start = 0; start < ( int ) ref.size(); start++ ) {
if ( cor[start] ) {
for ( int end = start; ( ( end < ( int ) ref.size() ) && ( end - start <= TAILLE_PERMUT_MAX ) && ( cor[end] ) ); end++ ) {
vector<string> ajouter = subVector ( ref, start, end + 1 );
string ajouterString = vectorToString ( ajouter );
vector<int> values = retour.getValue ( ajouterString );
values.push_back ( start );
if ( values.size() > 1 )
{
if ( values.size() > 1 ) {
retour.setValue ( ajouterString, values );
}
else
{
} else {
retour.addValue ( ajouterString, values );
}
}
}
}
return retour;
}
}
bool terCalc::trouverIntersection ( vecInt refSpan, vecInt hypSpan )
{
if ( ( refSpan.at ( 1 ) >= hypSpan.at ( 0 ) ) && ( refSpan.at ( 0 ) <= hypSpan.at ( 1 ) ) )
{
bool terCalc::trouverIntersection ( vecInt refSpan, vecInt hypSpan )
{
if ( ( refSpan.at ( 1 ) >= hypSpan.at ( 0 ) ) && ( refSpan.at ( 0 ) <= hypSpan.at ( 1 ) ) ) {
return true;
}
return false;
}
}
terAlignment terCalc::minimizeDistanceEdition ( vector<string> hyp, vector<string> ref, vector<vecInt> curHypSpans )
{
terAlignment terCalc::minimizeDistanceEdition ( vector<string> hyp, vector<string> ref, vector<vecInt> curHypSpans )
{
double current_best = infinite;
double last_best = infinite;
int first_good = 0;
@ -171,17 +151,14 @@ namespace TERCpp
NBR_BS_APPELS++;
for ( i = 0; i <= ( int ) ref.size(); i++ )
{
for ( j = 0; j <= ( int ) hyp.size(); j++ )
{
for ( i = 0; i <= ( int ) ref.size(); i++ ) {
for ( j = 0; j <= ( int ) hyp.size(); j++ ) {
S[i][j] = -1.0;
P[i][j] = '0';
}
}
S[0][0] = 0.0;
for ( j = 0; j <= ( int ) hyp.size(); j++ )
{
for ( j = 0; j <= ( int ) hyp.size(); j++ ) {
last_best = current_best;
current_best = infinite;
first_good = current_first_good;
@ -190,59 +167,43 @@ namespace TERCpp
cur_last_good = -1;
last_peak = cur_last_peak;
cur_last_peak = 0;
for ( i = first_good; i <= ( int ) ref.size(); i++ )
{
if ( i > last_good )
{
for ( i = first_good; i <= ( int ) ref.size(); i++ ) {
if ( i > last_good ) {
break;
}
if ( S[i][j] < 0 )
{
if ( S[i][j] < 0 ) {
continue;
}
score = S[i][j];
if ( ( j < ( int ) hyp.size() ) && ( score > last_best + TAILLE_BEAM ) )
{
if ( ( j < ( int ) hyp.size() ) && ( score > last_best + TAILLE_BEAM ) ) {
continue;
}
if ( current_first_good == -1 )
{
if ( current_first_good == -1 ) {
current_first_good = i ;
}
if ( ( i < ( int ) ref.size() ) && ( j < ( int ) hyp.size() ) )
{
if ( ( int ) refSpans.size() == 0 || ( int ) hypSpans.size() == 0 || trouverIntersection ( refSpans.at ( i ), curHypSpans.at ( j ) ) )
{
if ( ( int ) ( ref.at ( i ).compare ( hyp.at ( j ) ) ) == 0 )
{
if ( ( i < ( int ) ref.size() ) && ( j < ( int ) hyp.size() ) ) {
if ( ( int ) refSpans.size() == 0 || ( int ) hypSpans.size() == 0 || trouverIntersection ( refSpans.at ( i ), curHypSpans.at ( j ) ) ) {
if ( ( int ) ( ref.at ( i ).compare ( hyp.at ( j ) ) ) == 0 ) {
cost = match_cost + score;
if ( ( S[i+1][j+1] == -1 ) || ( cost < S[i+1][j+1] ) )
{
if ( ( S[i+1][j+1] == -1 ) || ( cost < S[i+1][j+1] ) ) {
S[i+1][j+1] = cost;
P[i+1][j+1] = 'A';
}
if ( cost < current_best )
{
if ( cost < current_best ) {
current_best = cost;
}
if ( current_best == cost )
{
if ( current_best == cost ) {
cur_last_peak = i + 1;
}
}
else
{
} else {
cost = substitute_cost + score;
if ( ( S[i+1][j+1] < 0 ) || ( cost < S[i+1][j+1] ) )
{
if ( ( S[i+1][j+1] < 0 ) || ( cost < S[i+1][j+1] ) ) {
S[i+1][j+1] = cost;
P[i+1][j+1] = 'S';
if ( cost < current_best )
{
if ( cost < current_best ) {
current_best = cost;
}
if ( current_best == cost )
{
if ( current_best == cost ) {
cur_last_peak = i + 1 ;
}
}
@ -250,28 +211,22 @@ namespace TERCpp
}
}
cur_last_good = i + 1;
if ( j < ( int ) hyp.size() )
{
if ( j < ( int ) hyp.size() ) {
icost = score + insert_cost;
if ( ( S[i][j+1] < 0 ) || ( S[i][j+1] > icost ) )
{
if ( ( S[i][j+1] < 0 ) || ( S[i][j+1] > icost ) ) {
S[i][j+1] = icost;
P[i][j+1] = 'I';
if ( ( cur_last_peak < i ) && ( current_best == icost ) )
{
if ( ( cur_last_peak < i ) && ( current_best == icost ) ) {
cur_last_peak = i;
}
}
}
if ( i < ( int ) ref.size() )
{
if ( i < ( int ) ref.size() ) {
dcost = score + delete_cost;
if ( ( S[ i+1][ j] < 0.0 ) || ( S[i+1][j] > dcost ) )
{
if ( ( S[ i+1][ j] < 0.0 ) || ( S[i+1][j] > dcost ) ) {
S[i+1][j] = dcost;
P[i+1][j] = 'D';
if ( i >= last_good )
{
if ( i >= last_good ) {
last_good = i + 1 ;
}
}
@ -283,32 +238,19 @@ namespace TERCpp
int tracelength = 0;
i = ref.size();
j = hyp.size();
while ( ( i > 0 ) || ( j > 0 ) )
{
while ( ( i > 0 ) || ( j > 0 ) ) {
tracelength++;
if ( P[i][j] == 'A' )
{
if ( P[i][j] == 'A' ) {
i--;
j--;
}
else
if ( P[i][j] == 'S' )
{
} else if ( P[i][j] == 'S' ) {
i--;
j--;
}
else
if ( P[i][j] == 'D' )
{
} else if ( P[i][j] == 'D' ) {
i--;
}
else
if ( P[i][j] == 'I' )
{
} else if ( P[i][j] == 'I' ) {
j--;
}
else
{
} else {
cerr << "ERROR : terCalc::minimizeDistanceEdition : Invalid path : " << P[i][j] << endl;
exit ( -1 );
}
@ -316,28 +258,17 @@ namespace TERCpp
vector<char> path ( tracelength );
i = ref.size();
j = hyp.size();
while ( ( i > 0 ) || ( j > 0 ) )
{
while ( ( i > 0 ) || ( j > 0 ) ) {
path[--tracelength] = P[i][j];
if ( P[i][j] == 'A' )
{
if ( P[i][j] == 'A' ) {
i--;
j--;
}
else
if ( P[i][j] == 'S' )
{
} else if ( P[i][j] == 'S' ) {
i--;
j--;
}
else
if ( P[i][j] == 'D' )
{
} else if ( P[i][j] == 'D' ) {
i--;
}
else
if ( P[i][j] == 'I' )
{
} else if ( P[i][j] == 'I' ) {
j--;
}
}
@ -348,15 +279,14 @@ namespace TERCpp
to_return.hyp = hyp;
to_return.ref = ref;
to_return.averageWords = (int)ref.size();
if ( PRINT_DEBUG )
{
if ( PRINT_DEBUG ) {
cerr << "BEGIN DEBUG : terCalc::minimizeDistanceEdition : to_return :" << endl << to_return.toString() << endl << "END DEBUG" << endl;
}
return to_return;
}
terAlignment terCalc::TER ( vector<string> hyp, vector<string> ref )
{
}
terAlignment terCalc::TER ( vector<string> hyp, vector<string> ref )
{
hashMapInfos rloc = createConcordMots ( hyp, ref );
terAlignment cur_align = minimizeDistanceEdition ( hyp, ref, hypSpans );
vector<string> cur = hyp;
@ -369,16 +299,13 @@ namespace TERCpp
vector<terShift> allshifts;
// cerr << "Initial Alignment:" << endl << cur_align.toString() <<endl;
if ( PRINT_DEBUG )
{
if ( PRINT_DEBUG ) {
cerr << "BEGIN DEBUG : terCalc::TER : cur_align :" << endl << cur_align.toString() << endl << "END DEBUG" << endl;
}
while ( true )
{
while ( true ) {
bestShiftStruct returns;
returns = findBestShift ( cur, hyp, ref, rloc, cur_align );
if ( returns.m_empty )
{
if ( returns.m_empty ) {
break;
}
terShift bestShift = returns.m_best_shift;
@ -395,9 +322,9 @@ namespace TERCpp
to_return.numEdits += edits;
NBR_SEGS_EVALUATED++;
return to_return;
}
bestShiftStruct terCalc::findBestShift ( vector<string> cur, vector<string> hyp, vector<string> ref, hashMapInfos rloc, terAlignment med_align )
{
}
bestShiftStruct terCalc::findBestShift ( vector<string> cur, vector<string> hyp, vector<string> ref, hashMapInfos rloc, terAlignment med_align )
{
bestShiftStruct to_return;
bool anygain = false;
bool herr[ ( int ) hyp.size() ];
@ -406,12 +333,10 @@ namespace TERCpp
calculateTerAlignment ( med_align, herr, rerr, ralign );
vector<vecTerShift> poss_shifts;
if ( PRINT_DEBUG )
{
if ( PRINT_DEBUG ) {
cerr << "BEGIN DEBUG : terCalc::findBestShift (after the calculateTerAlignment call) :" << endl;
cerr << "indices: ";
for (int l_i=0; l_i < ( int ) ref.size() ; l_i++)
{
for (int l_i=0; l_i < ( int ) ref.size() ; l_i++) {
cerr << l_i << "\t";
}
cerr << endl;
@ -425,14 +350,11 @@ namespace TERCpp
}
poss_shifts = calculerPermutations ( cur, ref, rloc, med_align, herr, rerr, ralign );
double curerr = med_align.numEdits;
if ( PRINT_DEBUG )
{
if ( PRINT_DEBUG ) {
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
cerr << "Possible Shifts:" << endl;
for ( int i = ( int ) poss_shifts.size() - 1; i >= 0; i-- )
{
for ( int j = 0; j < ( int ) ( poss_shifts.at ( i ) ).size(); j++ )
{
for ( int i = ( int ) poss_shifts.size() - 1; i >= 0; i-- ) {
for ( int j = 0; j < ( int ) ( poss_shifts.at ( i ) ).size(); j++ ) {
cerr << " [" << i << "] " << ( ( poss_shifts.at ( i ) ).at ( j ) ).toString() << endl;
}
}
@ -446,10 +368,8 @@ namespace TERCpp
for ( int i = ( int ) poss_shifts.size() - 1; i >= 0; i-- )
{
if ( PRINT_DEBUG )
{
for ( int i = ( int ) poss_shifts.size() - 1; i >= 0; i-- ) {
if ( PRINT_DEBUG ) {
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
cerr << "Considering shift of length " << i << " (" << ( poss_shifts.at ( i ) ).size() << ")" << endl;
cerr << "END DEBUG " << endl;
@ -457,21 +377,17 @@ namespace TERCpp
/* Consider shifts of length i+1 */
double curfix = curerr - ( cur_best_shift_cost + cur_best_align.numEdits );
double maxfix = ( 2 * ( 1 + i ) );
if ( ( curfix > maxfix ) || ( ( cur_best_shift_cost != 0 ) && ( curfix == maxfix ) ) )
{
if ( ( curfix > maxfix ) || ( ( cur_best_shift_cost != 0 ) && ( curfix == maxfix ) ) ) {
break;
}
for ( int s = 0; s < ( int ) ( poss_shifts.at ( i ) ).size(); s++ )
{
for ( int s = 0; s < ( int ) ( poss_shifts.at ( i ) ).size(); s++ ) {
curfix = curerr - ( cur_best_shift_cost + cur_best_align.numEdits );
if ( ( curfix > maxfix ) || ( ( cur_best_shift_cost != 0 ) && ( curfix == maxfix ) ) )
{
if ( ( curfix > maxfix ) || ( ( cur_best_shift_cost != 0 ) && ( curfix == maxfix ) ) ) {
break;
}
terShift curshift = ( poss_shifts.at ( i ) ).at ( s );
if ( PRINT_DEBUG )
{
if ( PRINT_DEBUG ) {
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
cerr << "cur : "<< join(" ",cur) << endl;
cerr << "curshift : "<< curshift.toString() << endl;
@ -481,8 +397,7 @@ namespace TERCpp
vector<string> shiftarr = shiftReturns.nwords;
vector<vecInt> curHypSpans = shiftReturns.aftershift;
if ( PRINT_DEBUG )
{
if ( PRINT_DEBUG ) {
cerr << "shiftarr : "<< join(" ",shiftarr) << endl;
// cerr << "curHypSpans : "<< curHypSpans.toString() << endl;
cerr << "END DEBUG " << endl;
@ -498,8 +413,7 @@ namespace TERCpp
// if (DEBUG) {
// string testeuh=terAlignment join(" ", shiftarr);
if ( PRINT_DEBUG )
{
if ( PRINT_DEBUG ) {
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
cerr << "Gain for " << curshift.toString() << " is " << gain << ". (result: [" << curalign.join ( " ", shiftarr ) << "]" << endl;
cerr << "Details of gains : gain = ( cur_best_align.numEdits + cur_best_shift_cost ) - ( curalign.numEdits + curshift.cost )"<<endl;
@ -509,15 +423,13 @@ namespace TERCpp
}
// }
//
if ( ( gain > 0 ) || ( ( cur_best_shift_cost == 0 ) && ( gain == 0 ) ) )
{
if ( ( gain > 0 ) || ( ( cur_best_shift_cost == 0 ) && ( gain == 0 ) ) ) {
anygain = true;
cur_best_shift = curshift;
cur_best_shift_cost = curshift.cost;
cur_best_align = curalign;
// if (DEBUG)
if ( PRINT_DEBUG )
{
if ( PRINT_DEBUG ) {
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
cerr << "Tmp Choosing shift: " << cur_best_shift.toString() << " gives:\n" << cur_best_align.toString() << "\n" << endl;
cerr << "END DEBUG " << endl;
@ -525,131 +437,102 @@ namespace TERCpp
}
}
}
if ( anygain )
{
if ( anygain ) {
to_return.m_best_shift = cur_best_shift;
to_return.m_best_align = cur_best_align;
to_return.m_empty = false;
}
else
{
} else {
to_return.m_empty = true;
}
return to_return;
}
}
void terCalc::calculateTerAlignment ( terAlignment align, bool* herr, bool* rerr, int* ralign )
{
void terCalc::calculateTerAlignment ( terAlignment align, bool* herr, bool* rerr, int* ralign )
{
int hpos = -1;
int rpos = -1;
if ( PRINT_DEBUG )
{
if ( PRINT_DEBUG ) {
cerr << "BEGIN DEBUG : terCalc::calculateTerAlignment : " << endl << align.toString() << endl;
cerr << "END DEBUG " << endl;
}
for ( int i = 0; i < ( int ) align.alignment.size(); i++ )
{
for ( int i = 0; i < ( int ) align.alignment.size(); i++ ) {
herr[i] = false;
rerr[i] = false;
ralign[i] = -1;
}
for ( int i = 0; i < ( int ) align.alignment.size(); i++ )
{
for ( int i = 0; i < ( int ) align.alignment.size(); i++ ) {
char sym = align.alignment[i];
if ( sym == 'A' )
{
if ( sym == 'A' ) {
hpos++;
rpos++;
herr[hpos] = false;
rerr[rpos] = false;
ralign[rpos] = hpos;
}
else
if ( sym == 'S' )
{
} else if ( sym == 'S' ) {
hpos++;
rpos++;
herr[hpos] = true;
rerr[rpos] = true;
ralign[rpos] = hpos;
}
else
if ( sym == 'I' )
{
} else if ( sym == 'I' ) {
hpos++;
herr[hpos] = true;
}
else
if ( sym == 'D' )
{
} else if ( sym == 'D' ) {
rpos++;
rerr[rpos] = true;
ralign[rpos] = hpos+1;
}
else
{
} else {
cerr << "ERROR : terCalc::calculateTerAlignment : Invalid mini align sequence " << sym << " at pos " << i << endl;
exit ( -1 );
}
}
}
}
vector<vecTerShift> terCalc::calculerPermutations ( vector<string> hyp, vector<string> ref, hashMapInfos rloc, terAlignment align, bool* herr, bool* rerr, int* ralign )
{
vector<vecTerShift> terCalc::calculerPermutations ( vector<string> hyp, vector<string> ref, hashMapInfos rloc, terAlignment align, bool* herr, bool* rerr, int* ralign )
{
vector<vecTerShift> to_return;
if ( ( TAILLE_PERMUT_MAX <= 0 ) || ( DIST_MAX_PERMUT <= 0 ) )
{
if ( ( TAILLE_PERMUT_MAX <= 0 ) || ( DIST_MAX_PERMUT <= 0 ) ) {
return to_return;
}
vector<vecTerShift> allshifts ( TAILLE_PERMUT_MAX + 1 );
for ( int start = 0; start < ( int ) hyp.size(); start++ )
{
for ( int start = 0; start < ( int ) hyp.size(); start++ ) {
string subVectorHypString = vectorToString ( subVector ( hyp, start, start + 1 ) );
if ( ! rloc.trouve ( subVectorHypString ) )
{
if ( ! rloc.trouve ( subVectorHypString ) ) {
continue;
}
bool ok = false;
vector<int> mtiVec = rloc.getValue ( subVectorHypString );
vector<int>::iterator mti = mtiVec.begin();
while ( mti != mtiVec.end() && ( ! ok ) )
{
while ( mti != mtiVec.end() && ( ! ok ) ) {
int moveto = ( *mti );
mti++;
if ( ( start != ralign[moveto] ) && ( ( ralign[moveto] - start ) <= DIST_MAX_PERMUT ) && ( ( start - ralign[moveto] - 1 ) <= DIST_MAX_PERMUT ) )
{
if ( ( start != ralign[moveto] ) && ( ( ralign[moveto] - start ) <= DIST_MAX_PERMUT ) && ( ( start - ralign[moveto] - 1 ) <= DIST_MAX_PERMUT ) ) {
ok = true;
}
}
if ( ! ok )
{
if ( ! ok ) {
continue;
}
ok = true;
for ( int end = start; ( ok && ( end < ( int ) hyp.size() ) && ( end < start + TAILLE_PERMUT_MAX ) ); end++ )
{
for ( int end = start; ( ok && ( end < ( int ) hyp.size() ) && ( end < start + TAILLE_PERMUT_MAX ) ); end++ ) {
/* check if cand is good if so, add it */
vector<string> cand = subVector ( hyp, start, end + 1 );
ok = false;
if ( ! ( rloc.trouve ( vectorToString ( cand ) ) ) )
{
if ( ! ( rloc.trouve ( vectorToString ( cand ) ) ) ) {
continue;
}
bool any_herr = false;
for ( int i = 0; ( ( i <= ( end - start ) ) && ( ! any_herr ) ); i++ )
{
if ( herr[start+i] )
{
for ( int i = 0; ( ( i <= ( end - start ) ) && ( ! any_herr ) ); i++ ) {
if ( herr[start+i] ) {
any_herr = true;
}
}
if ( any_herr == false )
{
if ( any_herr == false ) {
ok = true;
continue;
}
@ -658,12 +541,10 @@ namespace TERCpp
movetoitVec = rloc.getValue ( ( string ) vectorToString ( cand ) );
// cerr << "CANDIDATE " << ( string ) vectorToString ( cand ) <<" PLACED : " << ( string ) vectorToString ( movetoitVec," ") << endl;
vector<int>::iterator movetoit = movetoitVec.begin();
while ( movetoit != movetoitVec.end() )
{
while ( movetoit != movetoitVec.end() ) {
int moveto = ( *movetoit );
movetoit++;
if ( ! ( ( ralign[moveto] != start ) && ( ( ralign[moveto] < start ) || ( ralign[moveto] > end ) ) && ( ( ralign[moveto] - start ) <= DIST_MAX_PERMUT ) && ( ( start - ralign[moveto] ) <= DIST_MAX_PERMUT ) ) )
{
if ( ! ( ( ralign[moveto] != start ) && ( ( ralign[moveto] < start ) || ( ralign[moveto] > end ) ) && ( ( ralign[moveto] - start ) <= DIST_MAX_PERMUT ) && ( ( start - ralign[moveto] ) <= DIST_MAX_PERMUT ) ) ) {
continue;
}
ok = true;
@ -673,38 +554,28 @@ namespace TERCpp
*/
bool any_rerr = false;
for ( int i = 0; ( i <= end - start ) && ( ! any_rerr ); i++ )
{
if ( rerr[moveto+i] )
{
for ( int i = 0; ( i <= end - start ) && ( ! any_rerr ); i++ ) {
if ( rerr[moveto+i] ) {
any_rerr = true;
}
}
if ( ! any_rerr )
{
if ( ! any_rerr ) {
continue;
}
for ( int roff = -1; roff <= ( end - start ); roff++ )
{
for ( int roff = -1; roff <= ( end - start ); roff++ ) {
terShift topush;
bool topushNull = true;
if ( ( roff == -1 ) && ( moveto == 0 ) )
{
if ( PRINT_DEBUG )
{
if ( ( roff == -1 ) && ( moveto == 0 ) ) {
if ( PRINT_DEBUG ) {
cerr << "BEGIN DEBUG : terCalc::calculerPermutations 01 : " << endl << "Consider making " << start << "..." << end << " (" << vectorToString(cand," ")<< ") moveto: " << moveto << " roff: " << roff << " ralign[mt+roff]: -1" << endl << "END DEBUG" << endl;
}
terShift t01 ( start, end, -1, -1 );
topush = t01;
topushNull = false;
}
else
if ( ( start != ralign[moveto+roff] ) && ( ( roff == 0 ) || ( ralign[moveto+roff] != ralign[moveto] ) ) )
{
} else if ( ( start != ralign[moveto+roff] ) && ( ( roff == 0 ) || ( ralign[moveto+roff] != ralign[moveto] ) ) ) {
int newloc = ralign[moveto+roff];
if ( PRINT_DEBUG )
{
if ( PRINT_DEBUG ) {
cerr << "BEGIN DEBUG : terCalc::calculerPermutations 02 : " << endl << "Consider making " << start << "..." << end << " (" << vectorToString(cand," ")<< ") moveto: " << moveto << " roff: " << roff << " ralign[mt+roff]: " << newloc << endl << "END DEBUG" << endl;
}
@ -712,12 +583,10 @@ namespace TERCpp
topush = t02;
topushNull = false;
}
if ( !topushNull )
{
if ( !topushNull ) {
topush.shifted = cand;
topush.cost = shift_cost;
if ( PRINT_DEBUG )
{
if ( PRINT_DEBUG ) {
cerr << "BEGIN DEBUG : terCalc::calculerPermutations 02 : " << endl;
cerr << "start : " << start << endl;
@ -732,43 +601,36 @@ namespace TERCpp
}
}
to_return.clear();
for ( int i = 0; i < TAILLE_PERMUT_MAX + 1; i++ )
{
for ( int i = 0; i < TAILLE_PERMUT_MAX + 1; i++ ) {
to_return.push_back ( ( vecTerShift ) allshifts.at ( i ) );
}
return to_return;
}
}
alignmentStruct terCalc::permuter ( vector<string> words, terShift s )
{
alignmentStruct terCalc::permuter ( vector<string> words, terShift s )
{
return permuter ( words, s.start, s.end, s.newloc );
}
}
alignmentStruct terCalc::permuter ( vector<string> words, int start, int end, int newloc )
{
alignmentStruct terCalc::permuter ( vector<string> words, int start, int end, int newloc )
{
int c = 0;
vector<string> nwords ( words );
vector<vecInt> spans ( ( int ) hypSpans.size() );
alignmentStruct to_return;
if ( PRINT_DEBUG )
{
if ( PRINT_DEBUG ) {
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
cerr << "BEGIN DEBUG : terCalc::permuter :" << endl << "word length: " << ( int ) words.size() << " span length: " << ( int ) hypSpans.size() << endl ;
}
else
{
} else {
cerr << "BEGIN DEBUG : terCalc::permuter :" << endl << "word length: " << ( int ) words.size() << " span length: null" << endl ;
}
cerr << "BEGIN DEBUG : terCalc::permuter :" << endl << join(" ",words) << " start: " << start << " end: " << end << " newloc "<< newloc << endl << "END DEBUG " << endl;
}
if (newloc >= ( int ) words.size())
{
if ( PRINT_DEBUG )
{
if (newloc >= ( int ) words.size()) {
if ( PRINT_DEBUG ) {
cerr << "WARNING: Relocation over the size of the hypothesis, replacing at the end of it."<<endl;
}
newloc = ( int ) words.size()-1;
@ -776,140 +638,101 @@ namespace TERCpp
// }
if ( newloc == -1 )
{
for ( int i = start; i <= end;i++ )
{
if ( newloc == -1 ) {
for ( int i = start; i <= end; i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
for ( int i = 0; i <= start - 1;i++ )
{
for ( int i = 0; i <= start - 1; i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
for ( int i = end + 1; i < ( int ) words.size();i++ )
{
for ( int i = end + 1; i < ( int ) words.size(); i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
}
else
{
if ( newloc < start )
{
} else {
if ( newloc < start ) {
for ( int i = 0; i < newloc; i++ )
{
for ( int i = 0; i < newloc; i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
for ( int i = start; i <= end;i++ )
{
for ( int i = start; i <= end; i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
for ( int i = newloc ; i < start ;i++ )
{
for ( int i = newloc ; i < start ; i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
for ( int i = end + 1; i < ( int ) words.size();i++ )
{
for ( int i = end + 1; i < ( int ) words.size(); i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
}
else
{
if ( newloc > end )
{
for ( int i = 0; i <= start - 1; i++ )
{
} else {
if ( newloc > end ) {
for ( int i = 0; i <= start - 1; i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
for ( int i = end + 1; i <= newloc;i++ )
{
for ( int i = end + 1; i <= newloc; i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
for ( int i = start; i <= end;i++ )
{
for ( int i = start; i <= end; i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
for ( int i = newloc + 1; i < ( int ) words.size();i++ )
{
for ( int i = newloc + 1; i < ( int ) words.size(); i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
}
else
{
} else {
// we are moving inside of ourselves
for ( int i = 0; i <= start - 1; i++ )
{
for ( int i = 0; i <= start - 1; i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
for ( int i = end + 1; ( i < ( int ) words.size() ) && ( i <= ( end + ( newloc - start ) ) ); i++ )
{
for ( int i = end + 1; ( i < ( int ) words.size() ) && ( i <= ( end + ( newloc - start ) ) ); i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
for ( int i = start; i <= end;i++ )
{
for ( int i = start; i <= end; i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
for ( int i = ( end + ( newloc - start ) + 1 ); i < ( int ) words.size();i++ )
{
for ( int i = ( end + ( newloc - start ) + 1 ); i < ( int ) words.size(); i++ ) {
nwords.at ( c++ ) = words.at ( i );
if ( ( int ) hypSpans.size() > 0 )
{
if ( ( int ) hypSpans.size() > 0 ) {
spans.at ( c - 1 ) = hypSpans.at ( i );
}
}
@ -918,8 +741,7 @@ namespace TERCpp
}
NBR_PERMUTS_CONSID++;
if ( PRINT_DEBUG )
{
if ( PRINT_DEBUG ) {
cerr << "nwords" << join(" ",nwords) << endl;
// cerr << "spans" << spans. << endl;
}
@ -927,10 +749,10 @@ namespace TERCpp
to_return.nwords = nwords;
to_return.aftershift = spans;
return to_return;
}
void terCalc::setDebugMode ( bool b )
{
}
void terCalc::setDebugMode ( bool b )
{
PRINT_DEBUG = b;
}
}
}

16
mert/TER/tercalc.h
View File

@ -41,13 +41,13 @@ namespace TERCpp
{
// typedef size_t WERelement[2];
// Vecteur d'alignement contenant le hash du mot et son evaluation (0=ok, 1=sub, 2=ins, 3=del)
typedef vector<terShift> vecTerShift;
/**
typedef vector<terShift> vecTerShift;
/**
@author
*/
class terCalc
{
private :
*/
class terCalc
{
private :
// Vecteur d'alignement contenant le hash du mot et son evaluation (0=ok, 1=sub, 2=ins, 3=del)
WERalignment l_WERalignment;
// HashMap contenant les valeurs de hash de chaque mot
@ -67,7 +67,7 @@ namespace TERCpp
vector<vecInt> hypSpans;
int TAILLE_BEAM;
public:
public:
int shift_cost;
int insert_cost;
int delete_cost;
@ -96,7 +96,7 @@ namespace TERCpp
vector<vecTerShift> calculerPermutations ( vector<string> hyp, vector<string> ref, hashMapInfos rloc, terAlignment align, bool* herr, bool* rerr, int* ralign );
alignmentStruct permuter ( vector<string> words, terShift s );
alignmentStruct permuter ( vector<string> words, int start, int end, int newloc );
};
};
}

353
mert/TER/tools.cpp
View File

@ -25,367 +25,300 @@ using namespace boost::xpressive;
namespace Tools
{
string vectorToString ( vector<string> vec )
{
string vectorToString ( vector<string> vec )
{
string retour ( "" );
for ( vector<string>::iterator vecIter = vec.begin();vecIter != vec.end(); vecIter++ )
{
if ( vecIter == vec.begin() )
{
for ( vector<string>::iterator vecIter = vec.begin(); vecIter != vec.end(); vecIter++ ) {
if ( vecIter == vec.begin() ) {
retour += ( *vecIter );
}
else
{
} else {
retour += "\t" + ( *vecIter );
}
}
return retour;
}
string vectorToString ( vector<char> vec )
{
}
string vectorToString ( vector<char> vec )
{
stringstream retour;
retour.str("");
for ( vector<char>::iterator vecIter = vec.begin();vecIter != vec.end(); vecIter++ )
{
if ( vecIter == vec.begin() )
{
for ( vector<char>::iterator vecIter = vec.begin(); vecIter != vec.end(); vecIter++ ) {
if ( vecIter == vec.begin() ) {
retour << ( *vecIter );
}
else
{
} else {
retour << "\t" << ( *vecIter );
}
}
return retour.str();
}
string vectorToString ( vector<int> vec )
{
}
string vectorToString ( vector<int> vec )
{
stringstream retour;
retour.str("");
for ( vector<int>::iterator vecIter = vec.begin();vecIter != vec.end(); vecIter++ )
{
if ( vecIter == vec.begin() )
{
for ( vector<int>::iterator vecIter = vec.begin(); vecIter != vec.end(); vecIter++ ) {
if ( vecIter == vec.begin() ) {
retour << ( *vecIter );
}
else
{
} else {
retour << "\t" << ( *vecIter );
}
}
return retour.str();
}
}
string vectorToString ( vector< string > vec, string s )
{
string vectorToString ( vector< string > vec, string s )
{
string retour ( "" );
for ( vector<string>::iterator vecIter = vec.begin();vecIter != vec.end(); vecIter++ )
{
if ( vecIter == vec.begin() )
{
for ( vector<string>::iterator vecIter = vec.begin(); vecIter != vec.end(); vecIter++ ) {
if ( vecIter == vec.begin() ) {
retour += ( *vecIter );
}
else
{
} else {
retour += s + ( *vecIter );
}
}
return retour;
}
}
string vectorToString ( vector< char > vec, string s )
{
string vectorToString ( vector< char > vec, string s )
{
stringstream retour;
retour.str("");
for ( vector<char>::iterator vecIter = vec.begin();vecIter != vec.end(); vecIter++ )
{
if ( vecIter == vec.begin() )
{
for ( vector<char>::iterator vecIter = vec.begin(); vecIter != vec.end(); vecIter++ ) {
if ( vecIter == vec.begin() ) {
retour << ( *vecIter );
}
else
{
} else {
retour << s << ( *vecIter );
}
}
return retour.str();
}
}
string vectorToString ( vector< int > vec, string s )
{
string vectorToString ( vector< int > vec, string s )
{
stringstream retour;
retour.str("");
for ( vector<int>::iterator vecIter = vec.begin();vecIter != vec.end(); vecIter++ )
{
if ( vecIter == vec.begin() )
{
for ( vector<int>::iterator vecIter = vec.begin(); vecIter != vec.end(); vecIter++ ) {
if ( vecIter == vec.begin() ) {
retour << ( *vecIter );
}
else
{
} else {
retour << s << ( *vecIter );
}
}
return retour.str();
}
}
string vectorToString ( vector< bool > vec, string s )
{
string vectorToString ( vector< bool > vec, string s )
{
stringstream retour;
retour.str("");
for ( vector<bool>::iterator vecIter = vec.begin();vecIter != vec.end(); vecIter++ )
{
if ( vecIter == vec.begin() )
{
for ( vector<bool>::iterator vecIter = vec.begin(); vecIter != vec.end(); vecIter++ ) {
if ( vecIter == vec.begin() ) {
retour << ( *vecIter );
}
else
{
} else {
retour << s << ( *vecIter );
}
}
return retour.str();
}
string vectorToString ( char* vec, string s , int taille)
{
}
string vectorToString ( char* vec, string s , int taille)
{
stringstream retour;
retour.str("");
int l_i;
for ( l_i=0; l_i < taille ; l_i++)
{
if ( l_i == 0 )
{
for ( l_i=0; l_i < taille ; l_i++) {
if ( l_i == 0 ) {
retour << vec[l_i];
}
else
{
} else {
retour << s << vec[l_i];
}
}
return retour.str();
}
}
string vectorToString ( int* vec, string s , int taille)
{
string vectorToString ( int* vec, string s , int taille)
{
stringstream retour;
retour.str("");
int l_i;
for ( l_i=0; l_i < taille ; l_i++)
{
if ( l_i == 0 )
{
for ( l_i=0; l_i < taille ; l_i++) {
if ( l_i == 0 ) {
retour << vec[l_i];
}
else
{
} else {
retour << s << vec[l_i];
}
}
return retour.str();
}
}
string vectorToString ( bool* vec, string s , int taille)
{
string vectorToString ( bool* vec, string s , int taille)
{
stringstream retour;
retour.str("");
int l_i;
for ( l_i=0; l_i < taille ; l_i++)
{
if ( l_i == 0 )
{
for ( l_i=0; l_i < taille ; l_i++) {
if ( l_i == 0 ) {
retour << vec[l_i];
}
else
{
} else {
retour << s << vec[l_i];
}
}
return retour.str();
}
}
vector<string> subVector ( vector<string> vec, int start, int end )
{
vector<string> subVector ( vector<string> vec, int start, int end )
{
vector<string> retour;
if ( start > end )
{
if ( start > end ) {
cerr << "ERREUR : TERcalc::subVector : end > start" << endl;
exit ( 0 );
}
for ( int i = start; ( ( i < end ) && ( i < ( int ) vec.size() ) ); i++ )
{
for ( int i = start; ( ( i < end ) && ( i < ( int ) vec.size() ) ); i++ ) {
retour.push_back ( vec.at ( i ) );
}
return retour;
}
}
vector<int> subVector ( vector<int> vec, int start, int end )
{
vector<int> subVector ( vector<int> vec, int start, int end )
{
vector<int> retour;
if ( start > end )
{
if ( start > end ) {
cerr << "ERREUR : TERcalc::subVector : end > start" << endl;
exit ( 0 );
}
for ( int i = start; ( ( i < end ) && ( i < ( int ) vec.size() ) ); i++ )
{
for ( int i = start; ( ( i < end ) && ( i < ( int ) vec.size() ) ); i++ ) {
retour.push_back ( vec.at ( i ) );
}
return retour;
}
}
vector<float> subVector ( vector<float> vec, int start, int end )
{
vector<float> subVector ( vector<float> vec, int start, int end )
{
vector<float> retour;
if ( start > end )
{
if ( start > end ) {
cerr << "ERREUR : TERcalc::subVector : end > start" << endl;
exit ( 0 );
}
for ( int i = start; ( ( i < end ) && ( i < ( int ) vec.size() ) ); i++ )
{
for ( int i = start; ( ( i < end ) && ( i < ( int ) vec.size() ) ); i++ ) {
retour.push_back ( vec.at ( i ) );
}
return retour;
}
}
vector<string> copyVector ( vector<string> vec )
{
vector<string> copyVector ( vector<string> vec )
{
vector<string> retour;
for ( int i = 0; i < ( int ) vec.size(); i++ )
{
for ( int i = 0; i < ( int ) vec.size(); i++ ) {
retour.push_back ( vec.at ( i ) );
}
return retour;
}
vector<int> copyVector ( vector<int> vec )
{
}
vector<int> copyVector ( vector<int> vec )
{
vector<int> retour;
for ( int i = 0; i < ( int ) vec.size(); i++ )
{
for ( int i = 0; i < ( int ) vec.size(); i++ ) {
retour.push_back ( vec.at ( i ) );
}
return retour;
}
vector<float> copyVector ( vector<float> vec )
{
}
vector<float> copyVector ( vector<float> vec )
{
vector<float> retour;
for ( int i = 0; i < ( int ) vec.size(); i++ )
{
for ( int i = 0; i < ( int ) vec.size(); i++ ) {
retour.push_back ( vec.at ( i ) );
}
return retour;
}
vector<string> stringToVector ( string s, string tok )
{
}
vector<string> stringToVector ( string s, string tok )
{
vector<string> to_return;
string to_push ( "" );
bool pushed = false;
string::iterator sIt;
for ( sIt = s.begin(); sIt < s.end(); sIt++ )
{
for ( sIt = s.begin(); sIt < s.end(); sIt++ ) {
pushed = false;
for ( string::iterator sTok = tok.begin(); sTok < tok.end(); sTok++ )
{
if ( ( *sIt ) == ( *sTok ) )
{
for ( string::iterator sTok = tok.begin(); sTok < tok.end(); sTok++ ) {
if ( ( *sIt ) == ( *sTok ) ) {
to_return.push_back ( to_push );
to_push = "";
pushed = true;
}
}
if ( !pushed )
{
if ( !pushed ) {
to_push.push_back ( ( *sIt ) );
}
}
to_return.push_back ( to_push );
return to_return;
}
vector<int> stringToVectorInt ( string s, string tok )
{
}
vector<int> stringToVectorInt ( string s, string tok )
{
vector<int> to_return;
string to_push ( "" );
bool pushed = false;
string::iterator sIt;
for ( sIt = s.begin(); sIt < s.end(); sIt++ )
{
for ( sIt = s.begin(); sIt < s.end(); sIt++ ) {
pushed = false;
for ( string::iterator sTok = tok.begin(); sTok < tok.end(); sTok++ )
{
if ( ( *sIt ) == ( *sTok ) )
{
if ( ( int ) to_push.length() > 0 )
{
for ( string::iterator sTok = tok.begin(); sTok < tok.end(); sTok++ ) {
if ( ( *sIt ) == ( *sTok ) ) {
if ( ( int ) to_push.length() > 0 ) {
to_return.push_back ( atoi ( to_push.c_str() ) );
}
to_push = "";
pushed = true;
}
}
if ( !pushed )
{
if ( !pushed ) {
to_push.push_back ( ( *sIt ) );
}
}
if ( ( int ) to_push.length() > 0 )
{
if ( ( int ) to_push.length() > 0 ) {
to_return.push_back ( atoi ( to_push.c_str() ) );
}
return to_return;
}
vector<float> stringToVectorFloat ( string s, string tok )
{
}
vector<float> stringToVectorFloat ( string s, string tok )
{
vector<float> to_return;
string to_push ( "" );
bool pushed = false;
string::iterator sIt;
for ( sIt = s.begin(); sIt < s.end(); sIt++ )
{
for ( sIt = s.begin(); sIt < s.end(); sIt++ ) {
pushed = false;
for ( string::iterator sTok = tok.begin(); sTok < tok.end(); sTok++ )
{
if ( ( *sIt ) == ( *sTok ) )
{
if ( ( int ) to_push.length() > 0 )
{
for ( string::iterator sTok = tok.begin(); sTok < tok.end(); sTok++ ) {
if ( ( *sIt ) == ( *sTok ) ) {
if ( ( int ) to_push.length() > 0 ) {
to_return.push_back ( atof ( to_push.c_str() ) );
}
to_push = "";
pushed = true;
}
}
if ( !pushed )
{
if ( !pushed ) {
to_push.push_back ( ( *sIt ) );
}
}
if ( ( int ) to_push.length() > 0 )
{
if ( ( int ) to_push.length() > 0 ) {
to_return.push_back ( atoi ( to_push.c_str() ) );
}
return to_return;
}
}
string lowerCase ( string str )
{
for ( int i = 0;i < ( int ) str.size();i++ )
{
if ( ( str[i] >= 0x41 ) && ( str[i] <= 0x5A ) )
{
string lowerCase ( string str )
{
for ( int i = 0; i < ( int ) str.size(); i++ ) {
if ( ( str[i] >= 0x41 ) && ( str[i] <= 0x5A ) ) {
str[i] = str[i] + 0x20;
}
}
return str;
}
string removePunctTercom ( string str )
{
}
string removePunctTercom ( string str )
{
string str_mod = str;
sregex rex;
string replace;
@ -452,9 +385,9 @@ namespace Tools
str_mod = regex_replace ( str_mod, rex, replace );
return str_mod;
}
string removePunct ( string str )
{
}
string removePunct ( string str )
{
string str_mod = str;
sregex rex;
string replace;
@ -525,9 +458,9 @@ namespace Tools
str_mod = regex_replace ( str_mod, rex, replace );
return str_mod;
}
string tokenizePunct ( string str )
{
}
string tokenizePunct ( string str )
{
string str_mod = str;
sregex rex = sregex::compile ( "(([^0-9])([\\,])([^0-9]))" );
string replace ( "$2 $3 $4" );
@ -674,10 +607,10 @@ namespace Tools
str_mod = regex_replace ( str_mod, rex, replace );
return str_mod;
}
}
string normalizeStd ( string str )
{
string normalizeStd ( string str )
{
string str_mod = str;
sregex rex = sregex::compile ( "(<skipped>)" );
string replace ( "" );
@ -708,10 +641,10 @@ namespace Tools
str_mod = regex_replace ( str_mod, rex, replace );
return str_mod;
}
}
param copyParam ( param p )
{
param copyParam ( param p )
{
param to_return;
to_return.caseOn = p.caseOn;
to_return.noPunct = p.noPunct;
@ -728,9 +661,9 @@ namespace Tools
to_return.printAlignments = p.printAlignments;
to_return.WER=p.WER;
return to_return;
}
string printParams ( param p )
{
}
string printParams ( param p )
{
stringstream s;
s << "caseOn = " << p.caseOn << endl;
s << "noPunct = " << p.noPunct << endl;
@ -746,27 +679,23 @@ namespace Tools
s << "tercomLike = " << p.tercomLike << endl;
return s.str();
}
string join ( string delim, vector<string> arr )
{
}
string join ( string delim, vector<string> arr )
{
if ( ( int ) arr.size() == 0 ) return "";
// if ((int)delim.compare("") == 0) delim = new String("");
// String s = new String("");
stringstream s;
s.str ( "" );
for ( int i = 0; i < ( int ) arr.size(); i++ )
{
if ( i == 0 )
{
for ( int i = 0; i < ( int ) arr.size(); i++ ) {
if ( i == 0 ) {
s << arr.at ( i );
}
else
{
} else {
s << delim << arr.at ( i );
}
}
return s.str();
// return "";
}
}
}

77
mert/TER/tools.h
View File

@ -35,18 +35,17 @@ using namespace std;
namespace Tools
{
typedef vector<double> vecDouble;
typedef vector<char> vecChar;
typedef vector<int> vecInt;
typedef vector<float> vecFloat;
typedef vector<size_t> vecSize_t;
typedef vector<string> vecString;
typedef vector<string> alignmentElement;
typedef vector<alignmentElement> WERalignment;
typedef vector<double> vecDouble;
typedef vector<char> vecChar;
typedef vector<int> vecInt;
typedef vector<float> vecFloat;
typedef vector<size_t> vecSize_t;
typedef vector<string> vecString;
typedef vector<string> alignmentElement;
typedef vector<alignmentElement> WERalignment;
struct param
{
struct param {
bool debugMode;
string referenceFile; // path to the resources
string hypothesisFile; // path to the configuration files
@ -68,35 +67,35 @@ struct param
// private:
// public:
string vectorToString ( vector<string> vec );
string vectorToString ( vector<char> vec );
string vectorToString ( vector<int> vec );
string vectorToString ( vector<string> vec, string s );
string vectorToString ( vector<char> vec, string s );
string vectorToString ( vector<int> vec, string s );
string vectorToString ( vector<bool> vec, string s );
string vectorToString ( char* vec, string s, int taille );
string vectorToString ( int* vec, string s , int taille );
string vectorToString ( bool* vec, string s , int taille );
vector<string> subVector ( vector<string> vec, int start, int end );
vector<int> subVector ( vector<int> vec, int start, int end );
vector<float> subVector ( vector<float> vec, int start, int end );
vector<string> copyVector ( vector<string> vec );
vector<int> copyVector ( vector<int> vec );
vector<float> copyVector ( vector<float> vec );
vector<string> stringToVector ( string s, string tok );
vector<string> stringToVector ( char s, string tok );
vector<string> stringToVector ( int s, string tok );
vector<int> stringToVectorInt ( string s, string tok );
vector<float> stringToVectorFloat ( string s, string tok );
string lowerCase(string str);
string removePunct(string str);
string tokenizePunct(string str);
string removePunctTercom(string str);
string normalizeStd(string str);
string printParams(param p);
string join ( string delim, vector<string> arr );
string vectorToString ( vector<string> vec );
string vectorToString ( vector<char> vec );
string vectorToString ( vector<int> vec );
string vectorToString ( vector<string> vec, string s );
string vectorToString ( vector<char> vec, string s );
string vectorToString ( vector<int> vec, string s );
string vectorToString ( vector<bool> vec, string s );
string vectorToString ( char* vec, string s, int taille );
string vectorToString ( int* vec, string s , int taille );
string vectorToString ( bool* vec, string s , int taille );
vector<string> subVector ( vector<string> vec, int start, int end );
vector<int> subVector ( vector<int> vec, int start, int end );
vector<float> subVector ( vector<float> vec, int start, int end );
vector<string> copyVector ( vector<string> vec );
vector<int> copyVector ( vector<int> vec );
vector<float> copyVector ( vector<float> vec );
vector<string> stringToVector ( string s, string tok );
vector<string> stringToVector ( char s, string tok );
vector<string> stringToVector ( int s, string tok );
vector<int> stringToVectorInt ( string s, string tok );
vector<float> stringToVectorFloat ( string s, string tok );
string lowerCase(string str);
string removePunct(string str);
string tokenizePunct(string str);
string removePunctTercom(string str);
string normalizeStd(string str);
string printParams(param p);
string join ( string delim, vector<string> arr );
// };
param copyParam(param p);
param copyParam(param p);
}
#endif

9
mert/evaluator.cpp
View File

@ -43,7 +43,8 @@ private:
};
// load hypothesis from candidate output
vector<ScoreStats> EvaluatorUtil::loadCand(const string& candFile) {
vector<ScoreStats> EvaluatorUtil::loadCand(const string& candFile)
{
ifstream cand(candFile.c_str());
if (!cand.good()) throw runtime_error("Error opening candidate file");
@ -61,7 +62,8 @@ vector<ScoreStats> EvaluatorUtil::loadCand(const string& candFile) {
}
// load 1-best hypothesis from n-best file (useful if relying on alignment/tree information)
vector<ScoreStats> EvaluatorUtil::loadNBest(const string& nBestFile) {
vector<ScoreStats> EvaluatorUtil::loadNBest(const string& nBestFile)
{
vector<ScoreStats> entries;
Data data(g_scorer);
@ -81,8 +83,7 @@ void EvaluatorUtil::evaluate(const string& candFile, int bootstrap, bool nbest_i
if (nbest_input) {
entries = loadNBest(candFile);
}
else {
} else {
entries = loadCand(candFile);
}

2
mert/kbmira.cpp
View File

@ -246,7 +246,7 @@ int main(int argc, char** argv)
int iNumUpdates = 0;
ValType totalLoss = 0.0;
size_t sentenceIndex = 0;
for(decoder->reset();!decoder->finished(); decoder->next()) {
for(decoder->reset(); !decoder->finished(); decoder->next()) {
HopeFearData hfd;
decoder->HopeFear(bg,wv,&hfd);

3
misc/CreateProbingPT.cpp
View File

@ -3,7 +3,8 @@
int main(int argc, char* argv[]){
int main(int argc, char* argv[])
{
const char * is_reordering = "false";

9
misc/QueryProbingPT.cpp
View File

@ -26,7 +26,8 @@
#include <unistd.h>
#include <fcntl.h>
int main(int argc, char* argv[]) {
int main(int argc, char* argv[])
{
if (argc != 2) {
// Tell the user how to run the program
std::cerr << "Usage: " << argv[0] << " path_to_directory" << std::endl;
@ -37,12 +38,12 @@ int main(int argc, char* argv[]) {
//Interactive search
std::cout << "Please enter a string to be searched, or exit to exit." << std::endl;
while (true){
while (true) {
std::string cinstr = "";
getline(std::cin, cinstr);
if (cinstr == "exit"){
if (cinstr == "exit") {
break;
}else{
} else {
//Actual lookup
std::pair<bool, std::vector<target_text> > query_result;
query_result = queries.query(StringPiece(cinstr));

6
misc/prunePhraseTable.cpp
View File

@ -53,13 +53,15 @@ using namespace std;
namespace po = boost::program_options;
typedef multimap<float,string> Lines;
static void usage(const po::options_description& desc, char** argv) {
static void usage(const po::options_description& desc, char** argv)
{
cerr << "Usage: " + string(argv[0]) + " [options] input-file output-file" << endl;
cerr << desc << endl;
}
//Find top n translations of source, and send them to output
static void outputTopN(Lines lines, size_t maxPhrases, ostream& out) {
static void outputTopN(Lines lines, size_t maxPhrases, ostream& out)
{
size_t count = 0;
for (Lines::const_reverse_iterator i = lines.rbegin(); i != lines.rend(); ++i) {
out << i->second << endl;

6
moses-cmd/Main.cpp
View File

@ -171,16 +171,14 @@ int main(int argc, char** argv)
<< "missing update data for simulated post-editing.");
UTIL_THROW_IF2(!getline(*ioWrapper->spe_aln,aln), "[" << HERE << "] "
<< "missing update data for simulated post-editing.");
BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl())
{
BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl()) {
Mmsapt* sapt = dynamic_cast<Mmsapt*>(pd);
if (sapt) sapt->add(src,trg,aln);
VERBOSE(1,"[" << HERE << " added src] " << src << endl);
VERBOSE(1,"[" << HERE << " added trg] " << trg << endl);
VERBOSE(1,"[" << HERE << " added aln] " << aln << endl);
}
}
else
} else
#endif
pool.Submit(task);
#else

7
moses/AlignmentInfoCollection.h
View File

@ -46,14 +46,13 @@ public:
* contains such an object then returns a pointer to it; otherwise a new
* one is inserted.
*/
private:
private:
const AlignmentInfo* Add(AlignmentInfo const& ainfo);
public:
public:
template<typename ALNREP>
AlignmentInfo const *
Add(ALNREP const & aln)
{
Add(ALNREP const & aln) {
return this->Add(AlignmentInfo(aln));
}

11
moses/BaseManager.h
View File

@ -17,8 +17,8 @@ protected:
const InputType &m_source; /**< source sentence to be translated */
BaseManager(const InputType &source)
:m_source(source)
{}
:m_source(source) {
}
// output
typedef std::vector<std::pair<Moses::Word, Moses::WordsRange> > ApplicationContext;
@ -32,8 +32,7 @@ protected:
const ApplicationContext &context) const;
template <class T>
void ShiftOffsets(std::vector<T> &offsets, T shift) const
{
void ShiftOffsets(std::vector<T> &offsets, T shift) const {
T currPos = shift;
for (size_t i = 0; i < offsets.size(); ++i) {
if (offsets[i] == 0) {
@ -46,8 +45,8 @@ protected:
}
public:
virtual ~BaseManager()
{}
virtual ~BaseManager() {
}
//! the input sentence being decoded
const InputType& GetSource() const {

3
moses/ChartCellLabelSet.h
View File

@ -124,8 +124,7 @@ public:
const ChartCellLabel *Find(size_t idx) const {
try {
return m_map.at(idx);
}
catch (const std::out_of_range& oor) {
} catch (const std::out_of_range& oor) {
return NULL;
}
}

112
moses/ChartHypothesis.cpp
View File

@ -61,8 +61,7 @@ ChartHypothesis::ChartHypothesis(const ChartTranslationOptions &transOpt,
const std::vector<HypothesisDimension> &childEntries = item.GetHypothesisDimensions();
m_prevHypos.reserve(childEntries.size());
std::vector<HypothesisDimension>::const_iterator iter;
for (iter = childEntries.begin(); iter != childEntries.end(); ++iter)
{
for (iter = childEntries.begin(); iter != childEntries.end(); ++iter) {
m_prevHypos.push_back(iter->GetHypothesis());
}
}
@ -85,17 +84,14 @@ ChartHypothesis::ChartHypothesis(const ChartHypothesis &pred,
ChartHypothesis::~ChartHypothesis()
{
// delete feature function states
for (unsigned i = 0; i < m_ffStates.size(); ++i)
{
for (unsigned i = 0; i < m_ffStates.size(); ++i) {
delete m_ffStates[i];
}
// delete hypotheses that are not in the chart (recombined away)
if (m_arcList)
{
if (m_arcList) {
ChartArcList::iterator iter;
for (iter = m_arcList->begin() ; iter != m_arcList->end() ; ++iter)
{
for (iter = m_arcList->begin() ; iter != m_arcList->end() ; ++iter) {
ChartHypothesis *hypo = *iter;
Delete(hypo);
}
@ -112,25 +108,19 @@ void ChartHypothesis::GetOutputPhrase(Phrase &outPhrase) const
{
FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor();
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos)
{
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos) {
const Word &word = GetCurrTargetPhrase().GetWord(pos);
if (word.IsNonTerminal())
{
if (word.IsNonTerminal()) {
// non-term. fill out with prev hypo
size_t nonTermInd = GetCurrTargetPhrase().GetAlignNonTerm().GetNonTermIndexMap()[pos];
const ChartHypothesis *prevHypo = m_prevHypos[nonTermInd];
prevHypo->GetOutputPhrase(outPhrase);
}
else
{
} else {
outPhrase.AddWord(word);
if (placeholderFactor != NOT_FOUND)
{
if (placeholderFactor != NOT_FOUND) {
std::set<size_t> sourcePosSet = GetCurrTargetPhrase().GetAlignTerm().GetAlignmentsForTarget(pos);
if (sourcePosSet.size() == 1)
{
if (sourcePosSet.size() == 1) {
const std::vector<const Word*> *ruleSourceFromInputPath = GetTranslationOption().GetSourceRuleFromInputPath();
UTIL_THROW_IF2(ruleSourceFromInputPath == NULL,
"No source rule");
@ -140,8 +130,7 @@ void ChartHypothesis::GetOutputPhrase(Phrase &outPhrase) const
UTIL_THROW_IF2(sourceWord == NULL,
"No source word");
const Factor *factor = sourceWord->GetFactor(placeholderFactor);
if (factor)
{
if (factor) {
outPhrase.Back()[0] = factor;
}
}
@ -165,33 +154,24 @@ void ChartHypothesis::GetOutputPhrase(size_t leftRightMost, size_t numWords, Phr
const TargetPhrase &tp = GetCurrTargetPhrase();
size_t targetSize = tp.GetSize();
for (size_t i = 0; i < targetSize; ++i)
{
for (size_t i = 0; i < targetSize; ++i) {
size_t pos;
if (leftRightMost == 1)
{
if (leftRightMost == 1) {
pos = i;
}
else if (leftRightMost == 2)
{
} else if (leftRightMost == 2) {
pos = targetSize - i - 1;
}
else
{
} else {
abort();
}
const Word &word = tp.GetWord(pos);
if (word.IsNonTerminal())
{
if (word.IsNonTerminal()) {
// non-term. fill out with prev hypo
size_t nonTermInd = tp.GetAlignNonTerm().GetNonTermIndexMap()[pos];
const ChartHypothesis *prevHypo = m_prevHypos[nonTermInd];
prevHypo->GetOutputPhrase(outPhrase);
}
else
{
} else {
outPhrase.AddWord(word);
}
@ -236,20 +216,16 @@ void ChartHypothesis::EvaluateWhenApplied()
// cached in the translation option-- there is no principled distinction
const std::vector<const StatelessFeatureFunction*>& sfs =
StatelessFeatureFunction::GetStatelessFeatureFunctions();
for (unsigned i = 0; i < sfs.size(); ++i)
{
if (! staticData.IsFeatureFunctionIgnored( *sfs[i] ))
{
for (unsigned i = 0; i < sfs.size(); ++i) {
if (! staticData.IsFeatureFunctionIgnored( *sfs[i] )) {
sfs[i]->EvaluateWhenApplied(*this,&m_currScoreBreakdown);
}
}
const std::vector<const StatefulFeatureFunction*>& ffs =
StatefulFeatureFunction::GetStatefulFeatureFunctions();
for (unsigned i = 0; i < ffs.size(); ++i)
{
if (! staticData.IsFeatureFunctionIgnored( *ffs[i] ))
{
for (unsigned i = 0; i < ffs.size(); ++i) {
if (! staticData.IsFeatureFunctionIgnored( *ffs[i] )) {
m_ffStates[i] = ffs[i]->EvaluateWhenApplied(*this,i,&m_currScoreBreakdown);
}
}
@ -267,31 +243,25 @@ void ChartHypothesis::EvaluateWhenApplied()
void ChartHypothesis::AddArc(ChartHypothesis *loserHypo)
{
if (!m_arcList)
{
if (loserHypo->m_arcList)
{ // we don't have an arcList, but loser does
if (!m_arcList) {
if (loserHypo->m_arcList) {
// we don't have an arcList, but loser does
this->m_arcList = loserHypo->m_arcList; // take ownership, we'll delete
loserHypo->m_arcList = 0; // prevent a double deletion
}
else
{
} else {
this->m_arcList = new ChartArcList();
}
}
else
{
if (loserHypo->m_arcList)
{ // both have an arc list: merge. delete loser
} else {
if (loserHypo->m_arcList) {
// both have an arc list: merge. delete loser
size_t my_size = m_arcList->size();
size_t add_size = loserHypo->m_arcList->size();
this->m_arcList->resize(my_size + add_size, 0);
std::memcpy(&(*m_arcList)[0] + my_size, &(*loserHypo->m_arcList)[0], add_size * sizeof(ChartHypothesis *));
delete loserHypo->m_arcList;
loserHypo->m_arcList = 0;
}
else
{ // loserHypo doesn't have any arcs
} else {
// loserHypo doesn't have any arcs
// DO NOTHING
}
}
@ -299,10 +269,8 @@ void ChartHypothesis::AddArc(ChartHypothesis *loserHypo)
}
// sorting helper
struct CompareChartHypothesisTotalScore
{
bool operator()(const ChartHypothesis* hypo1, const ChartHypothesis* hypo2) const
{
struct CompareChartHypothesisTotalScore {
bool operator()(const ChartHypothesis* hypo1, const ChartHypothesis* hypo2) const {
return hypo1->GetTotalScore() > hypo2->GetTotalScore();
}
};
@ -322,8 +290,7 @@ void ChartHypothesis::CleanupArcList()
size_t nBestSize = staticData.GetNBestSize();
bool distinctNBest = staticData.GetDistinctNBest() || staticData.UseMBR() || staticData.GetOutputSearchGraph() || staticData.GetOutputSearchGraphHypergraph();
if (!distinctNBest && m_arcList->size() > nBestSize)
{
if (!distinctNBest && m_arcList->size() > nBestSize) {
// prune arc list only if there too many arcs
NTH_ELEMENT4(m_arcList->begin()
, m_arcList->begin() + nBestSize - 1
@ -332,8 +299,7 @@ void ChartHypothesis::CleanupArcList()
// delete bad ones
ChartArcList::iterator iter;
for (iter = m_arcList->begin() + nBestSize ; iter != m_arcList->end() ; ++iter)
{
for (iter = m_arcList->begin() + nBestSize ; iter != m_arcList->end() ; ++iter) {
ChartHypothesis *arc = *iter;
ChartHypothesis::Delete(arc);
}
@ -343,8 +309,7 @@ void ChartHypothesis::CleanupArcList()
// set all arc's main hypo variable to this hypo
ChartArcList::iterator iter = m_arcList->begin();
for (; iter != m_arcList->end() ; ++iter)
{
for (; iter != m_arcList->end() ; ++iter) {
ChartHypothesis *arc = *iter;
arc->SetWinningHypo(this);
}
@ -367,13 +332,11 @@ std::ostream& operator<<(std::ostream& out, const ChartHypothesis& hypo)
// recombination
if (hypo.GetWinningHypothesis() != NULL &&
hypo.GetWinningHypothesis() != &hypo)
{
hypo.GetWinningHypothesis() != &hypo) {
out << "->" << hypo.GetWinningHypothesis()->GetId();
}
if (StaticData::Instance().GetIncludeLHSInSearchGraph())
{
if (StaticData::Instance().GetIncludeLHSInSearchGraph()) {
out << " " << hypo.GetTargetLHS() << "=>";
}
out << " " << hypo.GetCurrTargetPhrase()
@ -381,8 +344,7 @@ std::ostream& operator<<(std::ostream& out, const ChartHypothesis& hypo)
<< " " << hypo.GetCurrSourceRange();
HypoList::const_iterator iter;
for (iter = hypo.GetPrevHypos().begin(); iter != hypo.GetPrevHypos().end(); ++iter)
{
for (iter = hypo.GetPrevHypos().begin(); iter != hypo.GetPrevHypos().end(); ++iter) {
const ChartHypothesis &prevHypo = **iter;
out << " " << prevHypo.GetId();
}

66
moses/ChartHypothesis.h
View File

@ -82,21 +82,18 @@ protected:
public:
#ifdef USE_HYPO_POOL
void *operator new(size_t /* num_bytes */)
{
void *operator new(size_t /* num_bytes */) {
void *ptr = s_objectPool.getPtr();
return ptr;
}
//! delete \param hypo. Works with object pool too
static void Delete(ChartHypothesis *hypo)
{
static void Delete(ChartHypothesis *hypo) {
s_objectPool.freeObject(hypo);
}
#else
//! delete \param hypo. Works with object pool too
static void Delete(ChartHypothesis *hypo)
{
static void Delete(ChartHypothesis *hypo) {
delete hypo;
}
#endif
@ -109,43 +106,36 @@ public:
~ChartHypothesis();
unsigned GetId() const
{
unsigned GetId() const {
return m_id;
}
const ChartTranslationOption &GetTranslationOption() const
{
const ChartTranslationOption &GetTranslationOption() const {
return *m_transOpt;
}
//! Get the rule that created this hypothesis
const TargetPhrase &GetCurrTargetPhrase() const
{
const TargetPhrase &GetCurrTargetPhrase() const {
return m_transOpt->GetPhrase();
}
//! the source range that this hypothesis spans
const WordsRange &GetCurrSourceRange() const
{
const WordsRange &GetCurrSourceRange() const {
return m_currSourceWordsRange;
}
//! the arc list when creating n-best lists
inline const ChartArcList* GetArcList() const
{
inline const ChartArcList* GetArcList() const {
return m_arcList;
}
//! the feature function states for a particular feature \param featureID
inline const FFState* GetFFState( size_t featureID ) const
{
inline const FFState* GetFFState( size_t featureID ) const {
return m_ffStates[ featureID ];
}
//! reference back to the manager
inline const ChartManager& GetManager() const
{
inline const ChartManager& GetManager() const {
return m_manager;
}
@ -165,21 +155,17 @@ public:
void SetWinningHypo(const ChartHypothesis *hypo);
//! get the unweighted score for each feature function
const ScoreComponentCollection &GetScoreBreakdown() const
{
const ScoreComponentCollection &GetScoreBreakdown() const {
// Note: never call this method before m_currScoreBreakdown is fully computed
if (!m_scoreBreakdown.get())
{
if (!m_scoreBreakdown.get()) {
m_scoreBreakdown.reset(new ScoreComponentCollection());
// score breakdown from current translation rule
if (m_transOpt)
{
if (m_transOpt) {
m_scoreBreakdown->PlusEquals(GetTranslationOption().GetScores());
}
m_scoreBreakdown->PlusEquals(m_currScoreBreakdown);
// score breakdowns from prev hypos
for (std::vector<const ChartHypothesis*>::const_iterator iter = m_prevHypos.begin(); iter != m_prevHypos.end(); ++iter)
{
for (std::vector<const ChartHypothesis*>::const_iterator iter = m_prevHypos.begin(); iter != m_prevHypos.end(); ++iter) {
const ChartHypothesis &prevHypo = **iter;
m_scoreBreakdown->PlusEquals(prevHypo.GetScoreBreakdown());
}
@ -188,15 +174,12 @@ public:
}
//! get the unweighted score delta for each feature function
const ScoreComponentCollection &GetDeltaScoreBreakdown() const
{
const ScoreComponentCollection &GetDeltaScoreBreakdown() const {
// Note: never call this method before m_currScoreBreakdown is fully computed
if (!m_deltaScoreBreakdown.get())
{
if (!m_deltaScoreBreakdown.get()) {
m_deltaScoreBreakdown.reset(new ScoreComponentCollection());
// score breakdown from current translation rule
if (m_transOpt)
{
if (m_transOpt) {
m_deltaScoreBreakdown->PlusEquals(GetTranslationOption().GetScores());
}
m_deltaScoreBreakdown->PlusEquals(m_currScoreBreakdown);
@ -206,33 +189,28 @@ public:
}
//! Get the weighted total score
float GetTotalScore() const
{
float GetTotalScore() const {
// scores from current translation rule. eg. translation models & word penalty
return m_totalScore;
}
//! vector of previous hypotheses this hypo is built on
const std::vector<const ChartHypothesis*> &GetPrevHypos() const
{
const std::vector<const ChartHypothesis*> &GetPrevHypos() const {
return m_prevHypos;
}
//! get a particular previous hypos
const ChartHypothesis* GetPrevHypo(size_t pos) const
{
const ChartHypothesis* GetPrevHypo(size_t pos) const {
return m_prevHypos[pos];
}
//! get the constituency label that covers this hypo
const Word &GetTargetLHS() const
{
const Word &GetTargetLHS() const {
return GetCurrTargetPhrase().GetTargetLHS();
}
//! get the best hypo in the arc list when doing n-best list creation. It's either this hypothesis, or the best hypo is this hypo is in the arc list
const ChartHypothesis* GetWinningHypothesis() const
{
const ChartHypothesis* GetWinningHypothesis() const {
return m_winningHypo;
}

3
moses/ChartKBestExtractor.cpp
View File

@ -169,8 +169,7 @@ TreePointer ChartKBestExtractor::GetOutputTree(const Derivation &d)
mytree->Combine(previous_trees);
return mytree;
}
else {
} else {
UTIL_THROW2("Error: TreeStructureFeature active, but no internal tree structure found");
}
}

6
moses/ChartManager.cpp
View File

@ -290,12 +290,14 @@ void ChartManager::FindReachableHypotheses(
}
}
void ChartManager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const {
void ChartManager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const
{
ChartSearchGraphWriterHypergraph writer(&outputSearchGraphStream);
WriteSearchGraph(writer);
}
void ChartManager::OutputSearchGraphMoses(std::ostream &outputSearchGraphStream) const {
void ChartManager::OutputSearchGraphMoses(std::ostream &outputSearchGraphStream) const
{
ChartSearchGraphWriterMoses writer(&outputSearchGraphStream, m_source.GetTranslationId());
WriteSearchGraph(writer);
}

20
moses/ChartManager.h
View File

@ -126,8 +126,8 @@ public:
return m_hypoStackColl;
}
void CalcDecoderStatistics() const
{}
void CalcDecoderStatistics() const {
}
void ResetSentenceStats(const InputType& source) {
m_sentenceStats = std::auto_ptr<SentenceStats>(new SentenceStats(source));
@ -138,22 +138,24 @@ public:
return m_hypothesisId++;
}
const ChartParser &GetParser() const { return m_parser; }
const ChartParser &GetParser() const {
return m_parser;
}
// outputs
void OutputBest(OutputCollector *collector) const;
void OutputNBest(OutputCollector *collector) const;
void OutputLatticeSamples(OutputCollector *collector) const
{}
void OutputLatticeSamples(OutputCollector *collector) const {
}
void OutputAlignment(OutputCollector *collector) const;
void OutputDetailedTranslationReport(OutputCollector *collector) const;
void OutputUnknowns(OutputCollector *collector) const;
void OutputDetailedTreeFragmentsTranslationReport(OutputCollector *collector) const;
void OutputWordGraph(OutputCollector *collector) const
{}
void OutputWordGraph(OutputCollector *collector) const {
}
void OutputSearchGraph(OutputCollector *collector) const;
void OutputSearchGraphSLF() const
{}
void OutputSearchGraphSLF() const {
}
void OutputSearchGraphHypergraph() const;
};

3
moses/ChartTranslationOptions.cpp
View File

@ -72,8 +72,7 @@ void ChartTranslationOptions::EvaluateWithSourceContext(const InputType &input,
if (transOpt->GetScores().GetWeightedScore() == - std::numeric_limits<float>::infinity()) {
++numDiscard;
}
else if (numDiscard) {
} else if (numDiscard) {
m_collection[i - numDiscard] = m_collection[i];
}
}

142
moses/ConfusionNet.cpp
View File

@ -13,7 +13,7 @@
namespace Moses
{
struct CNStats {
struct CNStats {
size_t created,destr,read,colls,words;
CNStats() : created(0),destr(0),read(0),colls(0),words(0) {}
@ -47,23 +47,23 @@ namespace Moses
"\n\n";
}
}
};
};
CNStats stats;
CNStats stats;
size_t
ConfusionNet::
GetColumnIncrement(size_t i, size_t j) const
{
size_t
ConfusionNet::
GetColumnIncrement(size_t i, size_t j) const
{
(void) i;
(void) j;
return 1;
}
}
ConfusionNet::
ConfusionNet()
ConfusionNet::
ConfusionNet()
: InputType()
{
{
stats.createOne();
const StaticData& staticData = StaticData::Instance();
@ -71,29 +71,29 @@ namespace Moses
m_defaultLabelSet.insert(StaticData::Instance().GetInputDefaultNonTerminal());
}
UTIL_THROW_IF2(&InputFeature::Instance() == NULL, "Input feature must be specified");
}
}
ConfusionNet::
~ConfusionNet()
{
ConfusionNet::
~ConfusionNet()
{
stats.destroyOne();
}
}
ConfusionNet::
ConfusionNet(Sentence const& s)
{
ConfusionNet::
ConfusionNet(Sentence const& s)
{
data.resize(s.GetSize());
for(size_t i=0; i<s.GetSize(); ++i) {
ScorePair scorePair;
std::pair<Word, ScorePair > temp = std::make_pair(s.GetWord(i), scorePair);
data[i].push_back(temp);
}
}
}
bool
ConfusionNet::
ReadF(std::istream& in, const std::vector<FactorType>& factorOrder, int format)
{
bool
ConfusionNet::
ReadF(std::istream& in, const std::vector<FactorType>& factorOrder, int format)
{
VERBOSE(2, "read confusion net with format "<<format<<"\n");
switch(format) {
case 0:
@ -105,38 +105,38 @@ namespace Moses
<<"' in ConfusionNet::Read";
}
return false;
}
}
int
ConfusionNet::
Read(std::istream& in,
int
ConfusionNet::
Read(std::istream& in,
const std::vector<FactorType>& factorOrder)
{
{
int rv=ReadF(in,factorOrder,0);
if(rv) stats.collect(*this);
return rv;
}
}
#if 0
// Deprecated due to code duplication;
// use Word::CreateFromString() instead
void
ConfusionNet::
String2Word(const std::string& s,Word& w,
// Deprecated due to code duplication;
// use Word::CreateFromString() instead
void
ConfusionNet::
String2Word(const std::string& s,Word& w,
const std::vector<FactorType>& factorOrder)
{
{
std::vector<std::string> factorStrVector = Tokenize(s, "|");
for(size_t i=0; i<factorOrder.size(); ++i)
w.SetFactor(factorOrder[i],
FactorCollection::Instance().AddFactor
(Input,factorOrder[i], factorStrVector[i]));
}
}
#endif
bool
ConfusionNet::
ReadFormat0(std::istream& in, const std::vector<FactorType>& factorOrder)
{
bool
ConfusionNet::
ReadFormat0(std::istream& in, const std::vector<FactorType>& factorOrder)
{
Clear();
// const StaticData &staticData = StaticData::Instance();
@ -188,12 +188,12 @@ namespace Moses
} else break;
}
return !data.empty();
}
}
bool
ConfusionNet::
ReadFormat1(std::istream& in, const std::vector<FactorType>& factorOrder)
{
bool
ConfusionNet::
ReadFormat1(std::istream& in, const std::vector<FactorType>& factorOrder)
{
Clear();
std::string line;
if(!getline(in,line)) return 0;
@ -223,10 +223,10 @@ namespace Moses
} else return 0;
}
return !data.empty();
}
}
void ConfusionNet::Print(std::ostream& out) const
{
void ConfusionNet::Print(std::ostream& out) const
{
out<<"conf net: "<<data.size()<<"\n";
for(size_t i=0; i<data.size(); ++i) {
out<<i<<" -- ";
@ -254,46 +254,46 @@ namespace Moses
out<<"\n";
}
out<<"\n\n";
}
}
#ifdef _WIN32
#pragma warning(disable:4716)
#endif
Phrase
ConfusionNet::
GetSubString(const WordsRange&) const
{
Phrase
ConfusionNet::
GetSubString(const WordsRange&) const
{
UTIL_THROW2("ERROR: call to ConfusionNet::GetSubString\n");
//return Phrase(Input);
}
}
std::string
ConfusionNet::
GetStringRep(const std::vector<FactorType> /* factorsToPrint */) const //not well defined yet
{
std::string
ConfusionNet::
GetStringRep(const std::vector<FactorType> /* factorsToPrint */) const //not well defined yet
{
TRACE_ERR("ERROR: call to ConfusionNet::GeStringRep\n");
return "";
}
}
#ifdef _WIN32
#pragma warning(disable:4716)
#endif
const Word& ConfusionNet::GetWord(size_t) const
{
const Word& ConfusionNet::GetWord(size_t) const
{
UTIL_THROW2("ERROR: call to ConfusionNet::GetFactorArray\n");
}
}
#ifdef _WIN32
#pragma warning(default:4716)
#endif
std::ostream& operator<<(std::ostream& out,const ConfusionNet& cn)
{
std::ostream& operator<<(std::ostream& out,const ConfusionNet& cn)
{
cn.Print(out);
return out;
}
}
TranslationOptionCollection*
ConfusionNet::
CreateTranslationOptionCollection() const
{
TranslationOptionCollection*
ConfusionNet::
CreateTranslationOptionCollection() const
{
size_t maxNoTransOptPerCoverage
= StaticData::Instance().GetMaxNoTransOptPerCoverage();
float translationOptionThreshold
@ -303,7 +303,7 @@ namespace Moses
(*this, maxNoTransOptPerCoverage, translationOptionThreshold);
assert(rv);
return rv;
}
}
}

4
moses/DecodeGraph.h
View File

@ -49,8 +49,8 @@ public:
DecodeGraph(size_t id)
: m_id(id)
, m_maxChartSpan(NOT_FOUND)
, m_backoff(0)
{}
, m_backoff(0) {
}
// for chart decoding
DecodeGraph(size_t id, size_t maxChartSpan)

12
moses/FF/BleuScoreFeature.h
View File

@ -126,17 +126,17 @@ public:
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
, ScoreComponentCollection &estimatedFutureScore) const {
}
bool Enabled() const {
return m_enabled;

16
moses/FF/ConstrainedDecoding.h
View File

@ -11,8 +11,8 @@ namespace Moses
class ConstrainedDecodingState : public FFState
{
public:
ConstrainedDecodingState()
{}
ConstrainedDecodingState() {
}
ConstrainedDecodingState(const Hypothesis &hypo);
ConstrainedDecodingState(const ChartHypothesis &hypo);
@ -44,20 +44,20 @@ public:
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
, ScoreComponentCollection &estimatedFutureScore) const {
}
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo,

16
moses/FF/ControlRecombination.h
View File

@ -20,8 +20,8 @@ class ControlRecombinationState : public FFState
{
public:
ControlRecombinationState(const ControlRecombination &ff)
:m_ff(ff)
{}
:m_ff(ff) {
}
ControlRecombinationState(const Hypothesis &hypo, const ControlRecombination &ff);
ControlRecombinationState(const ChartHypothesis &hypo, const ControlRecombination &ff);
@ -60,19 +60,19 @@ public:
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
, ScoreComponentCollection &estimatedFutureScore) const {
}
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo,

8
moses/FF/CountNonTerms.cpp
View File

@ -8,10 +8,10 @@ using namespace std;
namespace Moses
{
CountNonTerms::CountNonTerms(const std::string &line)
:StatelessFeatureFunction(line)
,m_all(true)
,m_sourceSyntax(false)
,m_targetSyntax(false)
:StatelessFeatureFunction(line)
,m_all(true)
,m_sourceSyntax(false)
,m_targetSyntax(false)
{
ReadParameters();
}

16
moses/FF/CountNonTerms.h
View File

@ -23,21 +23,21 @@ public:
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void EvaluateWhenApplied(
const ChartHypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void SetParameter(const std::string& key, const std::string& value);

3
moses/FF/CoveredReferenceFeature.cpp
View File

@ -67,7 +67,8 @@ void CoveredReferenceFeature::EvaluateWithSourceContext(const InputType &input
estimatedFutureScore->Assign(this, scores);
}
void CoveredReferenceFeature::Load() {
void CoveredReferenceFeature::Load()
{
InputFileStream refFile(m_path);
std::string line;
const StaticData &staticData = StaticData::Instance();

4
moses/FF/CoveredReferenceFeature.h
View File

@ -63,8 +63,8 @@ public:
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo,

24
moses/FF/DecodeFeature.h
View File

@ -63,30 +63,30 @@ public:
void SetParameter(const std::string& key, const std::string& value);
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void EvaluateWhenApplied(const Syntax::SHyperedge &hyperedge,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
, ScoreComponentCollection &estimatedFutureScore) const {
}
void SetContainer(const DecodeStep *container) {
m_container = container;

12
moses/FF/DistortionScoreProducer.h
View File

@ -52,18 +52,18 @@ public:
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
, ScoreComponentCollection &estimatedFutureScore) const {
}
};
}

24
moses/FF/DynamicCacheBasedLanguageModel.h
View File

@ -93,12 +93,16 @@ public:
}
static const DynamicCacheBasedLanguageModel* Instance(const std::string& name) {
if (s_instance_map.find(name) == s_instance_map.end()){ return NULL; }
if (s_instance_map.find(name) == s_instance_map.end()) {
return NULL;
}
return s_instance_map[name];
}
static DynamicCacheBasedLanguageModel* InstanceNonConst(const std::string& name) {
if (s_instance_map.find(name) == s_instance_map.end()){ return NULL; }
if (s_instance_map.find(name) == s_instance_map.end()) {
return NULL;
}
return s_instance_map[name];
}
@ -135,20 +139,20 @@ public:
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void SetQueryType(size_t type);
void SetScoreType(size_t type);

16
moses/FF/ExternalFeature.h
View File

@ -18,8 +18,8 @@ protected:
public:
ExternalFeatureState(int stateSize)
:m_stateSize(stateSize)
,m_data(NULL)
{}
,m_data(NULL) {
}
ExternalFeatureState(int stateSize, void *data);
~ExternalFeatureState() {
@ -54,19 +54,19 @@ public:
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
, ScoreComponentCollection &estimatedFutureScore) const {
}
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo,

4
moses/FF/FeatureFunction.h
View File

@ -116,8 +116,8 @@ public:
, ScoreComponentCollection &estimatedFutureScore) const = 0;
// override this method if you want to change the input before decoding
virtual void ChangeSource(InputType *&input) const
{}
virtual void ChangeSource(InputType *&input) const {
}
// This method is called once all the translation options are retrieved from the phrase table, and
// just before search.

16
moses/FF/GlobalLexicalModel.h
View File

@ -76,23 +76,23 @@ public:
, ScoreComponentCollection &estimatedFutureScore) const;
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
};

12
moses/FF/GlobalLexicalModelUnlimited.h
View File

@ -95,18 +95,18 @@ public:
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
, ScoreComponentCollection &estimatedFutureScore) const {
}
void AddFeature(ScoreComponentCollection* accumulator,
StringPiece sourceTrigger, StringPiece sourceWord, StringPiece targetTrigger,

20
moses/FF/HyperParameterAsWeight.h
View File

@ -21,31 +21,31 @@ public:
virtual void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
, ScoreComponentCollection &estimatedFutureScore) const {
}
virtual void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
virtual void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
/**
* Same for chart-based features.
**/
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
};

16
moses/FF/InputFeature.h
View File

@ -44,8 +44,8 @@ public:
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
, ScoreComponentCollection &estimatedFutureScore) const {
}
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
@ -55,15 +55,15 @@ public:
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
};

61
moses/FF/InternalTree.cpp
View File

@ -6,7 +6,7 @@ namespace Moses
InternalTree::InternalTree(const std::string & line, size_t start, size_t len, const bool terminal):
m_value_nt(0),
m_isTerminal(terminal)
{
{
if (len > 0) {
m_value.assign(line, start, len);
@ -16,25 +16,24 @@ InternalTree::InternalTree(const std::string & line, size_t start, size_t len, c
InternalTree::InternalTree(const std::string & line, const bool terminal):
m_value_nt(0),
m_isTerminal(terminal)
{
{
size_t found = line.find_first_of("[] ");
if (found == line.npos) {
m_value = line;
}
else {
} else {
AddSubTree(line, 0);
}
}
size_t InternalTree::AddSubTree(const std::string & line, size_t pos) {
size_t InternalTree::AddSubTree(const std::string & line, size_t pos)
{
char token = 0;
size_t len = 0;
while (token != ']' && pos != std::string::npos)
{
while (token != ']' && pos != std::string::npos) {
size_t oldpos = pos;
pos = line.find_first_of("[] ", pos);
if (pos == std::string::npos) break;
@ -45,19 +44,16 @@ size_t InternalTree::AddSubTree(const std::string & line, size_t pos) {
if (!m_value.empty()) {
m_children.push_back(boost::make_shared<InternalTree>(line, oldpos, len, false));
pos = m_children.back()->AddSubTree(line, pos+1);
}
else {
} else {
if (len > 0) {
m_value.assign(line, oldpos, len);
}
pos = AddSubTree(line, pos+1);
}
}
else if (token == ' ' || token == ']') {
} else if (token == ' ' || token == ']') {
if (len > 0 && m_value.empty()) {
m_value.assign(line, oldpos, len);
}
else if (len > 0) {
} else if (len > 0) {
m_isTerminal = false;
m_children.push_back(boost::make_shared<InternalTree>(line, oldpos, len, true));
}
@ -78,7 +74,8 @@ size_t InternalTree::AddSubTree(const std::string & line, size_t pos) {
}
std::string InternalTree::GetString(bool start) const {
std::string InternalTree::GetString(bool start) const
{
std::string ret = "";
if (!start) {
@ -90,8 +87,7 @@ std::string InternalTree::GetString(bool start) const {
}
ret += m_value;
for (std::vector<TreePointer>::const_iterator it = m_children.begin(); it != m_children.end(); ++it)
{
for (std::vector<TreePointer>::const_iterator it = m_children.begin(); it != m_children.end(); ++it) {
ret += (*it)->GetString(false);
}
@ -103,7 +99,8 @@ std::string InternalTree::GetString(bool start) const {
}
void InternalTree::Combine(const std::vector<TreePointer> &previous) {
void InternalTree::Combine(const std::vector<TreePointer> &previous)
{
std::vector<TreePointer>::iterator it;
bool found = false;
@ -112,15 +109,15 @@ void InternalTree::Combine(const std::vector<TreePointer> &previous) {
found = next_leafNT(it);
if (found) {
*it = *it_prev;
}
else {
} else {
std::cerr << "Warning: leaf nonterminal not found in rule; why did this happen?\n";
}
}
}
bool InternalTree::FlatSearch(const std::string & label, std::vector<TreePointer>::const_iterator & it) const {
bool InternalTree::FlatSearch(const std::string & label, std::vector<TreePointer>::const_iterator & it) const
{
for (it = m_children.begin(); it != m_children.end(); ++it) {
if ((*it)->GetLabel() == label) {
return true;
@ -129,7 +126,8 @@ bool InternalTree::FlatSearch(const std::string & label, std::vector<TreePointer
return false;
}
bool InternalTree::RecursiveSearch(const std::string & label, std::vector<TreePointer>::const_iterator & it) const {
bool InternalTree::RecursiveSearch(const std::string & label, std::vector<TreePointer>::const_iterator & it) const
{
for (it = m_children.begin(); it != m_children.end(); ++it) {
if ((*it)->GetLabel() == label) {
return true;
@ -143,7 +141,8 @@ bool InternalTree::RecursiveSearch(const std::string & label, std::vector<TreePo
return false;
}
bool InternalTree::RecursiveSearch(const std::string & label, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const {
bool InternalTree::RecursiveSearch(const std::string & label, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const
{
for (it = m_children.begin(); it != m_children.end(); ++it) {
if ((*it)->GetLabel() == label) {
parent = this;
@ -159,7 +158,8 @@ bool InternalTree::RecursiveSearch(const std::string & label, std::vector<TreePo
}
bool InternalTree::FlatSearch(const NTLabel & label, std::vector<TreePointer>::const_iterator & it) const {
bool InternalTree::FlatSearch(const NTLabel & label, std::vector<TreePointer>::const_iterator & it) const
{
for (it = m_children.begin(); it != m_children.end(); ++it) {
if ((*it)->GetNTLabel() == label) {
return true;
@ -168,7 +168,8 @@ bool InternalTree::FlatSearch(const NTLabel & label, std::vector<TreePointer>::c
return false;
}
bool InternalTree::RecursiveSearch(const NTLabel & label, std::vector<TreePointer>::const_iterator & it) const {
bool InternalTree::RecursiveSearch(const NTLabel & label, std::vector<TreePointer>::const_iterator & it) const
{
for (it = m_children.begin(); it != m_children.end(); ++it) {
if ((*it)->GetNTLabel() == label) {
return true;
@ -182,7 +183,8 @@ bool InternalTree::RecursiveSearch(const NTLabel & label, std::vector<TreePointe
return false;
}
bool InternalTree::RecursiveSearch(const NTLabel & label, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const {
bool InternalTree::RecursiveSearch(const NTLabel & label, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const
{
for (it = m_children.begin(); it != m_children.end(); ++it) {
if ((*it)->GetNTLabel() == label) {
parent = this;
@ -198,7 +200,8 @@ bool InternalTree::RecursiveSearch(const NTLabel & label, std::vector<TreePointe
}
bool InternalTree::FlatSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it) const {
bool InternalTree::FlatSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it) const
{
for (it = m_children.begin(); it != m_children.end(); ++it) {
if (std::binary_search(labels.begin(), labels.end(), (*it)->GetNTLabel())) {
return true;
@ -207,7 +210,8 @@ bool InternalTree::FlatSearch(const std::vector<NTLabel> & labels, std::vector<T
return false;
}
bool InternalTree::RecursiveSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it) const {
bool InternalTree::RecursiveSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it) const
{
for (it = m_children.begin(); it != m_children.end(); ++it) {
if (std::binary_search(labels.begin(), labels.end(), (*it)->GetNTLabel())) {
return true;
@ -221,7 +225,8 @@ bool InternalTree::RecursiveSearch(const std::vector<NTLabel> & labels, std::vec
return false;
}
bool InternalTree::RecursiveSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const {
bool InternalTree::RecursiveSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const
{
for (it = m_children.begin(); it != m_children.end(); ++it) {
if (std::binary_search(labels.begin(), labels.end(), (*it)->GetNTLabel())) {
parent = this;

34
moses/FF/InternalTree.h
View File

@ -19,10 +19,10 @@ typedef int NTLabel;
class InternalTree
{
std::string m_value;
NTLabel m_value_nt;
std::vector<TreePointer> m_children;
bool m_isTerminal;
std::string m_value;
NTLabel m_value_nt;
std::vector<TreePointer> m_children;
bool m_isTerminal;
public:
InternalTree(const std::string & line, size_t start, size_t len, const bool terminal);
InternalTree(const std::string & line, const bool terminal = false);
@ -101,18 +101,21 @@ class TreeState : public FFState
TreePointer m_tree;
public:
TreeState(TreePointer tree)
:m_tree(tree)
{}
:m_tree(tree) {
}
TreePointer GetTree() const {
return m_tree;
}
int Compare(const FFState& other) const {return 0;};
int Compare(const FFState& other) const {
return 0;
};
};
// Python-like generator that yields next nonterminal leaf on every call
$generator(leafNT) {
$generator(leafNT)
{
std::vector<TreePointer>::iterator it;
InternalTree* tree;
leafNT(InternalTree* root = 0): tree(root) {}
@ -120,8 +123,7 @@ $generator(leafNT) {
for (it = tree->GetChildren().begin(); it !=tree->GetChildren().end(); ++it) {
if (!(*it)->IsTerminal() && (*it)->GetLength() == 0) {
$yield(it);
}
else if ((*it)->GetLength() > 0) {
} else if ((*it)->GetLength() > 0) {
if ((*it).get()) { // normal pointer to same object that TreePointer points to
$restart(tree = (*it).get());
}
@ -132,7 +134,8 @@ $generator(leafNT) {
// Python-like generator that yields the parent of the next nonterminal leaf on every call
$generator(leafNTParent) {
$generator(leafNTParent)
{
std::vector<TreePointer>::iterator it;
InternalTree* tree;
leafNTParent(InternalTree* root = 0): tree(root) {}
@ -140,8 +143,7 @@ $generator(leafNTParent) {
for (it = tree->GetChildren().begin(); it !=tree->GetChildren().end(); ++it) {
if (!(*it)->IsTerminal() && (*it)->GetLength() == 0) {
$yield(tree);
}
else if ((*it)->GetLength() > 0) {
} else if ((*it)->GetLength() > 0) {
if ((*it).get()) {
$restart(tree = (*it).get());
}
@ -151,7 +153,8 @@ $generator(leafNTParent) {
};
// Python-like generator that yields the next nonterminal leaf on every call, and also stores the path from the root of the tree to the nonterminal
$generator(leafNTPath) {
$generator(leafNTPath)
{
std::vector<TreePointer>::iterator it;
InternalTree* tree;
std::vector<InternalTree*> * path;
@ -163,8 +166,7 @@ $generator(leafNTPath) {
path->push_back((*it).get());
$yield(it);
path->pop_back();
}
else if ((*it)->GetLength() > 0) {
} else if ((*it)->GetLength() > 0) {
if ((*it).get()) {
$restart(tree = (*it).get());
}

20
moses/FF/LexicalReordering/LexicalReordering.h
View File

@ -59,20 +59,24 @@ public:
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
bool GetHaveDefaultScores() { return m_haveDefaultScores; }
float GetDefaultScore( size_t i ) { return m_defaultScores[i]; }
, ScoreComponentCollection &estimatedFutureScore) const {
}
bool GetHaveDefaultScores() {
return m_haveDefaultScores;
}
float GetDefaultScore( size_t i ) {
return m_defaultScores[i];
}
private:
bool DecodeCondition(std::string s);

8
moses/FF/LexicalReordering/LexicalReorderingState.cpp
View File

@ -39,7 +39,7 @@ size_t LexicalReorderingConfiguration::GetNumScoreComponents() const
}
void LexicalReorderingConfiguration::ConfigureSparse
(const std::map<std::string,std::string>& sparseArgs, const LexicalReordering* producer)
(const std::map<std::string,std::string>& sparseArgs, const LexicalReordering* producer)
{
if (sparseArgs.size()) {
m_sparse.reset(new SparseReordering(sparseArgs, producer));
@ -146,8 +146,7 @@ void LexicalReorderingState::CopyScores(ScoreComponentCollection* accum, const
const Scores &scoreSet = *cachedScores;
if(m_configuration.CollapseScores()) {
scores[m_offset] = scoreSet[m_offset + reoType];
}
else {
} else {
std::fill(scores.begin() + m_offset, scores.begin() + m_offset + m_configuration.GetNumberOfTypes(), 0);
scores[m_offset + reoType] = scoreSet[m_offset + reoType];
}
@ -158,8 +157,7 @@ void LexicalReorderingState::CopyScores(ScoreComponentCollection* accum, const
Scores scores(m_configuration.GetScoreProducer()->GetNumScoreComponents(),0);
if(m_configuration.CollapseScores()) {
scores[m_offset] = m_configuration.GetScoreProducer()->GetDefaultScore(m_offset + reoType);
}
else {
} else {
scores[m_offset + reoType] = m_configuration.GetScoreProducer()->GetDefaultScore(m_offset + reoType);
}
accum->PlusEquals(m_configuration.GetScoreProducer(), scores);

2
moses/FF/LexicalReordering/LexicalReorderingState.h
View File

@ -124,7 +124,7 @@ protected:
int ComparePrevScores(const TranslationOption *other) const;
//constants for the different type of reorderings (corresponding to indexes in the table file)
public:
public:
static const ReorderingType M = 0; // monotonic
static const ReorderingType NM = 1; // non-monotonic
static const ReorderingType S = 1; // swap

15
moses/FF/LexicalReordering/SparseReordering.cpp
View File

@ -19,7 +19,8 @@ using namespace std;
namespace Moses
{
const std::string& SparseReorderingFeatureKey::Name (const string& wordListId) {
const std::string& SparseReorderingFeatureKey::Name (const string& wordListId)
{
static string kSep = "-";
static string name;
ostringstream buf;
@ -93,7 +94,8 @@ SparseReordering::SparseReordering(const map<string,string>& config, const Lexic
}
void SparseReordering::PreCalculateFeatureNames(size_t index, const string& id, SparseReorderingFeatureKey::Side side, const Factor* factor, bool isCluster) {
void SparseReordering::PreCalculateFeatureNames(size_t index, const string& id, SparseReorderingFeatureKey::Side side, const Factor* factor, bool isCluster)
{
for (size_t type = SparseReorderingFeatureKey::Stack;
type <= SparseReorderingFeatureKey::Between; ++type) {
for (size_t position = SparseReorderingFeatureKey::First;
@ -108,7 +110,8 @@ void SparseReordering::PreCalculateFeatureNames(size_t index, const string& id,
}
}
void SparseReordering::ReadWordList(const string& filename, const string& id, SparseReorderingFeatureKey::Side side, vector<WordList>* pWordLists) {
void SparseReordering::ReadWordList(const string& filename, const string& id, SparseReorderingFeatureKey::Side side, vector<WordList>* pWordLists)
{
ifstream fh(filename.c_str());
UTIL_THROW_IF(!fh, util::Exception, "Unable to open: " << filename);
string line;
@ -123,7 +126,8 @@ void SparseReordering::ReadWordList(const string& filename, const string& id, Sp
}
}
void SparseReordering::ReadClusterMap(const string& filename, const string& id, SparseReorderingFeatureKey::Side side, vector<ClusterMap>* pClusterMaps) {
void SparseReordering::ReadClusterMap(const string& filename, const string& id, SparseReorderingFeatureKey::Side side, vector<ClusterMap>* pClusterMaps)
{
pClusterMaps->push_back(ClusterMap());
pClusterMaps->back().first = id;
util::FilePiece file(filename.c_str());
@ -149,7 +153,8 @@ void SparseReordering::AddFeatures(
SparseReorderingFeatureKey::Type type, SparseReorderingFeatureKey::Side side,
const Word& word, SparseReorderingFeatureKey::Position position,
LexicalReorderingState::ReorderingType reoType,
ScoreComponentCollection* scores) const {
ScoreComponentCollection* scores) const
{
const Factor* wordFactor = word.GetFactor(0);

4
moses/FF/LexicalReordering/SparseReordering.h
View File

@ -53,8 +53,8 @@ struct SparseReorderingFeatureKey {
SparseReorderingFeatureKey(size_t id_, Type type_, const Factor* word_, bool isCluster_,
Position position_, Side side_, LexicalReorderingState::ReorderingType reoType_)
: id(id_), type(type_), word(word_), isCluster(isCluster_),
position(position_), side(side_), reoType(reoType_)
{}
position(position_), side(side_), reoType(reoType_) {
}
const std::string& Name(const std::string& wordListId) ;
};

8
moses/FF/MaxSpanFreeNonTermSource.cpp
View File

@ -14,10 +14,10 @@ using namespace std;
namespace Moses
{
MaxSpanFreeNonTermSource::MaxSpanFreeNonTermSource(const std::string &line)
:StatelessFeatureFunction(1, line)
,m_maxSpan(2)
,m_glueTargetLHSStr("S")
,m_glueTargetLHS(true)
:StatelessFeatureFunction(1, line)
,m_maxSpan(2)
,m_glueTargetLHSStr("S")
,m_glueTargetLHS(true)
{
m_tuneable = false;
ReadParameters();

17
moses/FF/MaxSpanFreeNonTermSource.h
View File

@ -12,8 +12,9 @@ class MaxSpanFreeNonTermSource : public StatelessFeatureFunction
public:
MaxSpanFreeNonTermSource(const std::string &line);
virtual bool IsUseable(const FactorMask &mask) const
{ return true; }
virtual bool IsUseable(const FactorMask &mask) const {
return true;
}
virtual void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
@ -28,16 +29,16 @@ public:
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void SetParameter(const std::string& key, const std::string& value);
std::vector<float> DefaultWeights() const;

4
moses/FF/NieceTerminal.h
View File

@ -31,8 +31,8 @@ public:
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const;

12
moses/FF/OSM-Feature/KenOSM.h
View File

@ -7,8 +7,9 @@
namespace Moses
{
class KenOSMBase {
public:
class KenOSMBase
{
public:
virtual float Score(const lm::ngram::State&, const std::string&,
lm::ngram::State&) const = 0;
@ -18,8 +19,9 @@ class KenOSMBase {
};
template <class KenModel>
class KenOSM : public KenOSMBase {
public:
class KenOSM : public KenOSMBase
{
public:
KenOSM(const std::string& file)
: m_kenlm(new KenModel(file.c_str())) {}
@ -38,7 +40,7 @@ class KenOSM : public KenOSMBase {
return m_kenlm->NullContextState();
}
private:
private:
boost::shared_ptr<KenModel> m_kenlm;
};

3
moses/FF/OSM-Feature/OpSequenceModel.cpp
View File

@ -19,7 +19,8 @@ OpSequenceModel::OpSequenceModel(const std::string &line)
ReadParameters();
}
OpSequenceModel::~OpSequenceModel() {
OpSequenceModel::~OpSequenceModel()
{
delete OSM;
}

8
moses/FF/OSM-Feature/OpSequenceModel.h
View File

@ -42,12 +42,12 @@ public:
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase

12
moses/FF/PhraseBoundaryFeature.h
View File

@ -58,17 +58,17 @@ public:
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
, ScoreComponentCollection &estimatedFutureScore) const {
}
void SetParameter(const std::string& key, const std::string& value);

16
moses/FF/PhraseLengthFeature.h
View File

@ -25,24 +25,24 @@ public:
}
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void EvaluateWhenApplied(const ChartHypothesis& hypo,
ScoreComponentCollection*) const
{}
ScoreComponentCollection*) const {
}
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
virtual void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown

184
moses/FF/PhraseOrientationFeature.cpp
View File

@ -40,12 +40,9 @@ PhraseOrientationFeature::PhraseOrientationFeature(const std::string &line)
void PhraseOrientationFeature::SetParameter(const std::string& key, const std::string& value)
{
if (key == "glueTargetLHS")
{
if (key == "glueTargetLHS") {
m_glueTargetLHSStr = value;
}
else
{
} else {
StatefulFeatureFunction::SetParameter(key, value);
}
}
@ -69,20 +66,17 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
// const Factor* targetLHS = currTarPhr.GetTargetLHS()[0];
// bool isGlueGrammarRule = false;
IFFEATUREVERBOSE(2)
{
IFFEATUREVERBOSE(2) {
FEATUREVERBOSE(2, *currSrcPhr << std::endl);
FEATUREVERBOSE(2, currTarPhr << std::endl);
for (AlignmentInfo::const_iterator it=currTarPhr.GetAlignTerm().begin();
it!=currTarPhr.GetAlignTerm().end(); ++it)
{
it!=currTarPhr.GetAlignTerm().end(); ++it) {
FEATUREVERBOSE(2, "alignTerm " << it->first << " " << it->second << std::endl);
}
for (AlignmentInfo::const_iterator it=currTarPhr.GetAlignNonTerm().begin();
it!=currTarPhr.GetAlignNonTerm().end(); ++it)
{
it!=currTarPhr.GetAlignNonTerm().end(); ++it) {
FEATUREVERBOSE(2, "alignNonTerm " << it->first << " " << it->second << std::endl);
}
}
@ -98,8 +92,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
// Determine & score orientations
for (AlignmentInfo::const_iterator it=currTarPhr.GetAlignNonTerm().begin();
it!=currTarPhr.GetAlignNonTerm().end(); ++it)
{
it!=currTarPhr.GetAlignNonTerm().end(); ++it) {
size_t sourceIndex = it->first;
size_t targetIndex = it->second;
size_t nonTermIndex = nonTermIndexMap[targetIndex];
@ -110,8 +103,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
const ChartHypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndex);
const TargetPhrase &prevTarPhr = prevHypo->GetCurrTargetPhrase();
if (const PhraseProperty *property = prevTarPhr.GetProperty("Orientation"))
{
if (const PhraseProperty *property = prevTarPhr.GetProperty("Orientation")) {
const OrientationPhraseProperty *orientationPhraseProperty = static_cast<const OrientationPhraseProperty*>(property);
FEATUREVERBOSE(5, "orientationPhraseProperty: "
@ -133,11 +125,9 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
Moses::GHKM::PhraseOrientation::REO_CLASS l2rOrientation = phraseOrientation.GetOrientationInfo(sourceIndex,sourceIndex,Moses::GHKM::PhraseOrientation::REO_DIR_L2R);
IFFEATUREVERBOSE(2)
{
IFFEATUREVERBOSE(2) {
FEATUREVERBOSE(2, "l2rOrientation ");
switch (l2rOrientation)
{
switch (l2rOrientation) {
case Moses::GHKM::PhraseOrientation::REO_CLASS_LEFT:
FEATUREVERBOSE2(2, "mono" << std::endl);
break;
@ -164,8 +154,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
bool delayedScoringL2R = false;
if ( ((targetIndex == 0) || !phraseOrientation.TargetSpanIsAligned(0,targetIndex)) // boundary non-terminal in rule-initial position (left boundary)
&& (currTarPhrLHS != m_glueTargetLHS) ) // and not glue rule
{
&& (currTarPhrLHS != m_glueTargetLHS) ) { // and not glue rule
// delay left-to-right scoring
FEATUREVERBOSE(3, "Left boundary");
@ -180,13 +169,10 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
FEATUREVERBOSE(4, "previousSourceSpanIsAligned = " << previousSourceSpanIsAligned << std::endl);
FEATUREVERBOSE(4, "followingSourceSpanIsAligned = " << followingSourceSpanIsAligned << std::endl;);
if (previousSourceSpanIsAligned && followingSourceSpanIsAligned)
{
if (previousSourceSpanIsAligned && followingSourceSpanIsAligned) {
// discontinuous
l2rOrientation = Moses::GHKM::PhraseOrientation::REO_CLASS_DLEFT;
}
else
{
} else {
FEATUREVERBOSE(3, "Delaying left-to-right scoring" << std::endl);
delayedScoringL2R = true;
@ -202,25 +188,20 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
scoresL2R.push_back( std::log(orientationPhraseProperty->GetLeftToRightProbabilitySwap()) );
scoresL2R.push_back( std::log(orientationPhraseProperty->GetLeftToRightProbabilityDiscontinuous()) );
std::vector<float> weightedScoresL2R;
for ( size_t i=0; i<3;++i )
{
for ( size_t i=0; i<3; ++i ) {
weightedScoresL2R.push_back( weightsVector[i] * scoresL2R[i] );
}
size_t heuristicScoreIndex = 0;
for (size_t i=1; i<3; ++i)
{
if (possibleFutureOrientationsL2R[i])
{
if (weightedScoresL2R[i] > weightedScoresL2R[heuristicScoreIndex])
{
for (size_t i=1; i<3; ++i) {
if (possibleFutureOrientationsL2R[i]) {
if (weightedScoresL2R[i] > weightedScoresL2R[heuristicScoreIndex]) {
heuristicScoreIndex = i;
}
}
}
IFFEATUREVERBOSE(5)
{
IFFEATUREVERBOSE(5) {
FEATUREVERBOSE(5, "Heuristic score computation (L2R): "
<< "heuristicScoreIndex= " << heuristicScoreIndex);
for (size_t i=0; i<3; ++i)
@ -231,8 +212,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
FEATUREVERBOSE2(5, " weightedScoresL2R[" << i << "]= " << weightedScoresL2R[i]);
for (size_t i=0; i<3; ++i)
FEATUREVERBOSE2(5, " possibleFutureOrientationsL2R[" << i << "]= " << possibleFutureOrientationsL2R[i]);
if ( possibleFutureOrientationsL2R == 0x7 )
{
if ( possibleFutureOrientationsL2R == 0x7 ) {
FEATUREVERBOSE2(5, " (all orientations possible)");
}
FEATUREVERBOSE2(5, std::endl);
@ -241,8 +221,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
newScores[heuristicScoreIndex] += scoresL2R[heuristicScoreIndex];
state->SetLeftBoundaryL2R(scoresL2R, heuristicScoreIndex, possibleFutureOrientationsL2R, prevState);
if ( (possibleFutureOrientationsL2R & prevState->m_leftBoundaryNonTerminalL2RPossibleFutureOrientations) == 0x4 )
{
if ( (possibleFutureOrientationsL2R & prevState->m_leftBoundaryNonTerminalL2RPossibleFutureOrientations) == 0x4 ) {
// recursive: discontinuous orientation
FEATUREVERBOSE(5, "previous state: L2R discontinuous orientation "
<< possibleFutureOrientationsL2R << " & " << prevState->m_leftBoundaryNonTerminalL2RPossibleFutureOrientations
@ -254,10 +233,8 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
}
}
if (!delayedScoringL2R)
{
switch (l2rOrientation)
{
if (!delayedScoringL2R) {
switch (l2rOrientation) {
case Moses::GHKM::PhraseOrientation::REO_CLASS_LEFT:
newScores[0] += std::log(orientationPhraseProperty->GetLeftToRightProbabilityMono());
// if sub-derivation has left-boundary non-terminal:
@ -301,11 +278,9 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
Moses::GHKM::PhraseOrientation::REO_CLASS r2lOrientation = phraseOrientation.GetOrientationInfo(sourceIndex,sourceIndex,Moses::GHKM::PhraseOrientation::REO_DIR_R2L);
IFFEATUREVERBOSE(2)
{
IFFEATUREVERBOSE(2) {
FEATUREVERBOSE(2, "r2lOrientation ");
switch (r2lOrientation)
{
switch (r2lOrientation) {
case Moses::GHKM::PhraseOrientation::REO_CLASS_LEFT:
FEATUREVERBOSE2(2, "mono" << std::endl);
break;
@ -332,8 +307,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
bool delayedScoringR2L = false;
if ( ((targetIndex == currTarPhr.GetSize()-1) || !phraseOrientation.TargetSpanIsAligned(targetIndex,currTarPhr.GetSize()-1)) // boundary non-terminal in rule-final position (right boundary)
&& (currTarPhrLHS != m_glueTargetLHS) ) // and not glue rule
{
&& (currTarPhrLHS != m_glueTargetLHS) ) { // and not glue rule
// delay right-to-left scoring
FEATUREVERBOSE(3, "Right boundary");
@ -348,13 +322,10 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
FEATUREVERBOSE(4, "previousSourceSpanIsAligned = " << previousSourceSpanIsAligned << std::endl);
FEATUREVERBOSE(4, "followingSourceSpanIsAligned = " << followingSourceSpanIsAligned << std::endl;);
if (previousSourceSpanIsAligned && followingSourceSpanIsAligned)
{
if (previousSourceSpanIsAligned && followingSourceSpanIsAligned) {
// discontinuous
r2lOrientation = Moses::GHKM::PhraseOrientation::REO_CLASS_DLEFT;
}
else
{
} else {
FEATUREVERBOSE(3, "Delaying right-to-left scoring" << std::endl);
delayedScoringR2L = true;
@ -370,25 +341,20 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
scoresR2L.push_back( std::log(orientationPhraseProperty->GetRightToLeftProbabilitySwap()) );
scoresR2L.push_back( std::log(orientationPhraseProperty->GetRightToLeftProbabilityDiscontinuous()) );
std::vector<float> weightedScoresR2L;
for ( size_t i=0; i<3;++i )
{
for ( size_t i=0; i<3; ++i ) {
weightedScoresR2L.push_back( weightsVector[m_offsetR2LScores+i] * scoresR2L[i] );
}
size_t heuristicScoreIndex = 0;
for (size_t i=1; i<3; ++i)
{
if (possibleFutureOrientationsR2L[i])
{
if (weightedScoresR2L[i] > weightedScoresR2L[heuristicScoreIndex])
{
for (size_t i=1; i<3; ++i) {
if (possibleFutureOrientationsR2L[i]) {
if (weightedScoresR2L[i] > weightedScoresR2L[heuristicScoreIndex]) {
heuristicScoreIndex = i;
}
}
}
IFFEATUREVERBOSE(5)
{
IFFEATUREVERBOSE(5) {
FEATUREVERBOSE(5, "Heuristic score computation (R2L): "
<< "heuristicScoreIndex= " << heuristicScoreIndex);
for (size_t i=0; i<3; ++i)
@ -399,8 +365,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
FEATUREVERBOSE2(5, " weightedScoresR2L[" << i << "]= " << weightedScoresR2L[i]);
for (size_t i=0; i<3; ++i)
FEATUREVERBOSE2(5, " possibleFutureOrientationsR2L[" << i << "]= " << possibleFutureOrientationsR2L[i]);
if ( possibleFutureOrientationsR2L == 0x7 )
{
if ( possibleFutureOrientationsR2L == 0x7 ) {
FEATUREVERBOSE2(5, " (all orientations possible)");
}
FEATUREVERBOSE2(5, std::endl);
@ -409,8 +374,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
newScores[m_offsetR2LScores+heuristicScoreIndex] += scoresR2L[heuristicScoreIndex];
state->SetRightBoundaryR2L(scoresR2L, heuristicScoreIndex, possibleFutureOrientationsR2L, prevState);
if ( (possibleFutureOrientationsR2L & prevState->m_rightBoundaryNonTerminalR2LPossibleFutureOrientations) == 0x4 )
{
if ( (possibleFutureOrientationsR2L & prevState->m_rightBoundaryNonTerminalR2LPossibleFutureOrientations) == 0x4 ) {
// recursive: discontinuous orientation
FEATUREVERBOSE(5, "previous state: R2L discontinuous orientation "
<< possibleFutureOrientationsR2L << " & " << prevState->m_rightBoundaryNonTerminalR2LPossibleFutureOrientations
@ -422,10 +386,8 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
}
}
if (!delayedScoringR2L)
{
switch (r2lOrientation)
{
if (!delayedScoringR2L) {
switch (r2lOrientation) {
case Moses::GHKM::PhraseOrientation::REO_CLASS_LEFT:
newScores[m_offsetR2LScores+0] += std::log(orientationPhraseProperty->GetRightToLeftProbabilityMono());
// if sub-derivation has right-boundary non-terminal:
@ -463,9 +425,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
break;
}
}
}
else
{
} else {
// abort with error message if the phrase does not translate an unknown word
UTIL_THROW_IF2(!prevTarPhr.GetWord(0).IsOOV(), GetScoreProducerDescription()
<< ": Missing Orientation property. "
@ -483,43 +443,30 @@ void PhraseOrientationFeature::LeftBoundaryL2RScoreRecursive(int featureID,
const std::bitset<3> orientation,
std::vector<float>& newScores) const
{
if (state->m_leftBoundaryIsSet)
{
if (state->m_leftBoundaryIsSet) {
// subtract heuristic score from subderivation
newScores[state->m_leftBoundaryNonTerminalL2RHeuristicScoreIndex] -= state->m_leftBoundaryNonTerminalL2RScores[state->m_leftBoundaryNonTerminalL2RHeuristicScoreIndex];
// add actual score
std::bitset<3> recursiveOrientation = orientation;
if ( (orientation == 0x4) || (orientation == 0x0) )
{
if ( (orientation == 0x4) || (orientation == 0x0) ) {
// discontinuous
newScores[2] += state->GetLeftBoundaryL2RScoreDiscontinuous();
}
else
{
} else {
recursiveOrientation &= state->m_leftBoundaryNonTerminalL2RPossibleFutureOrientations;
if ( recursiveOrientation == 0x1 )
{
if ( recursiveOrientation == 0x1 ) {
// monotone
newScores[0] += state->GetLeftBoundaryL2RScoreMono();
}
else if ( recursiveOrientation == 0x2 )
{
} else if ( recursiveOrientation == 0x2 ) {
// swap
newScores[1] += state->GetLeftBoundaryL2RScoreSwap();
}
else if ( recursiveOrientation == 0x4 )
{
} else if ( recursiveOrientation == 0x4 ) {
// discontinuous
newScores[2] += state->GetLeftBoundaryL2RScoreDiscontinuous();
}
else if ( recursiveOrientation == 0x0 )
{
} else if ( recursiveOrientation == 0x0 ) {
// discontinuous
newScores[2] += state->GetLeftBoundaryL2RScoreDiscontinuous();
}
else
{
} else {
UTIL_THROW2(GetScoreProducerDescription()
<< ": Error in recursive scoring.");
}
@ -528,14 +475,11 @@ void PhraseOrientationFeature::LeftBoundaryL2RScoreRecursive(int featureID,
FEATUREVERBOSE(6, "Left boundary recursion: " << orientation << " & " << state->m_leftBoundaryNonTerminalL2RPossibleFutureOrientations << " = " << recursiveOrientation
<< " --- Subtracted heuristic score: " << state->m_leftBoundaryNonTerminalL2RScores[state->m_leftBoundaryNonTerminalL2RHeuristicScoreIndex] << std::endl);
if (!state->m_leftBoundaryRecursionGuard)
{
if (!state->m_leftBoundaryRecursionGuard) {
// recursive call
const PhraseOrientationFeatureState* prevState = state->m_leftBoundaryPrevState;
LeftBoundaryL2RScoreRecursive(featureID, prevState, recursiveOrientation, newScores);
}
else
{
} else {
FEATUREVERBOSE(6, "m_leftBoundaryRecursionGuard" << std::endl);
}
}
@ -546,43 +490,30 @@ void PhraseOrientationFeature::RightBoundaryR2LScoreRecursive(int featureID,
const std::bitset<3> orientation,
std::vector<float>& newScores) const
{
if (state->m_rightBoundaryIsSet)
{
if (state->m_rightBoundaryIsSet) {
// subtract heuristic score from subderivation
newScores[m_offsetR2LScores+state->m_rightBoundaryNonTerminalR2LHeuristicScoreIndex] -= state->m_rightBoundaryNonTerminalR2LScores[state->m_rightBoundaryNonTerminalR2LHeuristicScoreIndex];
// add actual score
std::bitset<3> recursiveOrientation = orientation;
if ( (orientation == 0x4) || (orientation == 0x0) )
{
if ( (orientation == 0x4) || (orientation == 0x0) ) {
// discontinuous
newScores[m_offsetR2LScores+2] += state->GetRightBoundaryR2LScoreDiscontinuous();
}
else
{
} else {
recursiveOrientation &= state->m_rightBoundaryNonTerminalR2LPossibleFutureOrientations;
if ( recursiveOrientation == 0x1 )
{
if ( recursiveOrientation == 0x1 ) {
// monotone
newScores[m_offsetR2LScores+0] += state->GetRightBoundaryR2LScoreMono();
}
else if ( recursiveOrientation == 0x2 )
{
} else if ( recursiveOrientation == 0x2 ) {
// swap
newScores[m_offsetR2LScores+1] += state->GetRightBoundaryR2LScoreSwap();
}
else if ( recursiveOrientation == 0x4 )
{
} else if ( recursiveOrientation == 0x4 ) {
// discontinuous
newScores[m_offsetR2LScores+2] += state->GetRightBoundaryR2LScoreDiscontinuous();
}
else if ( recursiveOrientation == 0x0 )
{
} else if ( recursiveOrientation == 0x0 ) {
// discontinuous
newScores[m_offsetR2LScores+2] += state->GetRightBoundaryR2LScoreDiscontinuous();
}
else
{
} else {
UTIL_THROW2(GetScoreProducerDescription()
<< ": Error in recursive scoring.");
}
@ -591,14 +522,11 @@ void PhraseOrientationFeature::RightBoundaryR2LScoreRecursive(int featureID,
FEATUREVERBOSE(6, "Right boundary recursion: " << orientation << " & " << state->m_rightBoundaryNonTerminalR2LPossibleFutureOrientations << " = " << recursiveOrientation
<< " --- Subtracted heuristic score: " << state->m_rightBoundaryNonTerminalR2LScores[state->m_rightBoundaryNonTerminalR2LHeuristicScoreIndex] << std::endl);
if (!state->m_rightBoundaryRecursionGuard)
{
if (!state->m_rightBoundaryRecursionGuard) {
// recursive call
const PhraseOrientationFeatureState* prevState = state->m_rightBoundaryPrevState;
RightBoundaryR2LScoreRecursive(featureID, prevState, recursiveOrientation, newScores);
}
else
{
} else {
FEATUREVERBOSE(6, "m_rightBoundaryRecursionGuard" << std::endl);
}
}

168
moses/FF/PhraseOrientationFeature.h
View File

@ -36,16 +36,14 @@ public:
, m_leftBoundaryRecursionGuard(false)
, m_rightBoundaryRecursionGuard(false)
, m_leftBoundaryIsSet(false)
, m_rightBoundaryIsSet(false)
{}
, m_rightBoundaryIsSet(false) {
}
void SetLeftBoundaryL2R(const std::vector<float> &scores,
size_t heuristicScoreIndex,
std::bitset<3> &possibleFutureOrientations,
const PhraseOrientationFeatureState* prevState)
{
for (size_t i=0; i<3; ++i)
{
const PhraseOrientationFeatureState* prevState) {
for (size_t i=0; i<3; ++i) {
m_leftBoundaryNonTerminalL2RScores[i] = scores[i];
m_leftBoundaryNonTerminalL2RPossibleFutureOrientations[i] = possibleFutureOrientations[i];
}
@ -57,10 +55,8 @@ public:
void SetRightBoundaryR2L(const std::vector<float> &scores,
size_t heuristicScoreIndex,
std::bitset<3> &possibleFutureOrientations,
const PhraseOrientationFeatureState* prevState)
{
for (size_t i=0; i<3; ++i)
{
const PhraseOrientationFeatureState* prevState) {
for (size_t i=0; i<3; ++i) {
m_rightBoundaryNonTerminalR2LScores[i] = scores[i];
m_rightBoundaryNonTerminalR2LPossibleFutureOrientations[i] = possibleFutureOrientations[i];
}
@ -70,77 +66,61 @@ public:
}
float GetLeftBoundaryL2RScoreMono() const
{
float GetLeftBoundaryL2RScoreMono() const {
return m_leftBoundaryNonTerminalL2RScores[0];
}
float GetLeftBoundaryL2RScoreSwap() const
{
float GetLeftBoundaryL2RScoreSwap() const {
return m_leftBoundaryNonTerminalL2RScores[1];
}
float GetLeftBoundaryL2RScoreDiscontinuous() const
{
float GetLeftBoundaryL2RScoreDiscontinuous() const {
return m_leftBoundaryNonTerminalL2RScores[2];
}
float GetRightBoundaryR2LScoreMono() const
{
float GetRightBoundaryR2LScoreMono() const {
return m_rightBoundaryNonTerminalR2LScores[0];
}
float GetRightBoundaryR2LScoreSwap() const
{
float GetRightBoundaryR2LScoreSwap() const {
return m_rightBoundaryNonTerminalR2LScores[1];
}
float GetRightBoundaryR2LScoreDiscontinuous() const
{
float GetRightBoundaryR2LScoreDiscontinuous() const {
return m_rightBoundaryNonTerminalR2LScores[2];
}
int Compare(const FFState& other) const
{
int Compare(const FFState& other) const {
const PhraseOrientationFeatureState &otherState = static_cast<const PhraseOrientationFeatureState&>(other);
if (!m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet &&
!m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet)
{
!m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) {
return 0;
}
if (m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet)
{
if (m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet) {
return 1;
}
if (!m_leftBoundaryIsSet && otherState.m_leftBoundaryIsSet)
{
if (!m_leftBoundaryIsSet && otherState.m_leftBoundaryIsSet) {
return -1;
}
if (m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet)
{
if (m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) {
return 1;
}
if (!m_rightBoundaryIsSet && otherState.m_rightBoundaryIsSet)
{
if (!m_rightBoundaryIsSet && otherState.m_rightBoundaryIsSet) {
return -1;
}
if (m_leftBoundaryIsSet)
{
if (m_leftBoundaryIsSet) {
int compareLeft = CompareLeftBoundaryRecursive(*this, otherState);
if (compareLeft != 0)
{
if (compareLeft != 0) {
return compareLeft;
}
}
if (m_rightBoundaryIsSet)
{
if (m_rightBoundaryIsSet) {
int compareRight = CompareRightBoundaryRecursive(*this, otherState);
if (compareRight != 0)
{
if (compareRight != 0) {
return compareRight;
}
}
@ -150,59 +130,45 @@ public:
private:
static int CompareLeftBoundaryRecursive(const PhraseOrientationFeatureState& state, const PhraseOrientationFeatureState& otherState)
{
if (!state.m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet)
{
static int CompareLeftBoundaryRecursive(const PhraseOrientationFeatureState& state, const PhraseOrientationFeatureState& otherState) {
if (!state.m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet) {
return 0;
}
if (state.m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet)
{
if (state.m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet) {
return 1;
}
if (!state.m_leftBoundaryIsSet && otherState.m_leftBoundaryIsSet)
{
if (!state.m_leftBoundaryIsSet && otherState.m_leftBoundaryIsSet) {
return -1;
}
if ( otherState.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex < state.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex )
{
if ( otherState.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex < state.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex ) {
return 1;
}
if ( state.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex < otherState.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex )
{
if ( state.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex < otherState.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex ) {
return -1;
}
if ( Smaller(otherState.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations, state.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations) )
{
if ( Smaller(otherState.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations, state.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations) ) {
return 1;
}
if ( Smaller(state.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations, otherState.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations) )
{
if ( Smaller(state.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations, otherState.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations) ) {
return -1;
}
for (size_t i=0; i<state.m_leftBoundaryNonTerminalL2RScores.size(); ++i)
{
if (state.m_leftBoundaryNonTerminalL2RScores[i] > otherState.m_leftBoundaryNonTerminalL2RScores[i])
{
for (size_t i=0; i<state.m_leftBoundaryNonTerminalL2RScores.size(); ++i) {
if (state.m_leftBoundaryNonTerminalL2RScores[i] > otherState.m_leftBoundaryNonTerminalL2RScores[i]) {
return 1;
}
if (state.m_leftBoundaryNonTerminalL2RScores[i] < otherState.m_leftBoundaryNonTerminalL2RScores[i])
{
if (state.m_leftBoundaryNonTerminalL2RScores[i] < otherState.m_leftBoundaryNonTerminalL2RScores[i]) {
return -1;
}
}
if (state.m_leftBoundaryRecursionGuard && otherState.m_leftBoundaryRecursionGuard)
{
if (state.m_leftBoundaryRecursionGuard && otherState.m_leftBoundaryRecursionGuard) {
return 0;
}
if (state.m_leftBoundaryRecursionGuard && !otherState.m_leftBoundaryRecursionGuard)
{
if (state.m_leftBoundaryRecursionGuard && !otherState.m_leftBoundaryRecursionGuard) {
return 1;
}
if (!state.m_leftBoundaryRecursionGuard && otherState.m_leftBoundaryRecursionGuard)
{
if (!state.m_leftBoundaryRecursionGuard && otherState.m_leftBoundaryRecursionGuard) {
return -1;
}
@ -212,59 +178,45 @@ private:
return CompareLeftBoundaryRecursive(*prevState, *otherPrevState);
};
static int CompareRightBoundaryRecursive(const PhraseOrientationFeatureState& state, const PhraseOrientationFeatureState& otherState)
{
if (!state.m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet)
{
static int CompareRightBoundaryRecursive(const PhraseOrientationFeatureState& state, const PhraseOrientationFeatureState& otherState) {
if (!state.m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) {
return 0;
}
if (state.m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet)
{
if (state.m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) {
return 1;
}
if (!state.m_rightBoundaryIsSet && otherState.m_rightBoundaryIsSet)
{
if (!state.m_rightBoundaryIsSet && otherState.m_rightBoundaryIsSet) {
return -1;
}
if ( otherState.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex < state.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex )
{
if ( otherState.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex < state.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex ) {
return 1;
}
if ( state.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex < otherState.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex )
{
if ( state.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex < otherState.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex ) {
return -1;
}
if ( Smaller(otherState.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations, state.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations) )
{
if ( Smaller(otherState.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations, state.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations) ) {
return 1;
}
if ( Smaller(state.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations, otherState.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations) )
{
if ( Smaller(state.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations, otherState.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations) ) {
return -1;
}
for (size_t i=0; i<state.m_rightBoundaryNonTerminalR2LScores.size(); ++i)
{
if (state.m_rightBoundaryNonTerminalR2LScores[i] > otherState.m_rightBoundaryNonTerminalR2LScores[i])
{
for (size_t i=0; i<state.m_rightBoundaryNonTerminalR2LScores.size(); ++i) {
if (state.m_rightBoundaryNonTerminalR2LScores[i] > otherState.m_rightBoundaryNonTerminalR2LScores[i]) {
return 1;
}
if (state.m_rightBoundaryNonTerminalR2LScores[i] < otherState.m_rightBoundaryNonTerminalR2LScores[i])
{
if (state.m_rightBoundaryNonTerminalR2LScores[i] < otherState.m_rightBoundaryNonTerminalR2LScores[i]) {
return -1;
}
}
if (state.m_rightBoundaryRecursionGuard && otherState.m_rightBoundaryRecursionGuard)
{
if (state.m_rightBoundaryRecursionGuard && otherState.m_rightBoundaryRecursionGuard) {
return 0;
}
if (state.m_rightBoundaryRecursionGuard && !otherState.m_rightBoundaryRecursionGuard)
{
if (state.m_rightBoundaryRecursionGuard && !otherState.m_rightBoundaryRecursionGuard) {
return 1;
}
if (!state.m_rightBoundaryRecursionGuard && otherState.m_rightBoundaryRecursionGuard)
{
if (!state.m_rightBoundaryRecursionGuard && otherState.m_rightBoundaryRecursionGuard) {
return -1;
}
@ -274,10 +226,8 @@ private:
return CompareRightBoundaryRecursive(*prevState, *otherPrevState);
};
template<std::size_t N> static bool Smaller(const std::bitset<N>& x, const std::bitset<N>& y)
{
for (size_t i=0; i<N; ++i)
{
template<std::size_t N> static bool Smaller(const std::bitset<N>& x, const std::bitset<N>& y) {
for (size_t i=0; i<N; ++i) {
if (x[i] ^ y[i])
return y[i];
}
@ -325,8 +275,7 @@ public:
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
, ScoreComponentCollection &estimatedFutureScore) const {
targetPhrase.SetRuleSource(source);
};
@ -335,17 +284,16 @@ public:
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{};
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
};
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
{
ScoreComponentCollection* accumulator) const {
return new PhraseOrientationFeatureState();
};

16
moses/FF/PhrasePairFeature.h
View File

@ -43,12 +43,12 @@ public:
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
, ScoreComponentCollection &estimatedFutureScore) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
@ -57,12 +57,12 @@ public:
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void EvaluateWhenApplied(const ChartHypothesis& hypo,
ScoreComponentCollection*) const
{}
ScoreComponentCollection*) const {
}
};

10
moses/FF/PhrasePenalty.cpp
View File

@ -9,8 +9,8 @@ using namespace std;
namespace Moses
{
PhrasePenalty::PhrasePenalty(const std::string &line)
: StatelessFeatureFunction(1, line)
, m_perPhraseTable(false)
: StatelessFeatureFunction(1, line)
, m_perPhraseTable(false)
{
ReadParameters();
}
@ -32,8 +32,7 @@ void PhrasePenalty::EvaluateInIsolation(const Phrase &source
scoreBreakdown.Assign(this, scores);
}
}
else {
} else {
scoreBreakdown.Assign(this, 1.0f);
}
}
@ -42,8 +41,7 @@ void PhrasePenalty::SetParameter(const std::string& key, const std::string& valu
{
if (key == "per-phrase-table") {
m_perPhraseTable =Scan<bool>(value);
}
else {
} else {
StatelessFeatureFunction::SetParameter(key, value);
}
}

20
moses/FF/PhrasePenalty.h
View File

@ -20,26 +20,26 @@ public:
, ScoreComponentCollection &estimatedFutureScore) const;
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void EvaluateWhenApplied(const Syntax::SHyperedge &hyperedge,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
void SetParameter(const std::string& key, const std::string& value);
protected:

30
moses/FF/ReferenceComparison.h
View File

@ -12,37 +12,39 @@ class ReferenceComparison : public StatelessFeatureFunction
public:
ReferenceComparison(const std::string &line);
virtual bool IsUseable(const FactorMask &mask) const
{ return true; }
virtual bool IsUseable(const FactorMask &mask) const {
return true;
}
virtual void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
, ScoreComponentCollection &estimatedFutureScore) const {
}
virtual void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
std::vector<float> DefaultWeights() const
{ return std::vector<float>(); }
std::vector<float> DefaultWeights() const {
return std::vector<float>();
}
protected:

7
moses/FF/RuleScope.cpp
View File

@ -5,8 +5,8 @@
namespace Moses
{
RuleScope::RuleScope(const std::string &line)
:StatelessFeatureFunction(1, line)
,m_sourceSyntax(true)
:StatelessFeatureFunction(1, line)
,m_sourceSyntax(true)
{
}
@ -31,8 +31,7 @@ void RuleScope::EvaluateInIsolation(const Phrase &source
bool ambiguous = IsAmbiguous(word, m_sourceSyntax);
if (ambiguous) {
++count;
}
else {
} else {
if (count > 0) {
score += count;
}

21
moses/FF/RuleScope.h
View File

@ -11,8 +11,9 @@ class RuleScope : public StatelessFeatureFunction
public:
RuleScope(const std::string &line);
virtual bool IsUseable(const FactorMask &mask) const
{ return true; }
virtual bool IsUseable(const FactorMask &mask) const {
return true;
}
virtual void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
@ -24,21 +25,21 @@ public:
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void SetParameter(const std::string& key, const std::string& value);

2
moses/FF/SetSourcePhrase.cpp
View File

@ -4,7 +4,7 @@
namespace Moses
{
SetSourcePhrase::SetSourcePhrase(const std::string &line)
:StatelessFeatureFunction(0, line)
:StatelessFeatureFunction(0, line)
{
m_tuneable = false;
ReadParameters();

26
moses/FF/SetSourcePhrase.h
View File

@ -11,8 +11,9 @@ class SetSourcePhrase : public StatelessFeatureFunction
public:
SetSourcePhrase(const std::string &line);
virtual bool IsUseable(const FactorMask &mask) const
{ return true; }
virtual bool IsUseable(const FactorMask &mask) const {
return true;
}
virtual void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
@ -24,23 +25,24 @@ public:
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
std::vector<float> DefaultWeights() const
{ return std::vector<float>(); }
std::vector<float> DefaultWeights() const {
return std::vector<float>();
}
};

4
moses/FF/SkeletonStatefulFF.h
View File

@ -12,8 +12,8 @@ class SkeletonState : public FFState
int m_targetLen;
public:
SkeletonState(int targetLen)
:m_targetLen(targetLen)
{}
:m_targetLen(targetLen) {
}
int Compare(const FFState& other) const;
};

26
moses/FF/SkeletonTranslationOptionListFeature.h
View File

@ -10,8 +10,7 @@ class SkeletonTranslationOptionListFeature : public StatelessFeatureFunction
{
public:
SkeletonTranslationOptionListFeature(const std::string &line)
:StatelessFeatureFunction(1, line)
{
:StatelessFeatureFunction(1, line) {
ReadParameters();
}
@ -22,20 +21,19 @@ public:
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
, ScoreComponentCollection &estimatedFutureScore) const {
}
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
}
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{
, const TranslationOptionList &translationOptionList) const {
vector<float> newScores(m_numScoreComponents);
newScores[0] = translationOptionList.size();
@ -52,16 +50,16 @@ public:
}
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
void SetParameter(const std::string& key, const std::string& value)
{}
void SetParameter(const std::string& key, const std::string& value) {
}
};

11
moses/FF/SoftMatchingFeature.cpp
View File

@ -87,7 +87,8 @@ void SoftMatchingFeature::EvaluateWhenApplied(const ChartHypothesis& hypo,
}
// when loading, or when we notice that non-terminals have been added after loading, we resize vectors
void SoftMatchingFeature::ResizeCache() const {
void SoftMatchingFeature::ResizeCache() const
{
FactorCollection& fc = FactorCollection::Instance();
size_t numNonTerminals = fc.GetNumNonTerminals();
@ -98,7 +99,8 @@ void SoftMatchingFeature::ResizeCache() const {
}
const std::string& SoftMatchingFeature::GetOrSetFeatureName(const Word& RHS, const Word& LHS) const {
const std::string& SoftMatchingFeature::GetOrSetFeatureName(const Word& RHS, const Word& LHS) const
{
try {
#ifdef WITH_THREADS //try read-only lock
boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
@ -107,8 +109,7 @@ const std::string& SoftMatchingFeature::GetOrSetFeatureName(const Word& RHS, con
if (!name.empty()) {
return name;
}
}
catch (const std::out_of_range& oor) {
} catch (const std::out_of_range& oor) {
#ifdef WITH_THREADS //need to resize cache; write lock
boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
#endif
@ -123,7 +124,7 @@ const std::string& SoftMatchingFeature::GetOrSetFeatureName(const Word& RHS, con
std::string RHS_string = RHS.GetString(outputFactorOrder, false);
name = LHS_string + "->" + RHS_string;
return name;
}
}
}

4
moses/FF/SoftMatchingFeature.h
View File

@ -34,8 +34,8 @@ public:
, ScoreComponentCollection *estimatedFutureScore = NULL) const {};
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
, const TranslationOptionList &translationOptionList) const {
}
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const {};

Some files were not shown because too many files have changed in this diff Show More