mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
beautify
This commit is contained in:
parent
91cb549ccf
commit
05ead45e71
@ -113,13 +113,11 @@ public:
|
|||||||
|
|
||||||
virtual void DebugPrint(std::ostream &out, const Vocab &vocab) const;
|
virtual void DebugPrint(std::ostream &out, const Vocab &vocab) const;
|
||||||
|
|
||||||
void SetProperty(const std::string &value)
|
void SetProperty(const std::string &value) {
|
||||||
{
|
|
||||||
m_property = value;
|
m_property = value;
|
||||||
}
|
}
|
||||||
|
|
||||||
void SetSparseFeatures(const std::string &value)
|
void SetSparseFeatures(const std::string &value) {
|
||||||
{
|
|
||||||
m_sparseFeatures = value;
|
m_sparseFeatures = value;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -107,8 +107,7 @@ void Word::ConvertToMoses(
|
|||||||
if (m_isNonTerminal) {
|
if (m_isNonTerminal) {
|
||||||
const std::string &tok = vocab.GetString(m_vocabId);
|
const std::string &tok = vocab.GetString(m_vocabId);
|
||||||
overwrite.SetFactor(0, factorColl.AddFactor(tok, m_isNonTerminal));
|
overwrite.SetFactor(0, factorColl.AddFactor(tok, m_isNonTerminal));
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
// TODO: this conversion should have been done at load time.
|
// TODO: this conversion should have been done at load time.
|
||||||
util::TokenIter<util::SingleCharacter> tok(vocab.GetString(m_vocabId), '|');
|
util::TokenIter<util::SingleCharacter> tok(vocab.GetString(m_vocabId), '|');
|
||||||
|
|
||||||
|
@ -7,7 +7,8 @@ size_t lookup( string );
|
|||||||
vector<string> tokenize( const char input[] );
|
vector<string> tokenize( const char input[] );
|
||||||
SuffixArray suffixArray;
|
SuffixArray suffixArray;
|
||||||
|
|
||||||
int main(int argc, char* argv[]) {
|
int main(int argc, char* argv[])
|
||||||
|
{
|
||||||
// handle parameters
|
// handle parameters
|
||||||
string query;
|
string query;
|
||||||
string fileNameSuffix;
|
string fileNameSuffix;
|
||||||
@ -95,14 +96,14 @@ int main(int argc, char* argv[]) {
|
|||||||
}
|
}
|
||||||
cout << lookup( query ) << endl;
|
cout << lookup( query ) << endl;
|
||||||
}
|
}
|
||||||
}
|
} else if (queryFlag) {
|
||||||
else if (queryFlag) {
|
|
||||||
cout << lookup( query ) << endl;
|
cout << lookup( query ) << endl;
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t lookup( string query ) {
|
size_t lookup( string query )
|
||||||
|
{
|
||||||
cerr << "query is " << query << endl;
|
cerr << "query is " << query << endl;
|
||||||
vector< string > queryString = tokenize( query.c_str() );
|
vector< string > queryString = tokenize( query.c_str() );
|
||||||
return suffixArray.Count( queryString );
|
return suffixArray.Count( queryString );
|
||||||
|
@ -61,7 +61,8 @@ void SparseVector::set(const string& name, FeatureStatsType value)
|
|||||||
m_fvector[id] = value;
|
m_fvector[id] = value;
|
||||||
}
|
}
|
||||||
|
|
||||||
void SparseVector::set(size_t id, FeatureStatsType value) {
|
void SparseVector::set(size_t id, FeatureStatsType value)
|
||||||
|
{
|
||||||
assert(m_id_to_name.size() > id);
|
assert(m_id_to_name.size() > id);
|
||||||
m_fvector[id] = value;
|
m_fvector[id] = value;
|
||||||
}
|
}
|
||||||
|
@ -31,9 +31,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
|
||||||
namespace MosesTuning {
|
namespace MosesTuning
|
||||||
|
{
|
||||||
|
|
||||||
std::ostream& operator<<(std::ostream& out, const WordVec& wordVec) {
|
std::ostream& operator<<(std::ostream& out, const WordVec& wordVec)
|
||||||
|
{
|
||||||
out << "[";
|
out << "[";
|
||||||
for (size_t i = 0; i < wordVec.size(); ++i) {
|
for (size_t i = 0; i < wordVec.size(); ++i) {
|
||||||
out << wordVec[i]->first;
|
out << wordVec[i]->first;
|
||||||
@ -44,7 +46,8 @@ std::ostream& operator<<(std::ostream& out, const WordVec& wordVec) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void ReferenceSet::Load(const vector<string>& files, Vocab& vocab) {
|
void ReferenceSet::Load(const vector<string>& files, Vocab& vocab)
|
||||||
|
{
|
||||||
for (size_t i = 0; i < files.size(); ++i) {
|
for (size_t i = 0; i < files.size(); ++i) {
|
||||||
util::FilePiece fh(files[i].c_str());
|
util::FilePiece fh(files[i].c_str());
|
||||||
size_t sentenceId = 0;
|
size_t sentenceId = 0;
|
||||||
@ -62,7 +65,8 @@ void ReferenceSet::Load(const vector<string>& files, Vocab& vocab) {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void ReferenceSet::AddLine(size_t sentenceId, const StringPiece& line, Vocab& vocab) {
|
void ReferenceSet::AddLine(size_t sentenceId, const StringPiece& line, Vocab& vocab)
|
||||||
|
{
|
||||||
//cerr << line << endl;
|
//cerr << line << endl;
|
||||||
NgramCounter ngramCounts;
|
NgramCounter ngramCounts;
|
||||||
list<WordVec> openNgrams;
|
list<WordVec> openNgrams;
|
||||||
@ -105,7 +109,8 @@ void ReferenceSet::AddLine(size_t sentenceId, const StringPiece& line, Vocab& vo
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ReferenceSet::NgramMatches(size_t sentenceId, const WordVec& ngram, bool clip) const {
|
size_t ReferenceSet::NgramMatches(size_t sentenceId, const WordVec& ngram, bool clip) const
|
||||||
|
{
|
||||||
const NgramMap& ngramCounts = ngramCounts_.at(sentenceId);
|
const NgramMap& ngramCounts = ngramCounts_.at(sentenceId);
|
||||||
NgramMap::const_iterator ngi = ngramCounts.find(ngram);
|
NgramMap::const_iterator ngi = ngramCounts.find(ngram);
|
||||||
if (ngi == ngramCounts.end()) return 0;
|
if (ngi == ngramCounts.end()) return 0;
|
||||||
@ -114,7 +119,8 @@ size_t ReferenceSet::NgramMatches(size_t sentenceId, const WordVec& ngram, bool
|
|||||||
|
|
||||||
VertexState::VertexState(): bleuStats(kBleuNgramOrder), targetLength(0) {}
|
VertexState::VertexState(): bleuStats(kBleuNgramOrder), targetLength(0) {}
|
||||||
|
|
||||||
void HgBleuScorer::UpdateMatches(const NgramCounter& counts, vector<FeatureStatsType>& bleuStats ) const {
|
void HgBleuScorer::UpdateMatches(const NgramCounter& counts, vector<FeatureStatsType>& bleuStats ) const
|
||||||
|
{
|
||||||
for (NgramCounter::const_iterator ngi = counts.begin(); ngi != counts.end(); ++ngi) {
|
for (NgramCounter::const_iterator ngi = counts.begin(); ngi != counts.end(); ++ngi) {
|
||||||
//cerr << "Checking: " << *ngi << " matches " << references_.NgramMatches(sentenceId_,*ngi,false) << endl;
|
//cerr << "Checking: " << *ngi << " matches " << references_.NgramMatches(sentenceId_,*ngi,false) << endl;
|
||||||
size_t order = ngi->first.size();
|
size_t order = ngi->first.size();
|
||||||
@ -124,7 +130,8 @@ void HgBleuScorer::UpdateMatches(const NgramCounter& counts, vector<FeatureStats
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t HgBleuScorer::GetTargetLength(const Edge& edge) const {
|
size_t HgBleuScorer::GetTargetLength(const Edge& edge) const
|
||||||
|
{
|
||||||
size_t targetLength = 0;
|
size_t targetLength = 0;
|
||||||
for (size_t i = 0; i < edge.Words().size(); ++i) {
|
for (size_t i = 0; i < edge.Words().size(); ++i) {
|
||||||
const Vocab::Entry* word = edge.Words()[i];
|
const Vocab::Entry* word = edge.Words()[i];
|
||||||
@ -137,7 +144,8 @@ size_t HgBleuScorer::GetTargetLength(const Edge& edge) const {
|
|||||||
return targetLength;
|
return targetLength;
|
||||||
}
|
}
|
||||||
|
|
||||||
FeatureStatsType HgBleuScorer::Score(const Edge& edge, const Vertex& head, vector<FeatureStatsType>& bleuStats) {
|
FeatureStatsType HgBleuScorer::Score(const Edge& edge, const Vertex& head, vector<FeatureStatsType>& bleuStats)
|
||||||
|
{
|
||||||
NgramCounter ngramCounts;
|
NgramCounter ngramCounts;
|
||||||
size_t childId = 0;
|
size_t childId = 0;
|
||||||
size_t wordId = 0;
|
size_t wordId = 0;
|
||||||
@ -242,7 +250,8 @@ FeatureStatsType HgBleuScorer::Score(const Edge& edge, const Vertex& head, vecto
|
|||||||
return bleu;
|
return bleu;
|
||||||
}
|
}
|
||||||
|
|
||||||
void HgBleuScorer::UpdateState(const Edge& winnerEdge, size_t vertexId, const vector<FeatureStatsType>& bleuStats) {
|
void HgBleuScorer::UpdateState(const Edge& winnerEdge, size_t vertexId, const vector<FeatureStatsType>& bleuStats)
|
||||||
|
{
|
||||||
//TODO: Maybe more efficient to absorb into the Score() method
|
//TODO: Maybe more efficient to absorb into the Score() method
|
||||||
VertexState& vertexState = vertexStates_[vertexId];
|
VertexState& vertexState = vertexStates_[vertexId];
|
||||||
//cerr << "Updating state for " << vertexId << endl;
|
//cerr << "Updating state for " << vertexId << endl;
|
||||||
@ -314,7 +323,8 @@ typedef pair<const Edge*,FeatureStatsType> BackPointer;
|
|||||||
* Recurse through back pointers
|
* Recurse through back pointers
|
||||||
**/
|
**/
|
||||||
static void GetBestHypothesis(size_t vertexId, const Graph& graph, const vector<BackPointer>& bps,
|
static void GetBestHypothesis(size_t vertexId, const Graph& graph, const vector<BackPointer>& bps,
|
||||||
HgHypothesis* bestHypo) {
|
HgHypothesis* bestHypo)
|
||||||
|
{
|
||||||
//cerr << "Expanding " << vertexId << " Score: " << bps[vertexId].second << endl;
|
//cerr << "Expanding " << vertexId << " Score: " << bps[vertexId].second << endl;
|
||||||
//UTIL_THROW_IF(bps[vertexId].second == kMinScore+1, HypergraphException, "Landed at vertex " << vertexId << " which is a dead end");
|
//UTIL_THROW_IF(bps[vertexId].second == kMinScore+1, HypergraphException, "Landed at vertex " << vertexId << " which is a dead end");
|
||||||
if (!bps[vertexId].first) return;
|
if (!bps[vertexId].first) return;
|
||||||
|
@ -27,7 +27,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
#include "BleuScorer.h"
|
#include "BleuScorer.h"
|
||||||
#include "Hypergraph.h"
|
#include "Hypergraph.h"
|
||||||
|
|
||||||
namespace MosesTuning {
|
namespace MosesTuning
|
||||||
|
{
|
||||||
|
|
||||||
std::ostream& operator<<(std::ostream& out, const WordVec& wordVec);
|
std::ostream& operator<<(std::ostream& out, const WordVec& wordVec);
|
||||||
|
|
||||||
@ -47,7 +48,8 @@ struct NgramEquals : public std::binary_function<const WordVec&, const WordVec&,
|
|||||||
typedef boost::unordered_map<WordVec, size_t, NgramHash, NgramEquals> NgramCounter;
|
typedef boost::unordered_map<WordVec, size_t, NgramHash, NgramEquals> NgramCounter;
|
||||||
|
|
||||||
|
|
||||||
class ReferenceSet {
|
class ReferenceSet
|
||||||
|
{
|
||||||
|
|
||||||
|
|
||||||
public:
|
public:
|
||||||
@ -58,7 +60,9 @@ public:
|
|||||||
|
|
||||||
size_t NgramMatches(size_t sentenceId, const WordVec&, bool clip) const;
|
size_t NgramMatches(size_t sentenceId, const WordVec&, bool clip) const;
|
||||||
|
|
||||||
size_t Length(size_t sentenceId) const {return lengths_[sentenceId];}
|
size_t Length(size_t sentenceId) const {
|
||||||
|
return lengths_[sentenceId];
|
||||||
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
//ngrams to (clipped,unclipped) counts
|
//ngrams to (clipped,unclipped) counts
|
||||||
@ -80,7 +84,8 @@ struct VertexState {
|
|||||||
/**
|
/**
|
||||||
* Used to score an rule (ie edge) when we are applying it.
|
* Used to score an rule (ie edge) when we are applying it.
|
||||||
**/
|
**/
|
||||||
class HgBleuScorer {
|
class HgBleuScorer
|
||||||
|
{
|
||||||
public:
|
public:
|
||||||
HgBleuScorer(const ReferenceSet& references, const Graph& graph, size_t sentenceId, const std::vector<FeatureStatsType>& backgroundBleu):
|
HgBleuScorer(const ReferenceSet& references, const Graph& graph, size_t sentenceId, const std::vector<FeatureStatsType>& backgroundBleu):
|
||||||
references_(references), sentenceId_(sentenceId), graph_(graph), backgroundBleu_(backgroundBleu),
|
references_(references), sentenceId_(sentenceId), graph_(graph), backgroundBleu_(backgroundBleu),
|
||||||
|
@ -34,11 +34,13 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
using namespace std;
|
using namespace std;
|
||||||
namespace fs = boost::filesystem;
|
namespace fs = boost::filesystem;
|
||||||
|
|
||||||
namespace MosesTuning {
|
namespace MosesTuning
|
||||||
|
{
|
||||||
|
|
||||||
static const ValType BLEU_RATIO = 5;
|
static const ValType BLEU_RATIO = 5;
|
||||||
|
|
||||||
ValType HopeFearDecoder::Evaluate(const AvgWeightVector& wv) {
|
ValType HopeFearDecoder::Evaluate(const AvgWeightVector& wv)
|
||||||
|
{
|
||||||
vector<ValType> stats(scorer_->NumberOfScores(),0);
|
vector<ValType> stats(scorer_->NumberOfScores(),0);
|
||||||
for(reset(); !finished(); next()) {
|
for(reset(); !finished(); next()) {
|
||||||
vector<ValType> sent;
|
vector<ValType> sent;
|
||||||
@ -57,7 +59,8 @@ NbestHopeFearDecoder::NbestHopeFearDecoder(
|
|||||||
bool no_shuffle,
|
bool no_shuffle,
|
||||||
bool safe_hope,
|
bool safe_hope,
|
||||||
Scorer* scorer
|
Scorer* scorer
|
||||||
) : safe_hope_(safe_hope) {
|
) : safe_hope_(safe_hope)
|
||||||
|
{
|
||||||
scorer_ = scorer;
|
scorer_ = scorer;
|
||||||
if (streaming) {
|
if (streaming) {
|
||||||
train_.reset(new StreamingHypPackEnumerator(featureFiles, scoreFiles));
|
train_.reset(new StreamingHypPackEnumerator(featureFiles, scoreFiles));
|
||||||
@ -67,15 +70,18 @@ NbestHopeFearDecoder::NbestHopeFearDecoder(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void NbestHopeFearDecoder::next() {
|
void NbestHopeFearDecoder::next()
|
||||||
|
{
|
||||||
train_->next();
|
train_->next();
|
||||||
}
|
}
|
||||||
|
|
||||||
bool NbestHopeFearDecoder::finished() {
|
bool NbestHopeFearDecoder::finished()
|
||||||
|
{
|
||||||
return train_->finished();
|
return train_->finished();
|
||||||
}
|
}
|
||||||
|
|
||||||
void NbestHopeFearDecoder::reset() {
|
void NbestHopeFearDecoder::reset()
|
||||||
|
{
|
||||||
train_->reset();
|
train_->reset();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -83,7 +89,8 @@ void NbestHopeFearDecoder::HopeFear(
|
|||||||
const std::vector<ValType>& backgroundBleu,
|
const std::vector<ValType>& backgroundBleu,
|
||||||
const MiraWeightVector& wv,
|
const MiraWeightVector& wv,
|
||||||
HopeFearData* hopeFear
|
HopeFearData* hopeFear
|
||||||
) {
|
)
|
||||||
|
{
|
||||||
|
|
||||||
|
|
||||||
// Hope / fear decode
|
// Hope / fear decode
|
||||||
@ -134,7 +141,8 @@ void NbestHopeFearDecoder::HopeFear(
|
|||||||
hopeFear->hopeFearEqual = (hope_index == fear_index);
|
hopeFear->hopeFearEqual = (hope_index == fear_index);
|
||||||
}
|
}
|
||||||
|
|
||||||
void NbestHopeFearDecoder::MaxModel(const AvgWeightVector& wv, std::vector<ValType>* stats) {
|
void NbestHopeFearDecoder::MaxModel(const AvgWeightVector& wv, std::vector<ValType>* stats)
|
||||||
|
{
|
||||||
// Find max model
|
// Find max model
|
||||||
size_t max_index=0;
|
size_t max_index=0;
|
||||||
ValType max_score=0;
|
ValType max_score=0;
|
||||||
@ -163,7 +171,8 @@ HypergraphHopeFearDecoder::HypergraphHopeFearDecoder
|
|||||||
const MiraWeightVector& wv,
|
const MiraWeightVector& wv,
|
||||||
Scorer* scorer
|
Scorer* scorer
|
||||||
) :
|
) :
|
||||||
num_dense_(num_dense) {
|
num_dense_(num_dense)
|
||||||
|
{
|
||||||
|
|
||||||
UTIL_THROW_IF(streaming, util::Exception, "Streaming not currently supported for hypergraphs");
|
UTIL_THROW_IF(streaming, util::Exception, "Streaming not currently supported for hypergraphs");
|
||||||
UTIL_THROW_IF(!fs::exists(hypergraphDir), HypergraphException, "Directory '" << hypergraphDir << "' does not exist");
|
UTIL_THROW_IF(!fs::exists(hypergraphDir), HypergraphException, "Directory '" << hypergraphDir << "' does not exist");
|
||||||
@ -211,15 +220,18 @@ HypergraphHopeFearDecoder::HypergraphHopeFearDecoder
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void HypergraphHopeFearDecoder::reset() {
|
void HypergraphHopeFearDecoder::reset()
|
||||||
|
{
|
||||||
sentenceIdIter_ = sentenceIds_.begin();
|
sentenceIdIter_ = sentenceIds_.begin();
|
||||||
}
|
}
|
||||||
|
|
||||||
void HypergraphHopeFearDecoder::next() {
|
void HypergraphHopeFearDecoder::next()
|
||||||
|
{
|
||||||
sentenceIdIter_++;
|
sentenceIdIter_++;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool HypergraphHopeFearDecoder::finished() {
|
bool HypergraphHopeFearDecoder::finished()
|
||||||
|
{
|
||||||
return sentenceIdIter_ == sentenceIds_.end();
|
return sentenceIdIter_ == sentenceIds_.end();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -227,7 +239,8 @@ void HypergraphHopeFearDecoder::HopeFear(
|
|||||||
const vector<ValType>& backgroundBleu,
|
const vector<ValType>& backgroundBleu,
|
||||||
const MiraWeightVector& wv,
|
const MiraWeightVector& wv,
|
||||||
HopeFearData* hopeFear
|
HopeFearData* hopeFear
|
||||||
) {
|
)
|
||||||
|
{
|
||||||
size_t sentenceId = *sentenceIdIter_;
|
size_t sentenceId = *sentenceIdIter_;
|
||||||
SparseVector weights;
|
SparseVector weights;
|
||||||
wv.ToSparse(&weights);
|
wv.ToSparse(&weights);
|
||||||
@ -319,7 +332,8 @@ void HypergraphHopeFearDecoder::HopeFear(
|
|||||||
hopeFear->hopeFearEqual = hopeFear->hopeFearEqual && (hopeFear->fearFeatures == hopeFear->hopeFeatures);
|
hopeFear->hopeFearEqual = hopeFear->hopeFearEqual && (hopeFear->fearFeatures == hopeFear->hopeFeatures);
|
||||||
}
|
}
|
||||||
|
|
||||||
void HypergraphHopeFearDecoder::MaxModel(const AvgWeightVector& wv, vector<ValType>* stats) {
|
void HypergraphHopeFearDecoder::MaxModel(const AvgWeightVector& wv, vector<ValType>* stats)
|
||||||
|
{
|
||||||
assert(!finished());
|
assert(!finished());
|
||||||
HgHypothesis bestHypo;
|
HgHypothesis bestHypo;
|
||||||
size_t sentenceId = *sentenceIdIter_;
|
size_t sentenceId = *sentenceIdIter_;
|
||||||
|
@ -35,7 +35,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
// the n-best list and lattice/hypergraph implementations
|
// the n-best list and lattice/hypergraph implementations
|
||||||
//
|
//
|
||||||
|
|
||||||
namespace MosesTuning {
|
namespace MosesTuning
|
||||||
|
{
|
||||||
|
|
||||||
class Scorer;
|
class Scorer;
|
||||||
|
|
||||||
@ -55,7 +56,8 @@ struct HopeFearData {
|
|||||||
};
|
};
|
||||||
|
|
||||||
//Abstract base class
|
//Abstract base class
|
||||||
class HopeFearDecoder {
|
class HopeFearDecoder
|
||||||
|
{
|
||||||
public:
|
public:
|
||||||
//iterator methods
|
//iterator methods
|
||||||
virtual void reset() = 0;
|
virtual void reset() = 0;
|
||||||
@ -86,7 +88,8 @@ protected:
|
|||||||
|
|
||||||
|
|
||||||
/** Gets hope-fear from nbest lists */
|
/** Gets hope-fear from nbest lists */
|
||||||
class NbestHopeFearDecoder : public virtual HopeFearDecoder {
|
class NbestHopeFearDecoder : public virtual HopeFearDecoder
|
||||||
|
{
|
||||||
public:
|
public:
|
||||||
NbestHopeFearDecoder(const std::vector<std::string>& featureFiles,
|
NbestHopeFearDecoder(const std::vector<std::string>& featureFiles,
|
||||||
const std::vector<std::string>& scoreFiles,
|
const std::vector<std::string>& scoreFiles,
|
||||||
@ -117,7 +120,8 @@ private:
|
|||||||
|
|
||||||
|
|
||||||
/** Gets hope-fear from hypergraphs */
|
/** Gets hope-fear from hypergraphs */
|
||||||
class HypergraphHopeFearDecoder : public virtual HopeFearDecoder {
|
class HypergraphHopeFearDecoder : public virtual HopeFearDecoder
|
||||||
|
{
|
||||||
public:
|
public:
|
||||||
HypergraphHopeFearDecoder(
|
HypergraphHopeFearDecoder(
|
||||||
const std::string& hypergraphDir,
|
const std::string& hypergraphDir,
|
||||||
|
@ -55,7 +55,8 @@ void HwcmScorer::setReferenceFiles(const vector<string>& referenceFiles)
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void HwcmScorer::extractHeadWordChain(TreePointer tree, vector<string> & history, vector<map<string, int> > & hwc) {
|
void HwcmScorer::extractHeadWordChain(TreePointer tree, vector<string> & history, vector<map<string, int> > & hwc)
|
||||||
|
{
|
||||||
|
|
||||||
if (tree->GetLength() > 0) {
|
if (tree->GetLength() > 0) {
|
||||||
string head = getHead(tree);
|
string head = getHead(tree);
|
||||||
@ -64,8 +65,7 @@ void HwcmScorer::extractHeadWordChain(TreePointer tree, vector<string> & history
|
|||||||
for (std::vector<TreePointer>::const_iterator it = tree->GetChildren().begin(); it != tree->GetChildren().end(); ++it) {
|
for (std::vector<TreePointer>::const_iterator it = tree->GetChildren().begin(); it != tree->GetChildren().end(); ++it) {
|
||||||
extractHeadWordChain(*it, history, hwc);
|
extractHeadWordChain(*it, history, hwc);
|
||||||
}
|
}
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
vector<string> new_history(kHwcmOrder);
|
vector<string> new_history(kHwcmOrder);
|
||||||
new_history[0] = head;
|
new_history[0] = head;
|
||||||
hwc[0][head]++;
|
hwc[0][head]++;
|
||||||
@ -85,11 +85,11 @@ void HwcmScorer::extractHeadWordChain(TreePointer tree, vector<string> & history
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
string HwcmScorer::getHead(TreePointer tree) {
|
string HwcmScorer::getHead(TreePointer tree)
|
||||||
|
{
|
||||||
// assumption (only true for dependency parse: each constituent has a preterminal label, and corresponding terminal is head)
|
// assumption (only true for dependency parse: each constituent has a preterminal label, and corresponding terminal is head)
|
||||||
// if constituent has multiple preterminals, first one is picked; if it has no preterminals, empty string is returned
|
// if constituent has multiple preterminals, first one is picked; if it has no preterminals, empty string is returned
|
||||||
for (std::vector<TreePointer>::const_iterator it = tree->GetChildren().begin(); it != tree->GetChildren().end(); ++it)
|
for (std::vector<TreePointer>::const_iterator it = tree->GetChildren().begin(); it != tree->GetChildren().end(); ++it) {
|
||||||
{
|
|
||||||
TreePointer child = *it;
|
TreePointer child = *it;
|
||||||
|
|
||||||
if (child->GetLength() == 1 && child->GetChildren()[0]->IsTerminal()) {
|
if (child->GetLength() == 1 && child->GetChildren()[0]->IsTerminal()) {
|
||||||
|
@ -31,18 +31,22 @@ using namespace std;
|
|||||||
static const string kBOS = "<s>";
|
static const string kBOS = "<s>";
|
||||||
static const string kEOS = "</s>";
|
static const string kEOS = "</s>";
|
||||||
|
|
||||||
namespace MosesTuning {
|
namespace MosesTuning
|
||||||
|
{
|
||||||
|
|
||||||
StringPiece NextLine(util::FilePiece& from) {
|
StringPiece NextLine(util::FilePiece& from)
|
||||||
|
{
|
||||||
StringPiece line;
|
StringPiece line;
|
||||||
while ((line = from.ReadLine()).starts_with("#"));
|
while ((line = from.ReadLine()).starts_with("#"));
|
||||||
return line;
|
return line;
|
||||||
}
|
}
|
||||||
|
|
||||||
Vocab::Vocab() : eos_( FindOrAdd(kEOS)), bos_(FindOrAdd(kBOS)){
|
Vocab::Vocab() : eos_( FindOrAdd(kEOS)), bos_(FindOrAdd(kBOS))
|
||||||
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
const Vocab::Entry &Vocab::FindOrAdd(const StringPiece &str) {
|
const Vocab::Entry &Vocab::FindOrAdd(const StringPiece &str)
|
||||||
|
{
|
||||||
#if BOOST_VERSION >= 104200
|
#if BOOST_VERSION >= 104200
|
||||||
Map::const_iterator i= map_.find(str, Hash(), Equals());
|
Map::const_iterator i= map_.find(str, Hash(), Equals());
|
||||||
#else
|
#else
|
||||||
@ -62,7 +66,8 @@ double_conversion::StringToDoubleConverter converter(double_conversion::StringTo
|
|||||||
/**
|
/**
|
||||||
* Reads an incoming edge. Returns edge and source words covered.
|
* Reads an incoming edge. Returns edge and source words covered.
|
||||||
**/
|
**/
|
||||||
static pair<Edge*,size_t> ReadEdge(util::FilePiece &from, Graph &graph) {
|
static pair<Edge*,size_t> ReadEdge(util::FilePiece &from, Graph &graph)
|
||||||
|
{
|
||||||
Edge* edge = graph.NewEdge();
|
Edge* edge = graph.NewEdge();
|
||||||
StringPiece line = from.ReadLine(); //Don't allow comments within edge lists
|
StringPiece line = from.ReadLine(); //Don't allow comments within edge lists
|
||||||
util::TokenIter<util::MultiCharacter> pipes(line, util::MultiCharacter(" ||| "));
|
util::TokenIter<util::MultiCharacter> pipes(line, util::MultiCharacter(" ||| "));
|
||||||
@ -103,7 +108,8 @@ static pair<Edge*,size_t> ReadEdge(util::FilePiece &from, Graph &graph) {
|
|||||||
return pair<Edge*,size_t>(edge,sourceCovered);
|
return pair<Edge*,size_t>(edge,sourceCovered);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Graph::Prune(Graph* pNewGraph, const SparseVector& weights, size_t minEdgeCount) const {
|
void Graph::Prune(Graph* pNewGraph, const SparseVector& weights, size_t minEdgeCount) const
|
||||||
|
{
|
||||||
|
|
||||||
Graph& newGraph = *pNewGraph;
|
Graph& newGraph = *pNewGraph;
|
||||||
//TODO: Optimise case where no pruning required
|
//TODO: Optimise case where no pruning required
|
||||||
@ -283,7 +289,8 @@ void Graph::Prune(Graph* pNewGraph, const SparseVector& weights, size_t minEdgeC
|
|||||||
/**
|
/**
|
||||||
* Read from "Kenneth's hypergraph" aka cdec target_graph format (with comments)
|
* Read from "Kenneth's hypergraph" aka cdec target_graph format (with comments)
|
||||||
**/
|
**/
|
||||||
void ReadGraph(util::FilePiece &from, Graph &graph) {
|
void ReadGraph(util::FilePiece &from, Graph &graph)
|
||||||
|
{
|
||||||
|
|
||||||
//First line should contain field names
|
//First line should contain field names
|
||||||
StringPiece line = from.ReadLine();
|
StringPiece line = from.ReadLine();
|
||||||
@ -306,7 +313,9 @@ void ReadGraph(util::FilePiece &from, Graph &graph) {
|
|||||||
vertex->AddEdge(edge.first);
|
vertex->AddEdge(edge.first);
|
||||||
//Note: the file format attaches this to the edge, but it's really a property
|
//Note: the file format attaches this to the edge, but it's really a property
|
||||||
//of the vertex.
|
//of the vertex.
|
||||||
if (!e) {vertex->SetSourceCovered(edge.second);}
|
if (!e) {
|
||||||
|
vertex->SetSourceCovered(edge.second);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -37,13 +37,15 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
|
|
||||||
#include "FeatureStats.h"
|
#include "FeatureStats.h"
|
||||||
|
|
||||||
namespace MosesTuning {
|
namespace MosesTuning
|
||||||
|
{
|
||||||
|
|
||||||
typedef unsigned int WordIndex;
|
typedef unsigned int WordIndex;
|
||||||
const WordIndex kMaxWordIndex = UINT_MAX;
|
const WordIndex kMaxWordIndex = UINT_MAX;
|
||||||
const FeatureStatsType kMinScore = -1e10;
|
const FeatureStatsType kMinScore = -1e10;
|
||||||
|
|
||||||
template <class T> class FixedAllocator : boost::noncopyable {
|
template <class T> class FixedAllocator : boost::noncopyable
|
||||||
|
{
|
||||||
public:
|
public:
|
||||||
FixedAllocator() : current_(NULL), end_(NULL) {}
|
FixedAllocator() : current_(NULL), end_(NULL) {}
|
||||||
|
|
||||||
@ -81,7 +83,8 @@ template <class T> class FixedAllocator : boost::noncopyable {
|
|||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
class Vocab {
|
class Vocab
|
||||||
|
{
|
||||||
public:
|
public:
|
||||||
Vocab();
|
Vocab();
|
||||||
|
|
||||||
@ -89,9 +92,13 @@ class Vocab {
|
|||||||
|
|
||||||
const Entry &FindOrAdd(const StringPiece &str);
|
const Entry &FindOrAdd(const StringPiece &str);
|
||||||
|
|
||||||
const Entry& Bos() const {return bos_;}
|
const Entry& Bos() const {
|
||||||
|
return bos_;
|
||||||
|
}
|
||||||
|
|
||||||
const Entry& Eos() const {return eos_;}
|
const Entry& Eos() const {
|
||||||
|
return eos_;
|
||||||
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
util::Pool piece_backing_;
|
util::Pool piece_backing_;
|
||||||
@ -125,9 +132,12 @@ typedef boost::shared_ptr<SparseVector> FeaturePtr;
|
|||||||
/**
|
/**
|
||||||
* An edge has 1 head vertex, 0..n child (tail) vertices, a list of words and a feature vector.
|
* An edge has 1 head vertex, 0..n child (tail) vertices, a list of words and a feature vector.
|
||||||
**/
|
**/
|
||||||
class Edge {
|
class Edge
|
||||||
|
{
|
||||||
public:
|
public:
|
||||||
Edge() {features_.reset(new SparseVector());}
|
Edge() {
|
||||||
|
features_.reset(new SparseVector());
|
||||||
|
}
|
||||||
|
|
||||||
void AddWord(const Vocab::Entry *word) {
|
void AddWord(const Vocab::Entry *word) {
|
||||||
words_.push_back(word);
|
words_.push_back(word);
|
||||||
@ -173,17 +183,26 @@ class Edge {
|
|||||||
/*
|
/*
|
||||||
* A vertex has 0..n incoming edges
|
* A vertex has 0..n incoming edges
|
||||||
**/
|
**/
|
||||||
class Vertex {
|
class Vertex
|
||||||
|
{
|
||||||
public:
|
public:
|
||||||
Vertex() : sourceCovered_(0) {}
|
Vertex() : sourceCovered_(0) {}
|
||||||
|
|
||||||
void AddEdge(const Edge* edge) {incoming_.push_back(edge);}
|
void AddEdge(const Edge* edge) {
|
||||||
|
incoming_.push_back(edge);
|
||||||
|
}
|
||||||
|
|
||||||
void SetSourceCovered(size_t sourceCovered) {sourceCovered_ = sourceCovered;}
|
void SetSourceCovered(size_t sourceCovered) {
|
||||||
|
sourceCovered_ = sourceCovered;
|
||||||
|
}
|
||||||
|
|
||||||
const std::vector<const Edge*>& GetIncoming() const {return incoming_;}
|
const std::vector<const Edge*>& GetIncoming() const {
|
||||||
|
return incoming_;
|
||||||
|
}
|
||||||
|
|
||||||
size_t SourceCovered() const {return sourceCovered_;}
|
size_t SourceCovered() const {
|
||||||
|
return sourceCovered_;
|
||||||
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
std::vector<const Edge*> incoming_;
|
std::vector<const Edge*> incoming_;
|
||||||
@ -191,7 +210,8 @@ class Vertex {
|
|||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
class Graph : boost::noncopyable {
|
class Graph : boost::noncopyable
|
||||||
|
{
|
||||||
public:
|
public:
|
||||||
Graph(Vocab& vocab) : vocab_(vocab) {}
|
Graph(Vocab& vocab) : vocab_(vocab) {}
|
||||||
|
|
||||||
@ -200,7 +220,9 @@ class Graph : boost::noncopyable {
|
|||||||
edges_.Init(edges);
|
edges_.Init(edges);
|
||||||
}
|
}
|
||||||
|
|
||||||
Vocab &MutableVocab() { return vocab_; }
|
Vocab &MutableVocab() {
|
||||||
|
return vocab_;
|
||||||
|
}
|
||||||
|
|
||||||
Edge *NewEdge() {
|
Edge *NewEdge() {
|
||||||
return edges_.New();
|
return edges_.New();
|
||||||
@ -223,8 +245,12 @@ class Graph : boost::noncopyable {
|
|||||||
Colin Cherry */
|
Colin Cherry */
|
||||||
void Prune(Graph* newGraph, const SparseVector& weights, size_t minEdgeCount) const;
|
void Prune(Graph* newGraph, const SparseVector& weights, size_t minEdgeCount) const;
|
||||||
|
|
||||||
std::size_t VertexSize() const { return vertices_.Size(); }
|
std::size_t VertexSize() const {
|
||||||
std::size_t EdgeSize() const { return edges_.Size(); }
|
return vertices_.Size();
|
||||||
|
}
|
||||||
|
std::size_t EdgeSize() const {
|
||||||
|
return edges_.Size();
|
||||||
|
}
|
||||||
|
|
||||||
bool IsBoundary(const Vocab::Entry* word) const {
|
bool IsBoundary(const Vocab::Entry* word) const {
|
||||||
return word->second == vocab_.Bos().second || word->second == vocab_.Eos().second;
|
return word->second == vocab_.Bos().second || word->second == vocab_.Eos().second;
|
||||||
@ -236,7 +262,8 @@ class Graph : boost::noncopyable {
|
|||||||
Vocab& vocab_;
|
Vocab& vocab_;
|
||||||
};
|
};
|
||||||
|
|
||||||
class HypergraphException : public util::Exception {
|
class HypergraphException : public util::Exception
|
||||||
|
{
|
||||||
public:
|
public:
|
||||||
HypergraphException() {}
|
HypergraphException() {}
|
||||||
~HypergraphException() throw() {}
|
~HypergraphException() throw() {}
|
||||||
|
@ -9,7 +9,8 @@ namespace MosesTuning
|
|||||||
{
|
{
|
||||||
|
|
||||||
|
|
||||||
void MiraFeatureVector::InitSparse(const SparseVector& sparse, size_t ignoreLimit) {
|
void MiraFeatureVector::InitSparse(const SparseVector& sparse, size_t ignoreLimit)
|
||||||
|
{
|
||||||
vector<size_t> sparseFeats = sparse.feats();
|
vector<size_t> sparseFeats = sparse.feats();
|
||||||
bool bFirst = true;
|
bool bFirst = true;
|
||||||
size_t lastFeat = 0;
|
size_t lastFeat = 0;
|
||||||
@ -40,7 +41,8 @@ MiraFeatureVector::MiraFeatureVector(const FeatureDataItem& vec)
|
|||||||
InitSparse(vec.sparse);
|
InitSparse(vec.sparse);
|
||||||
}
|
}
|
||||||
|
|
||||||
MiraFeatureVector::MiraFeatureVector(const SparseVector& sparse, size_t num_dense) {
|
MiraFeatureVector::MiraFeatureVector(const SparseVector& sparse, size_t num_dense)
|
||||||
|
{
|
||||||
m_dense.resize(num_dense);
|
m_dense.resize(num_dense);
|
||||||
//Assume that features with id [0,num_dense) are the dense features
|
//Assume that features with id [0,num_dense) are the dense features
|
||||||
for (size_t id = 0; id < num_dense; ++id) {
|
for (size_t id = 0; id < num_dense; ++id) {
|
||||||
@ -162,7 +164,8 @@ MiraFeatureVector operator-(const MiraFeatureVector& a, const MiraFeatureVector&
|
|||||||
return MiraFeatureVector(dense,sparseFeats,sparseVals);
|
return MiraFeatureVector(dense,sparseFeats,sparseVals);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool operator==(const MiraFeatureVector& a,const MiraFeatureVector& b) {
|
bool operator==(const MiraFeatureVector& a,const MiraFeatureVector& b)
|
||||||
|
{
|
||||||
ValType eps = 1e-8;
|
ValType eps = 1e-8;
|
||||||
//dense features
|
//dense features
|
||||||
if (a.m_dense.size() != b.m_dense.size()) return false;
|
if (a.m_dense.size() != b.m_dense.size()) return false;
|
||||||
|
@ -93,7 +93,8 @@ void MiraWeightVector::update(size_t index, ValType delta)
|
|||||||
m_lastUpdated[index] = m_numUpdates;
|
m_lastUpdated[index] = m_numUpdates;
|
||||||
}
|
}
|
||||||
|
|
||||||
void MiraWeightVector::ToSparse(SparseVector* sparse) const {
|
void MiraWeightVector::ToSparse(SparseVector* sparse) const
|
||||||
|
{
|
||||||
for (size_t i = 0; i < m_weights.size(); ++i) {
|
for (size_t i = 0; i < m_weights.size(); ++i) {
|
||||||
if(abs(m_weights[i])>1e-8) {
|
if(abs(m_weights[i])>1e-8) {
|
||||||
sparse->set(i,m_weights[i]);
|
sparse->set(i,m_weights[i]);
|
||||||
@ -171,7 +172,8 @@ size_t AvgWeightVector::size() const
|
|||||||
return m_wv.m_weights.size();
|
return m_wv.m_weights.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
void AvgWeightVector::ToSparse(SparseVector* sparse) const {
|
void AvgWeightVector::ToSparse(SparseVector* sparse) const
|
||||||
|
{
|
||||||
for (size_t i = 0; i < size(); ++i) {
|
for (size_t i = 0; i < size(); ++i) {
|
||||||
ValType w = weight(i);
|
ValType w = weight(i);
|
||||||
if(abs(w)>1e-8) {
|
if(abs(w)>1e-8) {
|
||||||
|
@ -45,11 +45,9 @@ namespace HashMapSpace
|
|||||||
{
|
{
|
||||||
long foundKey;
|
long foundKey;
|
||||||
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
|
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey= ( *l_hasher ).getHashKey();
|
foundKey= ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -60,11 +58,9 @@ namespace HashMapSpace
|
|||||||
long searchKey=hashValue ( key );
|
long searchKey=hashValue ( key );
|
||||||
long foundKey;;
|
long foundKey;;
|
||||||
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
|
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey= ( *l_hasher ).getHashKey();
|
foundKey= ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -90,8 +86,7 @@ namespace HashMapSpace
|
|||||||
*/
|
*/
|
||||||
void hashMap::addHasher ( string key, string value )
|
void hashMap::addHasher ( string key, string value )
|
||||||
{
|
{
|
||||||
if ( trouve ( hashValue ( key ) ) ==0 )
|
if ( trouve ( hashValue ( key ) ) ==0 ) {
|
||||||
{
|
|
||||||
// cerr << "ICI1" <<endl;
|
// cerr << "ICI1" <<endl;
|
||||||
stringHasher H ( hashValue ( key ),key,value );
|
stringHasher H ( hashValue ( key ),key,value );
|
||||||
// cerr <<" "<< hashValue ( key )<<" "<< key<<" "<<value <<endl;
|
// cerr <<" "<< hashValue ( key )<<" "<< key<<" "<<value <<endl;
|
||||||
@ -106,11 +101,9 @@ namespace HashMapSpace
|
|||||||
long foundKey;
|
long foundKey;
|
||||||
stringHasher defaut(0,"","");
|
stringHasher defaut(0,"","");
|
||||||
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
|
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey= ( *l_hasher ).getHashKey();
|
foundKey= ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
return ( *l_hasher );
|
return ( *l_hasher );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -121,11 +114,9 @@ namespace HashMapSpace
|
|||||||
long searchKey=hashValue ( key );
|
long searchKey=hashValue ( key );
|
||||||
long foundKey;
|
long foundKey;
|
||||||
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
|
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey= ( *l_hasher ).getHashKey();
|
foundKey= ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
// cerr <<"value found : " << key<<"|"<< ( *l_hasher ).getValue()<<endl;
|
// cerr <<"value found : " << key<<"|"<< ( *l_hasher ).getValue()<<endl;
|
||||||
return ( *l_hasher ).getValue();
|
return ( *l_hasher ).getValue();
|
||||||
}
|
}
|
||||||
@ -139,11 +130,9 @@ namespace HashMapSpace
|
|||||||
string foundValue;
|
string foundValue;
|
||||||
|
|
||||||
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
|
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundValue= ( *l_hasher ).getValue();
|
foundValue= ( *l_hasher ).getValue();
|
||||||
if ( foundValue.compare ( value ) == 0 )
|
if ( foundValue.compare ( value ) == 0 ) {
|
||||||
{
|
|
||||||
return ( *l_hasher ).getKey();
|
return ( *l_hasher ).getKey();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -156,11 +145,9 @@ namespace HashMapSpace
|
|||||||
long searchKey=hashValue ( key );
|
long searchKey=hashValue ( key );
|
||||||
long foundKey;
|
long foundKey;
|
||||||
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<stringHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
|
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey= ( *l_hasher ).getHashKey();
|
foundKey= ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
( *l_hasher ).setValue ( value );
|
( *l_hasher ).setValue ( value );
|
||||||
// return ( *l_hasher ).getValue();
|
// return ( *l_hasher ).getValue();
|
||||||
}
|
}
|
||||||
@ -173,8 +160,7 @@ namespace HashMapSpace
|
|||||||
*/
|
*/
|
||||||
void hashMap::printHash()
|
void hashMap::printHash()
|
||||||
{
|
{
|
||||||
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
|
for ( vector<stringHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
|
cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -45,11 +45,9 @@ namespace HashMapSpace
|
|||||||
{
|
{
|
||||||
long foundKey;
|
long foundKey;
|
||||||
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
|
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey= ( *l_hasher ).getHashKey();
|
foundKey= ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -60,11 +58,9 @@ namespace HashMapSpace
|
|||||||
long searchKey=hashValue ( key );
|
long searchKey=hashValue ( key );
|
||||||
long foundKey;;
|
long foundKey;;
|
||||||
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
|
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey= ( *l_hasher ).getHashKey();
|
foundKey= ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -91,8 +87,7 @@ namespace HashMapSpace
|
|||||||
*/
|
*/
|
||||||
void hashMapInfos::addHasher ( string key, vector<int> value )
|
void hashMapInfos::addHasher ( string key, vector<int> value )
|
||||||
{
|
{
|
||||||
if ( trouve ( hashValue ( key ) ) ==0 )
|
if ( trouve ( hashValue ( key ) ) ==0 ) {
|
||||||
{
|
|
||||||
// cerr << "ICI1" <<endl;
|
// cerr << "ICI1" <<endl;
|
||||||
infosHasher H ( hashValue ( key ),key,value );
|
infosHasher H ( hashValue ( key ),key,value );
|
||||||
// cerr <<" "<< hashValue ( key )<<" "<< key<<" "<<value <<endl;
|
// cerr <<" "<< hashValue ( key )<<" "<< key<<" "<<value <<endl;
|
||||||
@ -110,11 +105,9 @@ namespace HashMapSpace
|
|||||||
long searchKey=hashValue ( key );
|
long searchKey=hashValue ( key );
|
||||||
long foundKey;
|
long foundKey;
|
||||||
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
|
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey= ( *l_hasher ).getHashKey();
|
foundKey= ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
return ( *l_hasher );
|
return ( *l_hasher );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -128,11 +121,9 @@ namespace HashMapSpace
|
|||||||
long foundKey;
|
long foundKey;
|
||||||
vector<int> retour;
|
vector<int> retour;
|
||||||
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
|
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey= ( *l_hasher ).getHashKey();
|
foundKey= ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
// cerr <<"value found : " << key<<"|"<< ( *l_hasher ).getValue()<<endl;
|
// cerr <<"value found : " << key<<"|"<< ( *l_hasher ).getValue()<<endl;
|
||||||
return ( *l_hasher ).getValue();
|
return ( *l_hasher ).getValue();
|
||||||
}
|
}
|
||||||
@ -163,11 +154,9 @@ namespace HashMapSpace
|
|||||||
long searchKey=hashValue ( key );
|
long searchKey=hashValue ( key );
|
||||||
long foundKey;
|
long foundKey;
|
||||||
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<infosHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
|
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey= ( *l_hasher ).getHashKey();
|
foundKey= ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
( *l_hasher ).setValue ( value );
|
( *l_hasher ).setValue ( value );
|
||||||
// return ( *l_hasher ).getValue();
|
// return ( *l_hasher ).getValue();
|
||||||
}
|
}
|
||||||
@ -176,8 +165,7 @@ namespace HashMapSpace
|
|||||||
string hashMapInfos::toString ()
|
string hashMapInfos::toString ()
|
||||||
{
|
{
|
||||||
stringstream to_return;
|
stringstream to_return;
|
||||||
for ( vector<infosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
|
for ( vector<infosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
to_return << (*l_hasher).toString();
|
to_return << (*l_hasher).toString();
|
||||||
// cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
|
// cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
|
||||||
}
|
}
|
||||||
@ -189,8 +177,7 @@ namespace HashMapSpace
|
|||||||
*/
|
*/
|
||||||
void hashMapInfos::printHash()
|
void hashMapInfos::printHash()
|
||||||
{
|
{
|
||||||
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ )
|
for ( vector<infosHasher>:: iterator l_hasher=m_hasher.begin() ; l_hasher!=m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
// cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
|
// cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -45,11 +45,9 @@ namespace HashMapSpace
|
|||||||
{
|
{
|
||||||
long foundKey;
|
long foundKey;
|
||||||
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
|
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey = ( *l_hasher ).getHashKey();
|
foundKey = ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -61,11 +59,9 @@ namespace HashMapSpace
|
|||||||
long searchKey = hashValue ( key );
|
long searchKey = hashValue ( key );
|
||||||
long foundKey;;
|
long foundKey;;
|
||||||
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
|
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey = ( *l_hasher ).getHashKey();
|
foundKey = ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -92,8 +88,7 @@ namespace HashMapSpace
|
|||||||
*/
|
*/
|
||||||
void hashMapStringInfos::addHasher ( string key, vector<string> value )
|
void hashMapStringInfos::addHasher ( string key, vector<string> value )
|
||||||
{
|
{
|
||||||
if ( trouve ( hashValue ( key ) ) == 0 )
|
if ( trouve ( hashValue ( key ) ) == 0 ) {
|
||||||
{
|
|
||||||
// cerr << "ICI1" <<endl;
|
// cerr << "ICI1" <<endl;
|
||||||
stringInfosHasher H ( hashValue ( key ), key, value );
|
stringInfosHasher H ( hashValue ( key ), key, value );
|
||||||
// cerr <<" "<< hashValue ( key )<<" "<< key<<" "<<value <<endl;
|
// cerr <<" "<< hashValue ( key )<<" "<< key<<" "<<value <<endl;
|
||||||
@ -111,11 +106,9 @@ namespace HashMapSpace
|
|||||||
long searchKey = hashValue ( key );
|
long searchKey = hashValue ( key );
|
||||||
long foundKey;
|
long foundKey;
|
||||||
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
|
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey = ( *l_hasher ).getHashKey();
|
foundKey = ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
return ( *l_hasher );
|
return ( *l_hasher );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -129,11 +122,9 @@ namespace HashMapSpace
|
|||||||
long foundKey;
|
long foundKey;
|
||||||
vector<string> retour;
|
vector<string> retour;
|
||||||
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
|
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey = ( *l_hasher ).getHashKey();
|
foundKey = ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
// cerr <<"value found : " << key<<"|"<< ( *l_hasher ).getValue()<<endl;
|
// cerr <<"value found : " << key<<"|"<< ( *l_hasher ).getValue()<<endl;
|
||||||
return ( *l_hasher ).getValue();
|
return ( *l_hasher ).getValue();
|
||||||
}
|
}
|
||||||
@ -164,11 +155,9 @@ namespace HashMapSpace
|
|||||||
long searchKey = hashValue ( key );
|
long searchKey = hashValue ( key );
|
||||||
long foundKey;
|
long foundKey;
|
||||||
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
|
// vector<stringInfosHasher>::const_iterator l_hasher=m_hasher.begin();
|
||||||
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
|
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
foundKey = ( *l_hasher ).getHashKey();
|
foundKey = ( *l_hasher ).getHashKey();
|
||||||
if ( searchKey == foundKey )
|
if ( searchKey == foundKey ) {
|
||||||
{
|
|
||||||
( *l_hasher ).setValue ( value );
|
( *l_hasher ).setValue ( value );
|
||||||
// return ( *l_hasher ).getValue();
|
// return ( *l_hasher ).getValue();
|
||||||
}
|
}
|
||||||
@ -178,8 +167,7 @@ namespace HashMapSpace
|
|||||||
string hashMapStringInfos::toString ()
|
string hashMapStringInfos::toString ()
|
||||||
{
|
{
|
||||||
stringstream to_return;
|
stringstream to_return;
|
||||||
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
|
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
to_return << (*l_hasher).toString();
|
to_return << (*l_hasher).toString();
|
||||||
// cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
|
// cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
|
||||||
}
|
}
|
||||||
@ -191,8 +179,7 @@ namespace HashMapSpace
|
|||||||
*/
|
*/
|
||||||
void hashMapStringInfos::printHash()
|
void hashMapStringInfos::printHash()
|
||||||
{
|
{
|
||||||
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ )
|
for ( vector<stringInfosHasher>:: iterator l_hasher = m_hasher.begin() ; l_hasher != m_hasher.end() ; l_hasher++ ) {
|
||||||
{
|
|
||||||
// cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
|
// cout << ( *l_hasher ).getHashKey() <<" | "<< ( *l_hasher ).getKey() << " | " << ( *l_hasher ).getValue() << endl;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -52,12 +52,10 @@ namespace TERCpp
|
|||||||
// s << "Hyp After Shift: " << join ( " ", aftershift );
|
// s << "Hyp After Shift: " << join ( " ", aftershift );
|
||||||
s << endl;
|
s << endl;
|
||||||
// string s = "Original Ref: " + join(" ", ref) + "\nOriginal Hyp: " + join(" ", hyp) + "\nHyp After Shift: " + join(" ", aftershift);
|
// string s = "Original Ref: " + join(" ", ref) + "\nOriginal Hyp: " + join(" ", hyp) + "\nHyp After Shift: " + join(" ", aftershift);
|
||||||
if ( ( int ) sizeof ( alignment ) >0 )
|
if ( ( int ) sizeof ( alignment ) >0 ) {
|
||||||
{
|
|
||||||
s << "Alignment: (";
|
s << "Alignment: (";
|
||||||
// s += "\nAlignment: (";
|
// s += "\nAlignment: (";
|
||||||
for ( int i = 0; i < ( int ) ( alignment.size() ); i++ )
|
for ( int i = 0; i < ( int ) ( alignment.size() ); i++ ) {
|
||||||
{
|
|
||||||
s << alignment[i];
|
s << alignment[i];
|
||||||
// s+=alignment[i];
|
// s+=alignment[i];
|
||||||
}
|
}
|
||||||
@ -65,17 +63,13 @@ namespace TERCpp
|
|||||||
s << ")";
|
s << ")";
|
||||||
}
|
}
|
||||||
s << endl;
|
s << endl;
|
||||||
if ( ( int ) allshifts.size() == 0 )
|
if ( ( int ) allshifts.size() == 0 ) {
|
||||||
{
|
|
||||||
// s += "\nNumShifts: 0";
|
// s += "\nNumShifts: 0";
|
||||||
s << "NumShifts: 0";
|
s << "NumShifts: 0";
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
// s += "\nNumShifts: " + (int)allshifts.size();
|
// s += "\nNumShifts: " + (int)allshifts.size();
|
||||||
s << "NumShifts: "<< ( int ) allshifts.size();
|
s << "NumShifts: "<< ( int ) allshifts.size();
|
||||||
for ( int i = 0; i < ( int ) allshifts.size(); i++ )
|
for ( int i = 0; i < ( int ) allshifts.size(); i++ ) {
|
||||||
{
|
|
||||||
s << endl << " " ;
|
s << endl << " " ;
|
||||||
s << ( ( terShift ) allshifts[i] ).toString();
|
s << ( ( terShift ) allshifts[i] ).toString();
|
||||||
// s += "\n " + allshifts[i];
|
// s += "\n " + allshifts[i];
|
||||||
@ -93,14 +87,10 @@ namespace TERCpp
|
|||||||
// String s = new String("");
|
// String s = new String("");
|
||||||
stringstream s;
|
stringstream s;
|
||||||
s.str ( "" );
|
s.str ( "" );
|
||||||
for ( int i = 0; i < ( int ) arr.size(); i++ )
|
for ( int i = 0; i < ( int ) arr.size(); i++ ) {
|
||||||
{
|
if ( i == 0 ) {
|
||||||
if ( i == 0 )
|
|
||||||
{
|
|
||||||
s << arr.at ( i );
|
s << arr.at ( i );
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
s << delim << arr.at ( i );
|
s << delim << arr.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -109,24 +99,20 @@ namespace TERCpp
|
|||||||
}
|
}
|
||||||
double terAlignment::score()
|
double terAlignment::score()
|
||||||
{
|
{
|
||||||
if ( ( numWords <= 0.0 ) && ( numEdits > 0.0 ) )
|
if ( ( numWords <= 0.0 ) && ( numEdits > 0.0 ) ) {
|
||||||
{
|
|
||||||
return 1.0;
|
return 1.0;
|
||||||
}
|
}
|
||||||
if ( numWords <= 0.0 )
|
if ( numWords <= 0.0 ) {
|
||||||
{
|
|
||||||
return 0.0;
|
return 0.0;
|
||||||
}
|
}
|
||||||
return ( double ) numEdits / numWords;
|
return ( double ) numEdits / numWords;
|
||||||
}
|
}
|
||||||
double terAlignment::scoreAv()
|
double terAlignment::scoreAv()
|
||||||
{
|
{
|
||||||
if ( ( averageWords <= 0.0 ) && ( numEdits > 0.0 ) )
|
if ( ( averageWords <= 0.0 ) && ( numEdits > 0.0 ) ) {
|
||||||
{
|
|
||||||
return 1.0;
|
return 1.0;
|
||||||
}
|
}
|
||||||
if ( averageWords <= 0.0 )
|
if ( averageWords <= 0.0 ) {
|
||||||
{
|
|
||||||
return 0.0;
|
return 0.0;
|
||||||
}
|
}
|
||||||
return ( double ) numEdits / averageWords;
|
return ( double ) numEdits / averageWords;
|
||||||
@ -135,21 +121,16 @@ namespace TERCpp
|
|||||||
void terAlignment::scoreDetails()
|
void terAlignment::scoreDetails()
|
||||||
{
|
{
|
||||||
numIns = numDel = numSub = numWsf = numSft = 0;
|
numIns = numDel = numSub = numWsf = numSft = 0;
|
||||||
if((int)allshifts.size()>0)
|
if((int)allshifts.size()>0) {
|
||||||
{
|
for(int i = 0; i < (int)allshifts.size(); ++i) {
|
||||||
for(int i = 0; i < (int)allshifts.size(); ++i)
|
|
||||||
{
|
|
||||||
numWsf += allshifts[i].size();
|
numWsf += allshifts[i].size();
|
||||||
}
|
}
|
||||||
numSft = allshifts.size();
|
numSft = allshifts.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
if((int)alignment.size()>0 )
|
if((int)alignment.size()>0 ) {
|
||||||
{
|
for(int i = 0; i < (int)alignment.size(); ++i) {
|
||||||
for(int i = 0; i < (int)alignment.size(); ++i)
|
switch (alignment[i]) {
|
||||||
{
|
|
||||||
switch (alignment[i])
|
|
||||||
{
|
|
||||||
case 'S':
|
case 'S':
|
||||||
case 'T':
|
case 'T':
|
||||||
numSub++;
|
numSub++;
|
||||||
@ -170,20 +151,15 @@ namespace TERCpp
|
|||||||
string terAlignment::printAlignments()
|
string terAlignment::printAlignments()
|
||||||
{
|
{
|
||||||
stringstream to_return;
|
stringstream to_return;
|
||||||
for(int i = 0; i < (int)alignment.size(); ++i)
|
for(int i = 0; i < (int)alignment.size(); ++i) {
|
||||||
{
|
|
||||||
char alignInfo=alignment.at(i);
|
char alignInfo=alignment.at(i);
|
||||||
if (alignInfo == 'A' )
|
if (alignInfo == 'A' ) {
|
||||||
{
|
|
||||||
alignInfo='A';
|
alignInfo='A';
|
||||||
}
|
}
|
||||||
|
|
||||||
if (i==0)
|
if (i==0) {
|
||||||
{
|
|
||||||
to_return << alignInfo;
|
to_return << alignInfo;
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
to_return << " " << alignInfo;
|
to_return << " " << alignInfo;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -192,17 +168,13 @@ namespace TERCpp
|
|||||||
string terAlignment::printAllShifts()
|
string terAlignment::printAllShifts()
|
||||||
{
|
{
|
||||||
stringstream to_return;
|
stringstream to_return;
|
||||||
if ( ( int ) allshifts.size() == 0 )
|
if ( ( int ) allshifts.size() == 0 ) {
|
||||||
{
|
|
||||||
// s += "\nNumShifts: 0";
|
// s += "\nNumShifts: 0";
|
||||||
to_return << "NbrShifts: 0";
|
to_return << "NbrShifts: 0";
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
// s += "\nNumShifts: " + (int)allshifts.size();
|
// s += "\nNumShifts: " + (int)allshifts.size();
|
||||||
to_return << "NbrShifts: "<< ( int ) allshifts.size();
|
to_return << "NbrShifts: "<< ( int ) allshifts.size();
|
||||||
for ( int i = 0; i < ( int ) allshifts.size(); i++ )
|
for ( int i = 0; i < ( int ) allshifts.size(); i++ ) {
|
||||||
{
|
|
||||||
to_return << "\t" ;
|
to_return << "\t" ;
|
||||||
to_return << ( ( terShift ) allshifts[i] ).toString();
|
to_return << ( ( terShift ) allshifts[i] ).toString();
|
||||||
// s += "\n " + allshifts[i];
|
// s += "\n " + allshifts[i];
|
||||||
|
@ -83,8 +83,7 @@ namespace TERCpp
|
|||||||
stringstream s;
|
stringstream s;
|
||||||
s.str ( "" );
|
s.str ( "" );
|
||||||
s << "[" << start << ", " << end << ", " << moveto << "/" << newloc << "]";
|
s << "[" << start << ", " << end << ", " << moveto << "/" << newloc << "]";
|
||||||
if ( ( int ) shifted.size() > 0 )
|
if ( ( int ) shifted.size() > 0 ) {
|
||||||
{
|
|
||||||
s << " (" << vectorToString ( shifted ) << ")";
|
s << " (" << vectorToString ( shifted ) << ")";
|
||||||
}
|
}
|
||||||
return s.str();
|
return s.str();
|
||||||
@ -93,16 +92,11 @@ namespace TERCpp
|
|||||||
/* The distance of the shift. */
|
/* The distance of the shift. */
|
||||||
int terShift::distance()
|
int terShift::distance()
|
||||||
{
|
{
|
||||||
if ( moveto < start )
|
if ( moveto < start ) {
|
||||||
{
|
|
||||||
return start - moveto;
|
return start - moveto;
|
||||||
}
|
} else if ( moveto > end ) {
|
||||||
else if ( moveto > end )
|
|
||||||
{
|
|
||||||
return moveto - end;
|
return moveto - end;
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
return moveto - start;
|
return moveto - start;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -68,27 +68,19 @@ namespace TERCpp
|
|||||||
s.str ( "" );
|
s.str ( "" );
|
||||||
string stringRef ( "" );
|
string stringRef ( "" );
|
||||||
string stringHyp ( "" );
|
string stringHyp ( "" );
|
||||||
for ( vector<int>::iterator l_it = ref.begin(); l_it != ref.end(); l_it++ )
|
for ( vector<int>::iterator l_it = ref.begin(); l_it != ref.end(); l_it++ ) {
|
||||||
{
|
if ( l_it == ref.begin() ) {
|
||||||
if ( l_it == ref.begin() )
|
|
||||||
{
|
|
||||||
s << ( *l_it );
|
s << ( *l_it );
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
s << " " << ( *l_it );
|
s << " " << ( *l_it );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
stringRef = s.str();
|
stringRef = s.str();
|
||||||
s.str ( "" );
|
s.str ( "" );
|
||||||
for ( vector<int>::iterator l_itHyp = hyp.begin(); l_itHyp != hyp.end(); l_itHyp++ )
|
for ( vector<int>::iterator l_itHyp = hyp.begin(); l_itHyp != hyp.end(); l_itHyp++ ) {
|
||||||
{
|
if ( l_itHyp == hyp.begin() ) {
|
||||||
if ( l_itHyp == hyp.begin() )
|
|
||||||
{
|
|
||||||
s << ( *l_itHyp );
|
s << ( *l_itHyp );
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
s << " " << ( *l_itHyp );
|
s << " " << ( *l_itHyp );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -102,38 +94,27 @@ namespace TERCpp
|
|||||||
{
|
{
|
||||||
hashMap tempHash;
|
hashMap tempHash;
|
||||||
hashMapInfos retour;
|
hashMapInfos retour;
|
||||||
for ( int i = 0; i < ( int ) hyp.size(); i++ )
|
for ( int i = 0; i < ( int ) hyp.size(); i++ ) {
|
||||||
{
|
|
||||||
tempHash.addHasher ( hyp.at ( i ), "" );
|
tempHash.addHasher ( hyp.at ( i ), "" );
|
||||||
}
|
}
|
||||||
bool cor[ref.size() ];
|
bool cor[ref.size() ];
|
||||||
for ( int i = 0; i < ( int ) ref.size(); i++ )
|
for ( int i = 0; i < ( int ) ref.size(); i++ ) {
|
||||||
{
|
if ( tempHash.trouve ( ( string ) ref.at ( i ) ) ) {
|
||||||
if ( tempHash.trouve ( ( string ) ref.at ( i ) ) )
|
|
||||||
{
|
|
||||||
cor[i] = true;
|
cor[i] = true;
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
cor[i] = false;
|
cor[i] = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for ( int start = 0; start < ( int ) ref.size(); start++ )
|
for ( int start = 0; start < ( int ) ref.size(); start++ ) {
|
||||||
{
|
if ( cor[start] ) {
|
||||||
if ( cor[start] )
|
for ( int end = start; ( ( end < ( int ) ref.size() ) && ( end - start <= TAILLE_PERMUT_MAX ) && ( cor[end] ) ); end++ ) {
|
||||||
{
|
|
||||||
for ( int end = start; ( ( end < ( int ) ref.size() ) && ( end - start <= TAILLE_PERMUT_MAX ) && ( cor[end] ) );end++ )
|
|
||||||
{
|
|
||||||
vector<string> ajouter = subVector ( ref, start, end + 1 );
|
vector<string> ajouter = subVector ( ref, start, end + 1 );
|
||||||
string ajouterString = vectorToString ( ajouter );
|
string ajouterString = vectorToString ( ajouter );
|
||||||
vector<int> values = retour.getValue ( ajouterString );
|
vector<int> values = retour.getValue ( ajouterString );
|
||||||
values.push_back ( start );
|
values.push_back ( start );
|
||||||
if ( values.size() > 1 )
|
if ( values.size() > 1 ) {
|
||||||
{
|
|
||||||
retour.setValue ( ajouterString, values );
|
retour.setValue ( ajouterString, values );
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
retour.addValue ( ajouterString, values );
|
retour.addValue ( ajouterString, values );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -144,8 +125,7 @@ namespace TERCpp
|
|||||||
|
|
||||||
bool terCalc::trouverIntersection ( vecInt refSpan, vecInt hypSpan )
|
bool terCalc::trouverIntersection ( vecInt refSpan, vecInt hypSpan )
|
||||||
{
|
{
|
||||||
if ( ( refSpan.at ( 1 ) >= hypSpan.at ( 0 ) ) && ( refSpan.at ( 0 ) <= hypSpan.at ( 1 ) ) )
|
if ( ( refSpan.at ( 1 ) >= hypSpan.at ( 0 ) ) && ( refSpan.at ( 0 ) <= hypSpan.at ( 1 ) ) ) {
|
||||||
{
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
@ -171,17 +151,14 @@ namespace TERCpp
|
|||||||
NBR_BS_APPELS++;
|
NBR_BS_APPELS++;
|
||||||
|
|
||||||
|
|
||||||
for ( i = 0; i <= ( int ) ref.size(); i++ )
|
for ( i = 0; i <= ( int ) ref.size(); i++ ) {
|
||||||
{
|
for ( j = 0; j <= ( int ) hyp.size(); j++ ) {
|
||||||
for ( j = 0; j <= ( int ) hyp.size(); j++ )
|
|
||||||
{
|
|
||||||
S[i][j] = -1.0;
|
S[i][j] = -1.0;
|
||||||
P[i][j] = '0';
|
P[i][j] = '0';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
S[0][0] = 0.0;
|
S[0][0] = 0.0;
|
||||||
for ( j = 0; j <= ( int ) hyp.size(); j++ )
|
for ( j = 0; j <= ( int ) hyp.size(); j++ ) {
|
||||||
{
|
|
||||||
last_best = current_best;
|
last_best = current_best;
|
||||||
current_best = infinite;
|
current_best = infinite;
|
||||||
first_good = current_first_good;
|
first_good = current_first_good;
|
||||||
@ -190,59 +167,43 @@ namespace TERCpp
|
|||||||
cur_last_good = -1;
|
cur_last_good = -1;
|
||||||
last_peak = cur_last_peak;
|
last_peak = cur_last_peak;
|
||||||
cur_last_peak = 0;
|
cur_last_peak = 0;
|
||||||
for ( i = first_good; i <= ( int ) ref.size(); i++ )
|
for ( i = first_good; i <= ( int ) ref.size(); i++ ) {
|
||||||
{
|
if ( i > last_good ) {
|
||||||
if ( i > last_good )
|
|
||||||
{
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if ( S[i][j] < 0 )
|
if ( S[i][j] < 0 ) {
|
||||||
{
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
score = S[i][j];
|
score = S[i][j];
|
||||||
if ( ( j < ( int ) hyp.size() ) && ( score > last_best + TAILLE_BEAM ) )
|
if ( ( j < ( int ) hyp.size() ) && ( score > last_best + TAILLE_BEAM ) ) {
|
||||||
{
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if ( current_first_good == -1 )
|
if ( current_first_good == -1 ) {
|
||||||
{
|
|
||||||
current_first_good = i ;
|
current_first_good = i ;
|
||||||
}
|
}
|
||||||
if ( ( i < ( int ) ref.size() ) && ( j < ( int ) hyp.size() ) )
|
if ( ( i < ( int ) ref.size() ) && ( j < ( int ) hyp.size() ) ) {
|
||||||
{
|
if ( ( int ) refSpans.size() == 0 || ( int ) hypSpans.size() == 0 || trouverIntersection ( refSpans.at ( i ), curHypSpans.at ( j ) ) ) {
|
||||||
if ( ( int ) refSpans.size() == 0 || ( int ) hypSpans.size() == 0 || trouverIntersection ( refSpans.at ( i ), curHypSpans.at ( j ) ) )
|
if ( ( int ) ( ref.at ( i ).compare ( hyp.at ( j ) ) ) == 0 ) {
|
||||||
{
|
|
||||||
if ( ( int ) ( ref.at ( i ).compare ( hyp.at ( j ) ) ) == 0 )
|
|
||||||
{
|
|
||||||
cost = match_cost + score;
|
cost = match_cost + score;
|
||||||
if ( ( S[i+1][j+1] == -1 ) || ( cost < S[i+1][j+1] ) )
|
if ( ( S[i+1][j+1] == -1 ) || ( cost < S[i+1][j+1] ) ) {
|
||||||
{
|
|
||||||
S[i+1][j+1] = cost;
|
S[i+1][j+1] = cost;
|
||||||
P[i+1][j+1] = 'A';
|
P[i+1][j+1] = 'A';
|
||||||
}
|
}
|
||||||
if ( cost < current_best )
|
if ( cost < current_best ) {
|
||||||
{
|
|
||||||
current_best = cost;
|
current_best = cost;
|
||||||
}
|
}
|
||||||
if ( current_best == cost )
|
if ( current_best == cost ) {
|
||||||
{
|
|
||||||
cur_last_peak = i + 1;
|
cur_last_peak = i + 1;
|
||||||
}
|
}
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
cost = substitute_cost + score;
|
cost = substitute_cost + score;
|
||||||
if ( ( S[i+1][j+1] < 0 ) || ( cost < S[i+1][j+1] ) )
|
if ( ( S[i+1][j+1] < 0 ) || ( cost < S[i+1][j+1] ) ) {
|
||||||
{
|
|
||||||
S[i+1][j+1] = cost;
|
S[i+1][j+1] = cost;
|
||||||
P[i+1][j+1] = 'S';
|
P[i+1][j+1] = 'S';
|
||||||
if ( cost < current_best )
|
if ( cost < current_best ) {
|
||||||
{
|
|
||||||
current_best = cost;
|
current_best = cost;
|
||||||
}
|
}
|
||||||
if ( current_best == cost )
|
if ( current_best == cost ) {
|
||||||
{
|
|
||||||
cur_last_peak = i + 1 ;
|
cur_last_peak = i + 1 ;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -250,28 +211,22 @@ namespace TERCpp
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
cur_last_good = i + 1;
|
cur_last_good = i + 1;
|
||||||
if ( j < ( int ) hyp.size() )
|
if ( j < ( int ) hyp.size() ) {
|
||||||
{
|
|
||||||
icost = score + insert_cost;
|
icost = score + insert_cost;
|
||||||
if ( ( S[i][j+1] < 0 ) || ( S[i][j+1] > icost ) )
|
if ( ( S[i][j+1] < 0 ) || ( S[i][j+1] > icost ) ) {
|
||||||
{
|
|
||||||
S[i][j+1] = icost;
|
S[i][j+1] = icost;
|
||||||
P[i][j+1] = 'I';
|
P[i][j+1] = 'I';
|
||||||
if ( ( cur_last_peak < i ) && ( current_best == icost ) )
|
if ( ( cur_last_peak < i ) && ( current_best == icost ) ) {
|
||||||
{
|
|
||||||
cur_last_peak = i;
|
cur_last_peak = i;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ( i < ( int ) ref.size() )
|
if ( i < ( int ) ref.size() ) {
|
||||||
{
|
|
||||||
dcost = score + delete_cost;
|
dcost = score + delete_cost;
|
||||||
if ( ( S[ i+1][ j] < 0.0 ) || ( S[i+1][j] > dcost ) )
|
if ( ( S[ i+1][ j] < 0.0 ) || ( S[i+1][j] > dcost ) ) {
|
||||||
{
|
|
||||||
S[i+1][j] = dcost;
|
S[i+1][j] = dcost;
|
||||||
P[i+1][j] = 'D';
|
P[i+1][j] = 'D';
|
||||||
if ( i >= last_good )
|
if ( i >= last_good ) {
|
||||||
{
|
|
||||||
last_good = i + 1 ;
|
last_good = i + 1 ;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -283,32 +238,19 @@ namespace TERCpp
|
|||||||
int tracelength = 0;
|
int tracelength = 0;
|
||||||
i = ref.size();
|
i = ref.size();
|
||||||
j = hyp.size();
|
j = hyp.size();
|
||||||
while ( ( i > 0 ) || ( j > 0 ) )
|
while ( ( i > 0 ) || ( j > 0 ) ) {
|
||||||
{
|
|
||||||
tracelength++;
|
tracelength++;
|
||||||
if ( P[i][j] == 'A' )
|
if ( P[i][j] == 'A' ) {
|
||||||
{
|
|
||||||
i--;
|
i--;
|
||||||
j--;
|
j--;
|
||||||
}
|
} else if ( P[i][j] == 'S' ) {
|
||||||
else
|
|
||||||
if ( P[i][j] == 'S' )
|
|
||||||
{
|
|
||||||
i--;
|
i--;
|
||||||
j--;
|
j--;
|
||||||
}
|
} else if ( P[i][j] == 'D' ) {
|
||||||
else
|
|
||||||
if ( P[i][j] == 'D' )
|
|
||||||
{
|
|
||||||
i--;
|
i--;
|
||||||
}
|
} else if ( P[i][j] == 'I' ) {
|
||||||
else
|
|
||||||
if ( P[i][j] == 'I' )
|
|
||||||
{
|
|
||||||
j--;
|
j--;
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
cerr << "ERROR : terCalc::minimizeDistanceEdition : Invalid path : " << P[i][j] << endl;
|
cerr << "ERROR : terCalc::minimizeDistanceEdition : Invalid path : " << P[i][j] << endl;
|
||||||
exit ( -1 );
|
exit ( -1 );
|
||||||
}
|
}
|
||||||
@ -316,28 +258,17 @@ namespace TERCpp
|
|||||||
vector<char> path ( tracelength );
|
vector<char> path ( tracelength );
|
||||||
i = ref.size();
|
i = ref.size();
|
||||||
j = hyp.size();
|
j = hyp.size();
|
||||||
while ( ( i > 0 ) || ( j > 0 ) )
|
while ( ( i > 0 ) || ( j > 0 ) ) {
|
||||||
{
|
|
||||||
path[--tracelength] = P[i][j];
|
path[--tracelength] = P[i][j];
|
||||||
if ( P[i][j] == 'A' )
|
if ( P[i][j] == 'A' ) {
|
||||||
{
|
|
||||||
i--;
|
i--;
|
||||||
j--;
|
j--;
|
||||||
}
|
} else if ( P[i][j] == 'S' ) {
|
||||||
else
|
|
||||||
if ( P[i][j] == 'S' )
|
|
||||||
{
|
|
||||||
i--;
|
i--;
|
||||||
j--;
|
j--;
|
||||||
}
|
} else if ( P[i][j] == 'D' ) {
|
||||||
else
|
|
||||||
if ( P[i][j] == 'D' )
|
|
||||||
{
|
|
||||||
i--;
|
i--;
|
||||||
}
|
} else if ( P[i][j] == 'I' ) {
|
||||||
else
|
|
||||||
if ( P[i][j] == 'I' )
|
|
||||||
{
|
|
||||||
j--;
|
j--;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -348,8 +279,7 @@ namespace TERCpp
|
|||||||
to_return.hyp = hyp;
|
to_return.hyp = hyp;
|
||||||
to_return.ref = ref;
|
to_return.ref = ref;
|
||||||
to_return.averageWords = (int)ref.size();
|
to_return.averageWords = (int)ref.size();
|
||||||
if ( PRINT_DEBUG )
|
if ( PRINT_DEBUG ) {
|
||||||
{
|
|
||||||
cerr << "BEGIN DEBUG : terCalc::minimizeDistanceEdition : to_return :" << endl << to_return.toString() << endl << "END DEBUG" << endl;
|
cerr << "BEGIN DEBUG : terCalc::minimizeDistanceEdition : to_return :" << endl << to_return.toString() << endl << "END DEBUG" << endl;
|
||||||
}
|
}
|
||||||
return to_return;
|
return to_return;
|
||||||
@ -369,16 +299,13 @@ namespace TERCpp
|
|||||||
vector<terShift> allshifts;
|
vector<terShift> allshifts;
|
||||||
|
|
||||||
// cerr << "Initial Alignment:" << endl << cur_align.toString() <<endl;
|
// cerr << "Initial Alignment:" << endl << cur_align.toString() <<endl;
|
||||||
if ( PRINT_DEBUG )
|
if ( PRINT_DEBUG ) {
|
||||||
{
|
|
||||||
cerr << "BEGIN DEBUG : terCalc::TER : cur_align :" << endl << cur_align.toString() << endl << "END DEBUG" << endl;
|
cerr << "BEGIN DEBUG : terCalc::TER : cur_align :" << endl << cur_align.toString() << endl << "END DEBUG" << endl;
|
||||||
}
|
}
|
||||||
while ( true )
|
while ( true ) {
|
||||||
{
|
|
||||||
bestShiftStruct returns;
|
bestShiftStruct returns;
|
||||||
returns = findBestShift ( cur, hyp, ref, rloc, cur_align );
|
returns = findBestShift ( cur, hyp, ref, rloc, cur_align );
|
||||||
if ( returns.m_empty )
|
if ( returns.m_empty ) {
|
||||||
{
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
terShift bestShift = returns.m_best_shift;
|
terShift bestShift = returns.m_best_shift;
|
||||||
@ -406,12 +333,10 @@ namespace TERCpp
|
|||||||
calculateTerAlignment ( med_align, herr, rerr, ralign );
|
calculateTerAlignment ( med_align, herr, rerr, ralign );
|
||||||
vector<vecTerShift> poss_shifts;
|
vector<vecTerShift> poss_shifts;
|
||||||
|
|
||||||
if ( PRINT_DEBUG )
|
if ( PRINT_DEBUG ) {
|
||||||
{
|
|
||||||
cerr << "BEGIN DEBUG : terCalc::findBestShift (after the calculateTerAlignment call) :" << endl;
|
cerr << "BEGIN DEBUG : terCalc::findBestShift (after the calculateTerAlignment call) :" << endl;
|
||||||
cerr << "indices: ";
|
cerr << "indices: ";
|
||||||
for (int l_i=0; l_i < ( int ) ref.size() ; l_i++)
|
for (int l_i=0; l_i < ( int ) ref.size() ; l_i++) {
|
||||||
{
|
|
||||||
cerr << l_i << "\t";
|
cerr << l_i << "\t";
|
||||||
}
|
}
|
||||||
cerr << endl;
|
cerr << endl;
|
||||||
@ -425,14 +350,11 @@ namespace TERCpp
|
|||||||
}
|
}
|
||||||
poss_shifts = calculerPermutations ( cur, ref, rloc, med_align, herr, rerr, ralign );
|
poss_shifts = calculerPermutations ( cur, ref, rloc, med_align, herr, rerr, ralign );
|
||||||
double curerr = med_align.numEdits;
|
double curerr = med_align.numEdits;
|
||||||
if ( PRINT_DEBUG )
|
if ( PRINT_DEBUG ) {
|
||||||
{
|
|
||||||
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
|
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
|
||||||
cerr << "Possible Shifts:" << endl;
|
cerr << "Possible Shifts:" << endl;
|
||||||
for ( int i = ( int ) poss_shifts.size() - 1; i >= 0; i-- )
|
for ( int i = ( int ) poss_shifts.size() - 1; i >= 0; i-- ) {
|
||||||
{
|
for ( int j = 0; j < ( int ) ( poss_shifts.at ( i ) ).size(); j++ ) {
|
||||||
for ( int j = 0; j < ( int ) ( poss_shifts.at ( i ) ).size(); j++ )
|
|
||||||
{
|
|
||||||
cerr << " [" << i << "] " << ( ( poss_shifts.at ( i ) ).at ( j ) ).toString() << endl;
|
cerr << " [" << i << "] " << ( ( poss_shifts.at ( i ) ).at ( j ) ).toString() << endl;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -446,10 +368,8 @@ namespace TERCpp
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
for ( int i = ( int ) poss_shifts.size() - 1; i >= 0; i-- )
|
for ( int i = ( int ) poss_shifts.size() - 1; i >= 0; i-- ) {
|
||||||
{
|
if ( PRINT_DEBUG ) {
|
||||||
if ( PRINT_DEBUG )
|
|
||||||
{
|
|
||||||
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
|
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
|
||||||
cerr << "Considering shift of length " << i << " (" << ( poss_shifts.at ( i ) ).size() << ")" << endl;
|
cerr << "Considering shift of length " << i << " (" << ( poss_shifts.at ( i ) ).size() << ")" << endl;
|
||||||
cerr << "END DEBUG " << endl;
|
cerr << "END DEBUG " << endl;
|
||||||
@ -457,21 +377,17 @@ namespace TERCpp
|
|||||||
/* Consider shifts of length i+1 */
|
/* Consider shifts of length i+1 */
|
||||||
double curfix = curerr - ( cur_best_shift_cost + cur_best_align.numEdits );
|
double curfix = curerr - ( cur_best_shift_cost + cur_best_align.numEdits );
|
||||||
double maxfix = ( 2 * ( 1 + i ) );
|
double maxfix = ( 2 * ( 1 + i ) );
|
||||||
if ( ( curfix > maxfix ) || ( ( cur_best_shift_cost != 0 ) && ( curfix == maxfix ) ) )
|
if ( ( curfix > maxfix ) || ( ( cur_best_shift_cost != 0 ) && ( curfix == maxfix ) ) ) {
|
||||||
{
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
for ( int s = 0; s < ( int ) ( poss_shifts.at ( i ) ).size(); s++ )
|
for ( int s = 0; s < ( int ) ( poss_shifts.at ( i ) ).size(); s++ ) {
|
||||||
{
|
|
||||||
curfix = curerr - ( cur_best_shift_cost + cur_best_align.numEdits );
|
curfix = curerr - ( cur_best_shift_cost + cur_best_align.numEdits );
|
||||||
if ( ( curfix > maxfix ) || ( ( cur_best_shift_cost != 0 ) && ( curfix == maxfix ) ) )
|
if ( ( curfix > maxfix ) || ( ( cur_best_shift_cost != 0 ) && ( curfix == maxfix ) ) ) {
|
||||||
{
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
terShift curshift = ( poss_shifts.at ( i ) ).at ( s );
|
terShift curshift = ( poss_shifts.at ( i ) ).at ( s );
|
||||||
if ( PRINT_DEBUG )
|
if ( PRINT_DEBUG ) {
|
||||||
{
|
|
||||||
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
|
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
|
||||||
cerr << "cur : "<< join(" ",cur) << endl;
|
cerr << "cur : "<< join(" ",cur) << endl;
|
||||||
cerr << "curshift : "<< curshift.toString() << endl;
|
cerr << "curshift : "<< curshift.toString() << endl;
|
||||||
@ -481,8 +397,7 @@ namespace TERCpp
|
|||||||
vector<string> shiftarr = shiftReturns.nwords;
|
vector<string> shiftarr = shiftReturns.nwords;
|
||||||
vector<vecInt> curHypSpans = shiftReturns.aftershift;
|
vector<vecInt> curHypSpans = shiftReturns.aftershift;
|
||||||
|
|
||||||
if ( PRINT_DEBUG )
|
if ( PRINT_DEBUG ) {
|
||||||
{
|
|
||||||
cerr << "shiftarr : "<< join(" ",shiftarr) << endl;
|
cerr << "shiftarr : "<< join(" ",shiftarr) << endl;
|
||||||
// cerr << "curHypSpans : "<< curHypSpans.toString() << endl;
|
// cerr << "curHypSpans : "<< curHypSpans.toString() << endl;
|
||||||
cerr << "END DEBUG " << endl;
|
cerr << "END DEBUG " << endl;
|
||||||
@ -498,8 +413,7 @@ namespace TERCpp
|
|||||||
|
|
||||||
// if (DEBUG) {
|
// if (DEBUG) {
|
||||||
// string testeuh=terAlignment join(" ", shiftarr);
|
// string testeuh=terAlignment join(" ", shiftarr);
|
||||||
if ( PRINT_DEBUG )
|
if ( PRINT_DEBUG ) {
|
||||||
{
|
|
||||||
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
|
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
|
||||||
cerr << "Gain for " << curshift.toString() << " is " << gain << ". (result: [" << curalign.join ( " ", shiftarr ) << "]" << endl;
|
cerr << "Gain for " << curshift.toString() << " is " << gain << ". (result: [" << curalign.join ( " ", shiftarr ) << "]" << endl;
|
||||||
cerr << "Details of gains : gain = ( cur_best_align.numEdits + cur_best_shift_cost ) - ( curalign.numEdits + curshift.cost )"<<endl;
|
cerr << "Details of gains : gain = ( cur_best_align.numEdits + cur_best_shift_cost ) - ( curalign.numEdits + curshift.cost )"<<endl;
|
||||||
@ -509,15 +423,13 @@ namespace TERCpp
|
|||||||
}
|
}
|
||||||
// }
|
// }
|
||||||
//
|
//
|
||||||
if ( ( gain > 0 ) || ( ( cur_best_shift_cost == 0 ) && ( gain == 0 ) ) )
|
if ( ( gain > 0 ) || ( ( cur_best_shift_cost == 0 ) && ( gain == 0 ) ) ) {
|
||||||
{
|
|
||||||
anygain = true;
|
anygain = true;
|
||||||
cur_best_shift = curshift;
|
cur_best_shift = curshift;
|
||||||
cur_best_shift_cost = curshift.cost;
|
cur_best_shift_cost = curshift.cost;
|
||||||
cur_best_align = curalign;
|
cur_best_align = curalign;
|
||||||
// if (DEBUG)
|
// if (DEBUG)
|
||||||
if ( PRINT_DEBUG )
|
if ( PRINT_DEBUG ) {
|
||||||
{
|
|
||||||
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
|
cerr << "BEGIN DEBUG : terCalc::findBestShift :" << endl;
|
||||||
cerr << "Tmp Choosing shift: " << cur_best_shift.toString() << " gives:\n" << cur_best_align.toString() << "\n" << endl;
|
cerr << "Tmp Choosing shift: " << cur_best_shift.toString() << " gives:\n" << cur_best_align.toString() << "\n" << endl;
|
||||||
cerr << "END DEBUG " << endl;
|
cerr << "END DEBUG " << endl;
|
||||||
@ -525,14 +437,11 @@ namespace TERCpp
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ( anygain )
|
if ( anygain ) {
|
||||||
{
|
|
||||||
to_return.m_best_shift = cur_best_shift;
|
to_return.m_best_shift = cur_best_shift;
|
||||||
to_return.m_best_align = cur_best_align;
|
to_return.m_best_align = cur_best_align;
|
||||||
to_return.m_empty = false;
|
to_return.m_empty = false;
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
to_return.m_empty = true;
|
to_return.m_empty = true;
|
||||||
}
|
}
|
||||||
return to_return;
|
return to_return;
|
||||||
@ -542,53 +451,38 @@ namespace TERCpp
|
|||||||
{
|
{
|
||||||
int hpos = -1;
|
int hpos = -1;
|
||||||
int rpos = -1;
|
int rpos = -1;
|
||||||
if ( PRINT_DEBUG )
|
if ( PRINT_DEBUG ) {
|
||||||
{
|
|
||||||
|
|
||||||
cerr << "BEGIN DEBUG : terCalc::calculateTerAlignment : " << endl << align.toString() << endl;
|
cerr << "BEGIN DEBUG : terCalc::calculateTerAlignment : " << endl << align.toString() << endl;
|
||||||
cerr << "END DEBUG " << endl;
|
cerr << "END DEBUG " << endl;
|
||||||
}
|
}
|
||||||
for ( int i = 0; i < ( int ) align.alignment.size(); i++ )
|
for ( int i = 0; i < ( int ) align.alignment.size(); i++ ) {
|
||||||
{
|
|
||||||
herr[i] = false;
|
herr[i] = false;
|
||||||
rerr[i] = false;
|
rerr[i] = false;
|
||||||
ralign[i] = -1;
|
ralign[i] = -1;
|
||||||
}
|
}
|
||||||
for ( int i = 0; i < ( int ) align.alignment.size(); i++ )
|
for ( int i = 0; i < ( int ) align.alignment.size(); i++ ) {
|
||||||
{
|
|
||||||
char sym = align.alignment[i];
|
char sym = align.alignment[i];
|
||||||
if ( sym == 'A' )
|
if ( sym == 'A' ) {
|
||||||
{
|
|
||||||
hpos++;
|
hpos++;
|
||||||
rpos++;
|
rpos++;
|
||||||
herr[hpos] = false;
|
herr[hpos] = false;
|
||||||
rerr[rpos] = false;
|
rerr[rpos] = false;
|
||||||
ralign[rpos] = hpos;
|
ralign[rpos] = hpos;
|
||||||
}
|
} else if ( sym == 'S' ) {
|
||||||
else
|
|
||||||
if ( sym == 'S' )
|
|
||||||
{
|
|
||||||
hpos++;
|
hpos++;
|
||||||
rpos++;
|
rpos++;
|
||||||
herr[hpos] = true;
|
herr[hpos] = true;
|
||||||
rerr[rpos] = true;
|
rerr[rpos] = true;
|
||||||
ralign[rpos] = hpos;
|
ralign[rpos] = hpos;
|
||||||
}
|
} else if ( sym == 'I' ) {
|
||||||
else
|
|
||||||
if ( sym == 'I' )
|
|
||||||
{
|
|
||||||
hpos++;
|
hpos++;
|
||||||
herr[hpos] = true;
|
herr[hpos] = true;
|
||||||
}
|
} else if ( sym == 'D' ) {
|
||||||
else
|
|
||||||
if ( sym == 'D' )
|
|
||||||
{
|
|
||||||
rpos++;
|
rpos++;
|
||||||
rerr[rpos] = true;
|
rerr[rpos] = true;
|
||||||
ralign[rpos] = hpos+1;
|
ralign[rpos] = hpos+1;
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
cerr << "ERROR : terCalc::calculateTerAlignment : Invalid mini align sequence " << sym << " at pos " << i << endl;
|
cerr << "ERROR : terCalc::calculateTerAlignment : Invalid mini align sequence " << sym << " at pos " << i << endl;
|
||||||
exit ( -1 );
|
exit ( -1 );
|
||||||
}
|
}
|
||||||
@ -598,58 +492,47 @@ namespace TERCpp
|
|||||||
vector<vecTerShift> terCalc::calculerPermutations ( vector<string> hyp, vector<string> ref, hashMapInfos rloc, terAlignment align, bool* herr, bool* rerr, int* ralign )
|
vector<vecTerShift> terCalc::calculerPermutations ( vector<string> hyp, vector<string> ref, hashMapInfos rloc, terAlignment align, bool* herr, bool* rerr, int* ralign )
|
||||||
{
|
{
|
||||||
vector<vecTerShift> to_return;
|
vector<vecTerShift> to_return;
|
||||||
if ( ( TAILLE_PERMUT_MAX <= 0 ) || ( DIST_MAX_PERMUT <= 0 ) )
|
if ( ( TAILLE_PERMUT_MAX <= 0 ) || ( DIST_MAX_PERMUT <= 0 ) ) {
|
||||||
{
|
|
||||||
return to_return;
|
return to_return;
|
||||||
}
|
}
|
||||||
|
|
||||||
vector<vecTerShift> allshifts ( TAILLE_PERMUT_MAX + 1 );
|
vector<vecTerShift> allshifts ( TAILLE_PERMUT_MAX + 1 );
|
||||||
for ( int start = 0; start < ( int ) hyp.size(); start++ )
|
for ( int start = 0; start < ( int ) hyp.size(); start++ ) {
|
||||||
{
|
|
||||||
string subVectorHypString = vectorToString ( subVector ( hyp, start, start + 1 ) );
|
string subVectorHypString = vectorToString ( subVector ( hyp, start, start + 1 ) );
|
||||||
if ( ! rloc.trouve ( subVectorHypString ) )
|
if ( ! rloc.trouve ( subVectorHypString ) ) {
|
||||||
{
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ok = false;
|
bool ok = false;
|
||||||
vector<int> mtiVec = rloc.getValue ( subVectorHypString );
|
vector<int> mtiVec = rloc.getValue ( subVectorHypString );
|
||||||
vector<int>::iterator mti = mtiVec.begin();
|
vector<int>::iterator mti = mtiVec.begin();
|
||||||
while ( mti != mtiVec.end() && ( ! ok ) )
|
while ( mti != mtiVec.end() && ( ! ok ) ) {
|
||||||
{
|
|
||||||
int moveto = ( *mti );
|
int moveto = ( *mti );
|
||||||
mti++;
|
mti++;
|
||||||
if ( ( start != ralign[moveto] ) && ( ( ralign[moveto] - start ) <= DIST_MAX_PERMUT ) && ( ( start - ralign[moveto] - 1 ) <= DIST_MAX_PERMUT ) )
|
if ( ( start != ralign[moveto] ) && ( ( ralign[moveto] - start ) <= DIST_MAX_PERMUT ) && ( ( start - ralign[moveto] - 1 ) <= DIST_MAX_PERMUT ) ) {
|
||||||
{
|
|
||||||
ok = true;
|
ok = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ( ! ok )
|
if ( ! ok ) {
|
||||||
{
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
ok = true;
|
ok = true;
|
||||||
for ( int end = start; ( ok && ( end < ( int ) hyp.size() ) && ( end < start + TAILLE_PERMUT_MAX ) ); end++ )
|
for ( int end = start; ( ok && ( end < ( int ) hyp.size() ) && ( end < start + TAILLE_PERMUT_MAX ) ); end++ ) {
|
||||||
{
|
|
||||||
/* check if cand is good if so, add it */
|
/* check if cand is good if so, add it */
|
||||||
vector<string> cand = subVector ( hyp, start, end + 1 );
|
vector<string> cand = subVector ( hyp, start, end + 1 );
|
||||||
ok = false;
|
ok = false;
|
||||||
if ( ! ( rloc.trouve ( vectorToString ( cand ) ) ) )
|
if ( ! ( rloc.trouve ( vectorToString ( cand ) ) ) ) {
|
||||||
{
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool any_herr = false;
|
bool any_herr = false;
|
||||||
|
|
||||||
for ( int i = 0; ( ( i <= ( end - start ) ) && ( ! any_herr ) ); i++ )
|
for ( int i = 0; ( ( i <= ( end - start ) ) && ( ! any_herr ) ); i++ ) {
|
||||||
{
|
if ( herr[start+i] ) {
|
||||||
if ( herr[start+i] )
|
|
||||||
{
|
|
||||||
any_herr = true;
|
any_herr = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ( any_herr == false )
|
if ( any_herr == false ) {
|
||||||
{
|
|
||||||
ok = true;
|
ok = true;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@ -658,12 +541,10 @@ namespace TERCpp
|
|||||||
movetoitVec = rloc.getValue ( ( string ) vectorToString ( cand ) );
|
movetoitVec = rloc.getValue ( ( string ) vectorToString ( cand ) );
|
||||||
// cerr << "CANDIDATE " << ( string ) vectorToString ( cand ) <<" PLACED : " << ( string ) vectorToString ( movetoitVec," ") << endl;
|
// cerr << "CANDIDATE " << ( string ) vectorToString ( cand ) <<" PLACED : " << ( string ) vectorToString ( movetoitVec," ") << endl;
|
||||||
vector<int>::iterator movetoit = movetoitVec.begin();
|
vector<int>::iterator movetoit = movetoitVec.begin();
|
||||||
while ( movetoit != movetoitVec.end() )
|
while ( movetoit != movetoitVec.end() ) {
|
||||||
{
|
|
||||||
int moveto = ( *movetoit );
|
int moveto = ( *movetoit );
|
||||||
movetoit++;
|
movetoit++;
|
||||||
if ( ! ( ( ralign[moveto] != start ) && ( ( ralign[moveto] < start ) || ( ralign[moveto] > end ) ) && ( ( ralign[moveto] - start ) <= DIST_MAX_PERMUT ) && ( ( start - ralign[moveto] ) <= DIST_MAX_PERMUT ) ) )
|
if ( ! ( ( ralign[moveto] != start ) && ( ( ralign[moveto] < start ) || ( ralign[moveto] > end ) ) && ( ( ralign[moveto] - start ) <= DIST_MAX_PERMUT ) && ( ( start - ralign[moveto] ) <= DIST_MAX_PERMUT ) ) ) {
|
||||||
{
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
ok = true;
|
ok = true;
|
||||||
@ -673,38 +554,28 @@ namespace TERCpp
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
bool any_rerr = false;
|
bool any_rerr = false;
|
||||||
for ( int i = 0; ( i <= end - start ) && ( ! any_rerr ); i++ )
|
for ( int i = 0; ( i <= end - start ) && ( ! any_rerr ); i++ ) {
|
||||||
{
|
if ( rerr[moveto+i] ) {
|
||||||
if ( rerr[moveto+i] )
|
|
||||||
{
|
|
||||||
any_rerr = true;
|
any_rerr = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ( ! any_rerr )
|
if ( ! any_rerr ) {
|
||||||
{
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
for ( int roff = -1; roff <= ( end - start ); roff++ )
|
for ( int roff = -1; roff <= ( end - start ); roff++ ) {
|
||||||
{
|
|
||||||
terShift topush;
|
terShift topush;
|
||||||
bool topushNull = true;
|
bool topushNull = true;
|
||||||
if ( ( roff == -1 ) && ( moveto == 0 ) )
|
if ( ( roff == -1 ) && ( moveto == 0 ) ) {
|
||||||
{
|
if ( PRINT_DEBUG ) {
|
||||||
if ( PRINT_DEBUG )
|
|
||||||
{
|
|
||||||
|
|
||||||
cerr << "BEGIN DEBUG : terCalc::calculerPermutations 01 : " << endl << "Consider making " << start << "..." << end << " (" << vectorToString(cand," ")<< ") moveto: " << moveto << " roff: " << roff << " ralign[mt+roff]: -1" << endl << "END DEBUG" << endl;
|
cerr << "BEGIN DEBUG : terCalc::calculerPermutations 01 : " << endl << "Consider making " << start << "..." << end << " (" << vectorToString(cand," ")<< ") moveto: " << moveto << " roff: " << roff << " ralign[mt+roff]: -1" << endl << "END DEBUG" << endl;
|
||||||
}
|
}
|
||||||
terShift t01 ( start, end, -1, -1 );
|
terShift t01 ( start, end, -1, -1 );
|
||||||
topush = t01;
|
topush = t01;
|
||||||
topushNull = false;
|
topushNull = false;
|
||||||
}
|
} else if ( ( start != ralign[moveto+roff] ) && ( ( roff == 0 ) || ( ralign[moveto+roff] != ralign[moveto] ) ) ) {
|
||||||
else
|
|
||||||
if ( ( start != ralign[moveto+roff] ) && ( ( roff == 0 ) || ( ralign[moveto+roff] != ralign[moveto] ) ) )
|
|
||||||
{
|
|
||||||
int newloc = ralign[moveto+roff];
|
int newloc = ralign[moveto+roff];
|
||||||
if ( PRINT_DEBUG )
|
if ( PRINT_DEBUG ) {
|
||||||
{
|
|
||||||
|
|
||||||
cerr << "BEGIN DEBUG : terCalc::calculerPermutations 02 : " << endl << "Consider making " << start << "..." << end << " (" << vectorToString(cand," ")<< ") moveto: " << moveto << " roff: " << roff << " ralign[mt+roff]: " << newloc << endl << "END DEBUG" << endl;
|
cerr << "BEGIN DEBUG : terCalc::calculerPermutations 02 : " << endl << "Consider making " << start << "..." << end << " (" << vectorToString(cand," ")<< ") moveto: " << moveto << " roff: " << roff << " ralign[mt+roff]: " << newloc << endl << "END DEBUG" << endl;
|
||||||
}
|
}
|
||||||
@ -712,12 +583,10 @@ namespace TERCpp
|
|||||||
topush = t02;
|
topush = t02;
|
||||||
topushNull = false;
|
topushNull = false;
|
||||||
}
|
}
|
||||||
if ( !topushNull )
|
if ( !topushNull ) {
|
||||||
{
|
|
||||||
topush.shifted = cand;
|
topush.shifted = cand;
|
||||||
topush.cost = shift_cost;
|
topush.cost = shift_cost;
|
||||||
if ( PRINT_DEBUG )
|
if ( PRINT_DEBUG ) {
|
||||||
{
|
|
||||||
|
|
||||||
cerr << "BEGIN DEBUG : terCalc::calculerPermutations 02 : " << endl;
|
cerr << "BEGIN DEBUG : terCalc::calculerPermutations 02 : " << endl;
|
||||||
cerr << "start : " << start << endl;
|
cerr << "start : " << start << endl;
|
||||||
@ -732,8 +601,7 @@ namespace TERCpp
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
to_return.clear();
|
to_return.clear();
|
||||||
for ( int i = 0; i < TAILLE_PERMUT_MAX + 1; i++ )
|
for ( int i = 0; i < TAILLE_PERMUT_MAX + 1; i++ ) {
|
||||||
{
|
|
||||||
to_return.push_back ( ( vecTerShift ) allshifts.at ( i ) );
|
to_return.push_back ( ( vecTerShift ) allshifts.at ( i ) );
|
||||||
}
|
}
|
||||||
return to_return;
|
return to_return;
|
||||||
@ -752,23 +620,17 @@ namespace TERCpp
|
|||||||
vector<string> nwords ( words );
|
vector<string> nwords ( words );
|
||||||
vector<vecInt> spans ( ( int ) hypSpans.size() );
|
vector<vecInt> spans ( ( int ) hypSpans.size() );
|
||||||
alignmentStruct to_return;
|
alignmentStruct to_return;
|
||||||
if ( PRINT_DEBUG )
|
if ( PRINT_DEBUG ) {
|
||||||
{
|
|
||||||
|
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
cerr << "BEGIN DEBUG : terCalc::permuter :" << endl << "word length: " << ( int ) words.size() << " span length: " << ( int ) hypSpans.size() << endl ;
|
cerr << "BEGIN DEBUG : terCalc::permuter :" << endl << "word length: " << ( int ) words.size() << " span length: " << ( int ) hypSpans.size() << endl ;
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
cerr << "BEGIN DEBUG : terCalc::permuter :" << endl << "word length: " << ( int ) words.size() << " span length: null" << endl ;
|
cerr << "BEGIN DEBUG : terCalc::permuter :" << endl << "word length: " << ( int ) words.size() << " span length: null" << endl ;
|
||||||
}
|
}
|
||||||
cerr << "BEGIN DEBUG : terCalc::permuter :" << endl << join(" ",words) << " start: " << start << " end: " << end << " newloc "<< newloc << endl << "END DEBUG " << endl;
|
cerr << "BEGIN DEBUG : terCalc::permuter :" << endl << join(" ",words) << " start: " << start << " end: " << end << " newloc "<< newloc << endl << "END DEBUG " << endl;
|
||||||
}
|
}
|
||||||
if (newloc >= ( int ) words.size())
|
if (newloc >= ( int ) words.size()) {
|
||||||
{
|
if ( PRINT_DEBUG ) {
|
||||||
if ( PRINT_DEBUG )
|
|
||||||
{
|
|
||||||
cerr << "WARNING: Relocation over the size of the hypothesis, replacing at the end of it."<<endl;
|
cerr << "WARNING: Relocation over the size of the hypothesis, replacing at the end of it."<<endl;
|
||||||
}
|
}
|
||||||
newloc = ( int ) words.size()-1;
|
newloc = ( int ) words.size()-1;
|
||||||
@ -776,140 +638,101 @@ namespace TERCpp
|
|||||||
|
|
||||||
// }
|
// }
|
||||||
|
|
||||||
if ( newloc == -1 )
|
if ( newloc == -1 ) {
|
||||||
{
|
for ( int i = start; i <= end; i++ ) {
|
||||||
for ( int i = start; i <= end;i++ )
|
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for ( int i = 0; i <= start - 1;i++ )
|
for ( int i = 0; i <= start - 1; i++ ) {
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for ( int i = end + 1; i < ( int ) words.size();i++ )
|
for ( int i = end + 1; i < ( int ) words.size(); i++ ) {
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
} else {
|
||||||
else
|
if ( newloc < start ) {
|
||||||
{
|
|
||||||
if ( newloc < start )
|
|
||||||
{
|
|
||||||
|
|
||||||
for ( int i = 0; i < newloc; i++ )
|
for ( int i = 0; i < newloc; i++ ) {
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for ( int i = start; i <= end;i++ )
|
for ( int i = start; i <= end; i++ ) {
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for ( int i = newloc ; i < start ;i++ )
|
for ( int i = newloc ; i < start ; i++ ) {
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for ( int i = end + 1; i < ( int ) words.size();i++ )
|
for ( int i = end + 1; i < ( int ) words.size(); i++ ) {
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
} else {
|
||||||
else
|
if ( newloc > end ) {
|
||||||
{
|
for ( int i = 0; i <= start - 1; i++ ) {
|
||||||
if ( newloc > end )
|
|
||||||
{
|
|
||||||
for ( int i = 0; i <= start - 1; i++ )
|
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for ( int i = end + 1; i <= newloc;i++ )
|
for ( int i = end + 1; i <= newloc; i++ ) {
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for ( int i = start; i <= end;i++ )
|
for ( int i = start; i <= end; i++ ) {
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for ( int i = newloc + 1; i < ( int ) words.size();i++ )
|
for ( int i = newloc + 1; i < ( int ) words.size(); i++ ) {
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
// we are moving inside of ourselves
|
// we are moving inside of ourselves
|
||||||
for ( int i = 0; i <= start - 1; i++ )
|
for ( int i = 0; i <= start - 1; i++ ) {
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for ( int i = end + 1; ( i < ( int ) words.size() ) && ( i <= ( end + ( newloc - start ) ) ); i++ )
|
for ( int i = end + 1; ( i < ( int ) words.size() ) && ( i <= ( end + ( newloc - start ) ) ); i++ ) {
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for ( int i = start; i <= end;i++ )
|
for ( int i = start; i <= end; i++ ) {
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for ( int i = ( end + ( newloc - start ) + 1 ); i < ( int ) words.size();i++ )
|
for ( int i = ( end + ( newloc - start ) + 1 ); i < ( int ) words.size(); i++ ) {
|
||||||
{
|
|
||||||
nwords.at ( c++ ) = words.at ( i );
|
nwords.at ( c++ ) = words.at ( i );
|
||||||
if ( ( int ) hypSpans.size() > 0 )
|
if ( ( int ) hypSpans.size() > 0 ) {
|
||||||
{
|
|
||||||
spans.at ( c - 1 ) = hypSpans.at ( i );
|
spans.at ( c - 1 ) = hypSpans.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -918,8 +741,7 @@ namespace TERCpp
|
|||||||
}
|
}
|
||||||
NBR_PERMUTS_CONSID++;
|
NBR_PERMUTS_CONSID++;
|
||||||
|
|
||||||
if ( PRINT_DEBUG )
|
if ( PRINT_DEBUG ) {
|
||||||
{
|
|
||||||
cerr << "nwords" << join(" ",nwords) << endl;
|
cerr << "nwords" << join(" ",nwords) << endl;
|
||||||
// cerr << "spans" << spans. << endl;
|
// cerr << "spans" << spans. << endl;
|
||||||
}
|
}
|
||||||
|
@ -28,14 +28,10 @@ namespace Tools
|
|||||||
string vectorToString ( vector<string> vec )
|
string vectorToString ( vector<string> vec )
|
||||||
{
|
{
|
||||||
string retour ( "" );
|
string retour ( "" );
|
||||||
for ( vector<string>::iterator vecIter = vec.begin();vecIter != vec.end(); vecIter++ )
|
for ( vector<string>::iterator vecIter = vec.begin(); vecIter != vec.end(); vecIter++ ) {
|
||||||
{
|
if ( vecIter == vec.begin() ) {
|
||||||
if ( vecIter == vec.begin() )
|
|
||||||
{
|
|
||||||
retour += ( *vecIter );
|
retour += ( *vecIter );
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
retour += "\t" + ( *vecIter );
|
retour += "\t" + ( *vecIter );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -45,14 +41,10 @@ namespace Tools
|
|||||||
{
|
{
|
||||||
stringstream retour;
|
stringstream retour;
|
||||||
retour.str("");
|
retour.str("");
|
||||||
for ( vector<char>::iterator vecIter = vec.begin();vecIter != vec.end(); vecIter++ )
|
for ( vector<char>::iterator vecIter = vec.begin(); vecIter != vec.end(); vecIter++ ) {
|
||||||
{
|
if ( vecIter == vec.begin() ) {
|
||||||
if ( vecIter == vec.begin() )
|
|
||||||
{
|
|
||||||
retour << ( *vecIter );
|
retour << ( *vecIter );
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
retour << "\t" << ( *vecIter );
|
retour << "\t" << ( *vecIter );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -62,14 +54,10 @@ namespace Tools
|
|||||||
{
|
{
|
||||||
stringstream retour;
|
stringstream retour;
|
||||||
retour.str("");
|
retour.str("");
|
||||||
for ( vector<int>::iterator vecIter = vec.begin();vecIter != vec.end(); vecIter++ )
|
for ( vector<int>::iterator vecIter = vec.begin(); vecIter != vec.end(); vecIter++ ) {
|
||||||
{
|
if ( vecIter == vec.begin() ) {
|
||||||
if ( vecIter == vec.begin() )
|
|
||||||
{
|
|
||||||
retour << ( *vecIter );
|
retour << ( *vecIter );
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
retour << "\t" << ( *vecIter );
|
retour << "\t" << ( *vecIter );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -79,14 +67,10 @@ namespace Tools
|
|||||||
string vectorToString ( vector< string > vec, string s )
|
string vectorToString ( vector< string > vec, string s )
|
||||||
{
|
{
|
||||||
string retour ( "" );
|
string retour ( "" );
|
||||||
for ( vector<string>::iterator vecIter = vec.begin();vecIter != vec.end(); vecIter++ )
|
for ( vector<string>::iterator vecIter = vec.begin(); vecIter != vec.end(); vecIter++ ) {
|
||||||
{
|
if ( vecIter == vec.begin() ) {
|
||||||
if ( vecIter == vec.begin() )
|
|
||||||
{
|
|
||||||
retour += ( *vecIter );
|
retour += ( *vecIter );
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
retour += s + ( *vecIter );
|
retour += s + ( *vecIter );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -98,14 +82,10 @@ namespace Tools
|
|||||||
{
|
{
|
||||||
stringstream retour;
|
stringstream retour;
|
||||||
retour.str("");
|
retour.str("");
|
||||||
for ( vector<char>::iterator vecIter = vec.begin();vecIter != vec.end(); vecIter++ )
|
for ( vector<char>::iterator vecIter = vec.begin(); vecIter != vec.end(); vecIter++ ) {
|
||||||
{
|
if ( vecIter == vec.begin() ) {
|
||||||
if ( vecIter == vec.begin() )
|
|
||||||
{
|
|
||||||
retour << ( *vecIter );
|
retour << ( *vecIter );
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
retour << s << ( *vecIter );
|
retour << s << ( *vecIter );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -117,14 +97,10 @@ namespace Tools
|
|||||||
{
|
{
|
||||||
stringstream retour;
|
stringstream retour;
|
||||||
retour.str("");
|
retour.str("");
|
||||||
for ( vector<int>::iterator vecIter = vec.begin();vecIter != vec.end(); vecIter++ )
|
for ( vector<int>::iterator vecIter = vec.begin(); vecIter != vec.end(); vecIter++ ) {
|
||||||
{
|
if ( vecIter == vec.begin() ) {
|
||||||
if ( vecIter == vec.begin() )
|
|
||||||
{
|
|
||||||
retour << ( *vecIter );
|
retour << ( *vecIter );
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
retour << s << ( *vecIter );
|
retour << s << ( *vecIter );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -136,14 +112,10 @@ namespace Tools
|
|||||||
{
|
{
|
||||||
stringstream retour;
|
stringstream retour;
|
||||||
retour.str("");
|
retour.str("");
|
||||||
for ( vector<bool>::iterator vecIter = vec.begin();vecIter != vec.end(); vecIter++ )
|
for ( vector<bool>::iterator vecIter = vec.begin(); vecIter != vec.end(); vecIter++ ) {
|
||||||
{
|
if ( vecIter == vec.begin() ) {
|
||||||
if ( vecIter == vec.begin() )
|
|
||||||
{
|
|
||||||
retour << ( *vecIter );
|
retour << ( *vecIter );
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
retour << s << ( *vecIter );
|
retour << s << ( *vecIter );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -155,14 +127,10 @@ namespace Tools
|
|||||||
stringstream retour;
|
stringstream retour;
|
||||||
retour.str("");
|
retour.str("");
|
||||||
int l_i;
|
int l_i;
|
||||||
for ( l_i=0; l_i < taille ; l_i++)
|
for ( l_i=0; l_i < taille ; l_i++) {
|
||||||
{
|
if ( l_i == 0 ) {
|
||||||
if ( l_i == 0 )
|
|
||||||
{
|
|
||||||
retour << vec[l_i];
|
retour << vec[l_i];
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
retour << s << vec[l_i];
|
retour << s << vec[l_i];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -175,14 +143,10 @@ namespace Tools
|
|||||||
stringstream retour;
|
stringstream retour;
|
||||||
retour.str("");
|
retour.str("");
|
||||||
int l_i;
|
int l_i;
|
||||||
for ( l_i=0; l_i < taille ; l_i++)
|
for ( l_i=0; l_i < taille ; l_i++) {
|
||||||
{
|
if ( l_i == 0 ) {
|
||||||
if ( l_i == 0 )
|
|
||||||
{
|
|
||||||
retour << vec[l_i];
|
retour << vec[l_i];
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
retour << s << vec[l_i];
|
retour << s << vec[l_i];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -195,14 +159,10 @@ namespace Tools
|
|||||||
stringstream retour;
|
stringstream retour;
|
||||||
retour.str("");
|
retour.str("");
|
||||||
int l_i;
|
int l_i;
|
||||||
for ( l_i=0; l_i < taille ; l_i++)
|
for ( l_i=0; l_i < taille ; l_i++) {
|
||||||
{
|
if ( l_i == 0 ) {
|
||||||
if ( l_i == 0 )
|
|
||||||
{
|
|
||||||
retour << vec[l_i];
|
retour << vec[l_i];
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
retour << s << vec[l_i];
|
retour << s << vec[l_i];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -213,13 +173,11 @@ namespace Tools
|
|||||||
vector<string> subVector ( vector<string> vec, int start, int end )
|
vector<string> subVector ( vector<string> vec, int start, int end )
|
||||||
{
|
{
|
||||||
vector<string> retour;
|
vector<string> retour;
|
||||||
if ( start > end )
|
if ( start > end ) {
|
||||||
{
|
|
||||||
cerr << "ERREUR : TERcalc::subVector : end > start" << endl;
|
cerr << "ERREUR : TERcalc::subVector : end > start" << endl;
|
||||||
exit ( 0 );
|
exit ( 0 );
|
||||||
}
|
}
|
||||||
for ( int i = start; ( ( i < end ) && ( i < ( int ) vec.size() ) ); i++ )
|
for ( int i = start; ( ( i < end ) && ( i < ( int ) vec.size() ) ); i++ ) {
|
||||||
{
|
|
||||||
retour.push_back ( vec.at ( i ) );
|
retour.push_back ( vec.at ( i ) );
|
||||||
}
|
}
|
||||||
return retour;
|
return retour;
|
||||||
@ -228,13 +186,11 @@ namespace Tools
|
|||||||
vector<int> subVector ( vector<int> vec, int start, int end )
|
vector<int> subVector ( vector<int> vec, int start, int end )
|
||||||
{
|
{
|
||||||
vector<int> retour;
|
vector<int> retour;
|
||||||
if ( start > end )
|
if ( start > end ) {
|
||||||
{
|
|
||||||
cerr << "ERREUR : TERcalc::subVector : end > start" << endl;
|
cerr << "ERREUR : TERcalc::subVector : end > start" << endl;
|
||||||
exit ( 0 );
|
exit ( 0 );
|
||||||
}
|
}
|
||||||
for ( int i = start; ( ( i < end ) && ( i < ( int ) vec.size() ) ); i++ )
|
for ( int i = start; ( ( i < end ) && ( i < ( int ) vec.size() ) ); i++ ) {
|
||||||
{
|
|
||||||
retour.push_back ( vec.at ( i ) );
|
retour.push_back ( vec.at ( i ) );
|
||||||
}
|
}
|
||||||
return retour;
|
return retour;
|
||||||
@ -243,13 +199,11 @@ namespace Tools
|
|||||||
vector<float> subVector ( vector<float> vec, int start, int end )
|
vector<float> subVector ( vector<float> vec, int start, int end )
|
||||||
{
|
{
|
||||||
vector<float> retour;
|
vector<float> retour;
|
||||||
if ( start > end )
|
if ( start > end ) {
|
||||||
{
|
|
||||||
cerr << "ERREUR : TERcalc::subVector : end > start" << endl;
|
cerr << "ERREUR : TERcalc::subVector : end > start" << endl;
|
||||||
exit ( 0 );
|
exit ( 0 );
|
||||||
}
|
}
|
||||||
for ( int i = start; ( ( i < end ) && ( i < ( int ) vec.size() ) ); i++ )
|
for ( int i = start; ( ( i < end ) && ( i < ( int ) vec.size() ) ); i++ ) {
|
||||||
{
|
|
||||||
retour.push_back ( vec.at ( i ) );
|
retour.push_back ( vec.at ( i ) );
|
||||||
}
|
}
|
||||||
return retour;
|
return retour;
|
||||||
@ -258,8 +212,7 @@ namespace Tools
|
|||||||
vector<string> copyVector ( vector<string> vec )
|
vector<string> copyVector ( vector<string> vec )
|
||||||
{
|
{
|
||||||
vector<string> retour;
|
vector<string> retour;
|
||||||
for ( int i = 0; i < ( int ) vec.size(); i++ )
|
for ( int i = 0; i < ( int ) vec.size(); i++ ) {
|
||||||
{
|
|
||||||
retour.push_back ( vec.at ( i ) );
|
retour.push_back ( vec.at ( i ) );
|
||||||
}
|
}
|
||||||
return retour;
|
return retour;
|
||||||
@ -267,8 +220,7 @@ namespace Tools
|
|||||||
vector<int> copyVector ( vector<int> vec )
|
vector<int> copyVector ( vector<int> vec )
|
||||||
{
|
{
|
||||||
vector<int> retour;
|
vector<int> retour;
|
||||||
for ( int i = 0; i < ( int ) vec.size(); i++ )
|
for ( int i = 0; i < ( int ) vec.size(); i++ ) {
|
||||||
{
|
|
||||||
retour.push_back ( vec.at ( i ) );
|
retour.push_back ( vec.at ( i ) );
|
||||||
}
|
}
|
||||||
return retour;
|
return retour;
|
||||||
@ -276,8 +228,7 @@ namespace Tools
|
|||||||
vector<float> copyVector ( vector<float> vec )
|
vector<float> copyVector ( vector<float> vec )
|
||||||
{
|
{
|
||||||
vector<float> retour;
|
vector<float> retour;
|
||||||
for ( int i = 0; i < ( int ) vec.size(); i++ )
|
for ( int i = 0; i < ( int ) vec.size(); i++ ) {
|
||||||
{
|
|
||||||
retour.push_back ( vec.at ( i ) );
|
retour.push_back ( vec.at ( i ) );
|
||||||
}
|
}
|
||||||
return retour;
|
return retour;
|
||||||
@ -288,20 +239,16 @@ namespace Tools
|
|||||||
string to_push ( "" );
|
string to_push ( "" );
|
||||||
bool pushed = false;
|
bool pushed = false;
|
||||||
string::iterator sIt;
|
string::iterator sIt;
|
||||||
for ( sIt = s.begin(); sIt < s.end(); sIt++ )
|
for ( sIt = s.begin(); sIt < s.end(); sIt++ ) {
|
||||||
{
|
|
||||||
pushed = false;
|
pushed = false;
|
||||||
for ( string::iterator sTok = tok.begin(); sTok < tok.end(); sTok++ )
|
for ( string::iterator sTok = tok.begin(); sTok < tok.end(); sTok++ ) {
|
||||||
{
|
if ( ( *sIt ) == ( *sTok ) ) {
|
||||||
if ( ( *sIt ) == ( *sTok ) )
|
|
||||||
{
|
|
||||||
to_return.push_back ( to_push );
|
to_return.push_back ( to_push );
|
||||||
to_push = "";
|
to_push = "";
|
||||||
pushed = true;
|
pushed = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ( !pushed )
|
if ( !pushed ) {
|
||||||
{
|
|
||||||
to_push.push_back ( ( *sIt ) );
|
to_push.push_back ( ( *sIt ) );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -314,28 +261,22 @@ namespace Tools
|
|||||||
string to_push ( "" );
|
string to_push ( "" );
|
||||||
bool pushed = false;
|
bool pushed = false;
|
||||||
string::iterator sIt;
|
string::iterator sIt;
|
||||||
for ( sIt = s.begin(); sIt < s.end(); sIt++ )
|
for ( sIt = s.begin(); sIt < s.end(); sIt++ ) {
|
||||||
{
|
|
||||||
pushed = false;
|
pushed = false;
|
||||||
for ( string::iterator sTok = tok.begin(); sTok < tok.end(); sTok++ )
|
for ( string::iterator sTok = tok.begin(); sTok < tok.end(); sTok++ ) {
|
||||||
{
|
if ( ( *sIt ) == ( *sTok ) ) {
|
||||||
if ( ( *sIt ) == ( *sTok ) )
|
if ( ( int ) to_push.length() > 0 ) {
|
||||||
{
|
|
||||||
if ( ( int ) to_push.length() > 0 )
|
|
||||||
{
|
|
||||||
to_return.push_back ( atoi ( to_push.c_str() ) );
|
to_return.push_back ( atoi ( to_push.c_str() ) );
|
||||||
}
|
}
|
||||||
to_push = "";
|
to_push = "";
|
||||||
pushed = true;
|
pushed = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ( !pushed )
|
if ( !pushed ) {
|
||||||
{
|
|
||||||
to_push.push_back ( ( *sIt ) );
|
to_push.push_back ( ( *sIt ) );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ( ( int ) to_push.length() > 0 )
|
if ( ( int ) to_push.length() > 0 ) {
|
||||||
{
|
|
||||||
to_return.push_back ( atoi ( to_push.c_str() ) );
|
to_return.push_back ( atoi ( to_push.c_str() ) );
|
||||||
}
|
}
|
||||||
return to_return;
|
return to_return;
|
||||||
@ -346,28 +287,22 @@ namespace Tools
|
|||||||
string to_push ( "" );
|
string to_push ( "" );
|
||||||
bool pushed = false;
|
bool pushed = false;
|
||||||
string::iterator sIt;
|
string::iterator sIt;
|
||||||
for ( sIt = s.begin(); sIt < s.end(); sIt++ )
|
for ( sIt = s.begin(); sIt < s.end(); sIt++ ) {
|
||||||
{
|
|
||||||
pushed = false;
|
pushed = false;
|
||||||
for ( string::iterator sTok = tok.begin(); sTok < tok.end(); sTok++ )
|
for ( string::iterator sTok = tok.begin(); sTok < tok.end(); sTok++ ) {
|
||||||
{
|
if ( ( *sIt ) == ( *sTok ) ) {
|
||||||
if ( ( *sIt ) == ( *sTok ) )
|
if ( ( int ) to_push.length() > 0 ) {
|
||||||
{
|
|
||||||
if ( ( int ) to_push.length() > 0 )
|
|
||||||
{
|
|
||||||
to_return.push_back ( atof ( to_push.c_str() ) );
|
to_return.push_back ( atof ( to_push.c_str() ) );
|
||||||
}
|
}
|
||||||
to_push = "";
|
to_push = "";
|
||||||
pushed = true;
|
pushed = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ( !pushed )
|
if ( !pushed ) {
|
||||||
{
|
|
||||||
to_push.push_back ( ( *sIt ) );
|
to_push.push_back ( ( *sIt ) );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ( ( int ) to_push.length() > 0 )
|
if ( ( int ) to_push.length() > 0 ) {
|
||||||
{
|
|
||||||
to_return.push_back ( atoi ( to_push.c_str() ) );
|
to_return.push_back ( atoi ( to_push.c_str() ) );
|
||||||
}
|
}
|
||||||
return to_return;
|
return to_return;
|
||||||
@ -375,10 +310,8 @@ namespace Tools
|
|||||||
|
|
||||||
string lowerCase ( string str )
|
string lowerCase ( string str )
|
||||||
{
|
{
|
||||||
for ( int i = 0;i < ( int ) str.size();i++ )
|
for ( int i = 0; i < ( int ) str.size(); i++ ) {
|
||||||
{
|
if ( ( str[i] >= 0x41 ) && ( str[i] <= 0x5A ) ) {
|
||||||
if ( ( str[i] >= 0x41 ) && ( str[i] <= 0x5A ) )
|
|
||||||
{
|
|
||||||
str[i] = str[i] + 0x20;
|
str[i] = str[i] + 0x20;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -754,14 +687,10 @@ namespace Tools
|
|||||||
// String s = new String("");
|
// String s = new String("");
|
||||||
stringstream s;
|
stringstream s;
|
||||||
s.str ( "" );
|
s.str ( "" );
|
||||||
for ( int i = 0; i < ( int ) arr.size(); i++ )
|
for ( int i = 0; i < ( int ) arr.size(); i++ ) {
|
||||||
{
|
if ( i == 0 ) {
|
||||||
if ( i == 0 )
|
|
||||||
{
|
|
||||||
s << arr.at ( i );
|
s << arr.at ( i );
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
s << delim << arr.at ( i );
|
s << delim << arr.at ( i );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -45,8 +45,7 @@ namespace Tools
|
|||||||
typedef vector<alignmentElement> WERalignment;
|
typedef vector<alignmentElement> WERalignment;
|
||||||
|
|
||||||
|
|
||||||
struct param
|
struct param {
|
||||||
{
|
|
||||||
bool debugMode;
|
bool debugMode;
|
||||||
string referenceFile; // path to the resources
|
string referenceFile; // path to the resources
|
||||||
string hypothesisFile; // path to the configuration files
|
string hypothesisFile; // path to the configuration files
|
||||||
|
@ -43,7 +43,8 @@ private:
|
|||||||
};
|
};
|
||||||
|
|
||||||
// load hypothesis from candidate output
|
// load hypothesis from candidate output
|
||||||
vector<ScoreStats> EvaluatorUtil::loadCand(const string& candFile) {
|
vector<ScoreStats> EvaluatorUtil::loadCand(const string& candFile)
|
||||||
|
{
|
||||||
|
|
||||||
ifstream cand(candFile.c_str());
|
ifstream cand(candFile.c_str());
|
||||||
if (!cand.good()) throw runtime_error("Error opening candidate file");
|
if (!cand.good()) throw runtime_error("Error opening candidate file");
|
||||||
@ -61,7 +62,8 @@ vector<ScoreStats> EvaluatorUtil::loadCand(const string& candFile) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// load 1-best hypothesis from n-best file (useful if relying on alignment/tree information)
|
// load 1-best hypothesis from n-best file (useful if relying on alignment/tree information)
|
||||||
vector<ScoreStats> EvaluatorUtil::loadNBest(const string& nBestFile) {
|
vector<ScoreStats> EvaluatorUtil::loadNBest(const string& nBestFile)
|
||||||
|
{
|
||||||
vector<ScoreStats> entries;
|
vector<ScoreStats> entries;
|
||||||
|
|
||||||
Data data(g_scorer);
|
Data data(g_scorer);
|
||||||
@ -81,8 +83,7 @@ void EvaluatorUtil::evaluate(const string& candFile, int bootstrap, bool nbest_i
|
|||||||
|
|
||||||
if (nbest_input) {
|
if (nbest_input) {
|
||||||
entries = loadNBest(candFile);
|
entries = loadNBest(candFile);
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
entries = loadCand(candFile);
|
entries = loadCand(candFile);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3,7 +3,8 @@
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
int main(int argc, char* argv[]){
|
int main(int argc, char* argv[])
|
||||||
|
{
|
||||||
|
|
||||||
const char * is_reordering = "false";
|
const char * is_reordering = "false";
|
||||||
|
|
||||||
|
@ -26,7 +26,8 @@
|
|||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
|
|
||||||
int main(int argc, char* argv[]) {
|
int main(int argc, char* argv[])
|
||||||
|
{
|
||||||
if (argc != 2) {
|
if (argc != 2) {
|
||||||
// Tell the user how to run the program
|
// Tell the user how to run the program
|
||||||
std::cerr << "Usage: " << argv[0] << " path_to_directory" << std::endl;
|
std::cerr << "Usage: " << argv[0] << " path_to_directory" << std::endl;
|
||||||
|
@ -53,13 +53,15 @@ using namespace std;
|
|||||||
namespace po = boost::program_options;
|
namespace po = boost::program_options;
|
||||||
typedef multimap<float,string> Lines;
|
typedef multimap<float,string> Lines;
|
||||||
|
|
||||||
static void usage(const po::options_description& desc, char** argv) {
|
static void usage(const po::options_description& desc, char** argv)
|
||||||
|
{
|
||||||
cerr << "Usage: " + string(argv[0]) + " [options] input-file output-file" << endl;
|
cerr << "Usage: " + string(argv[0]) + " [options] input-file output-file" << endl;
|
||||||
cerr << desc << endl;
|
cerr << desc << endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
//Find top n translations of source, and send them to output
|
//Find top n translations of source, and send them to output
|
||||||
static void outputTopN(Lines lines, size_t maxPhrases, ostream& out) {
|
static void outputTopN(Lines lines, size_t maxPhrases, ostream& out)
|
||||||
|
{
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
for (Lines::const_reverse_iterator i = lines.rbegin(); i != lines.rend(); ++i) {
|
for (Lines::const_reverse_iterator i = lines.rbegin(); i != lines.rend(); ++i) {
|
||||||
out << i->second << endl;
|
out << i->second << endl;
|
||||||
|
@ -171,16 +171,14 @@ int main(int argc, char** argv)
|
|||||||
<< "missing update data for simulated post-editing.");
|
<< "missing update data for simulated post-editing.");
|
||||||
UTIL_THROW_IF2(!getline(*ioWrapper->spe_aln,aln), "[" << HERE << "] "
|
UTIL_THROW_IF2(!getline(*ioWrapper->spe_aln,aln), "[" << HERE << "] "
|
||||||
<< "missing update data for simulated post-editing.");
|
<< "missing update data for simulated post-editing.");
|
||||||
BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl())
|
BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl()) {
|
||||||
{
|
|
||||||
Mmsapt* sapt = dynamic_cast<Mmsapt*>(pd);
|
Mmsapt* sapt = dynamic_cast<Mmsapt*>(pd);
|
||||||
if (sapt) sapt->add(src,trg,aln);
|
if (sapt) sapt->add(src,trg,aln);
|
||||||
VERBOSE(1,"[" << HERE << " added src] " << src << endl);
|
VERBOSE(1,"[" << HERE << " added src] " << src << endl);
|
||||||
VERBOSE(1,"[" << HERE << " added trg] " << trg << endl);
|
VERBOSE(1,"[" << HERE << " added trg] " << trg << endl);
|
||||||
VERBOSE(1,"[" << HERE << " added aln] " << aln << endl);
|
VERBOSE(1,"[" << HERE << " added aln] " << aln << endl);
|
||||||
}
|
}
|
||||||
}
|
} else
|
||||||
else
|
|
||||||
#endif
|
#endif
|
||||||
pool.Submit(task);
|
pool.Submit(task);
|
||||||
#else
|
#else
|
||||||
|
@ -52,8 +52,7 @@ public:
|
|||||||
public:
|
public:
|
||||||
template<typename ALNREP>
|
template<typename ALNREP>
|
||||||
AlignmentInfo const *
|
AlignmentInfo const *
|
||||||
Add(ALNREP const & aln)
|
Add(ALNREP const & aln) {
|
||||||
{
|
|
||||||
return this->Add(AlignmentInfo(aln));
|
return this->Add(AlignmentInfo(aln));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -17,8 +17,8 @@ protected:
|
|||||||
const InputType &m_source; /**< source sentence to be translated */
|
const InputType &m_source; /**< source sentence to be translated */
|
||||||
|
|
||||||
BaseManager(const InputType &source)
|
BaseManager(const InputType &source)
|
||||||
:m_source(source)
|
:m_source(source) {
|
||||||
{}
|
}
|
||||||
|
|
||||||
// output
|
// output
|
||||||
typedef std::vector<std::pair<Moses::Word, Moses::WordsRange> > ApplicationContext;
|
typedef std::vector<std::pair<Moses::Word, Moses::WordsRange> > ApplicationContext;
|
||||||
@ -32,8 +32,7 @@ protected:
|
|||||||
const ApplicationContext &context) const;
|
const ApplicationContext &context) const;
|
||||||
|
|
||||||
template <class T>
|
template <class T>
|
||||||
void ShiftOffsets(std::vector<T> &offsets, T shift) const
|
void ShiftOffsets(std::vector<T> &offsets, T shift) const {
|
||||||
{
|
|
||||||
T currPos = shift;
|
T currPos = shift;
|
||||||
for (size_t i = 0; i < offsets.size(); ++i) {
|
for (size_t i = 0; i < offsets.size(); ++i) {
|
||||||
if (offsets[i] == 0) {
|
if (offsets[i] == 0) {
|
||||||
@ -46,8 +45,8 @@ protected:
|
|||||||
}
|
}
|
||||||
|
|
||||||
public:
|
public:
|
||||||
virtual ~BaseManager()
|
virtual ~BaseManager() {
|
||||||
{}
|
}
|
||||||
|
|
||||||
//! the input sentence being decoded
|
//! the input sentence being decoded
|
||||||
const InputType& GetSource() const {
|
const InputType& GetSource() const {
|
||||||
|
@ -124,8 +124,7 @@ public:
|
|||||||
const ChartCellLabel *Find(size_t idx) const {
|
const ChartCellLabel *Find(size_t idx) const {
|
||||||
try {
|
try {
|
||||||
return m_map.at(idx);
|
return m_map.at(idx);
|
||||||
}
|
} catch (const std::out_of_range& oor) {
|
||||||
catch (const std::out_of_range& oor) {
|
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -61,8 +61,7 @@ ChartHypothesis::ChartHypothesis(const ChartTranslationOptions &transOpt,
|
|||||||
const std::vector<HypothesisDimension> &childEntries = item.GetHypothesisDimensions();
|
const std::vector<HypothesisDimension> &childEntries = item.GetHypothesisDimensions();
|
||||||
m_prevHypos.reserve(childEntries.size());
|
m_prevHypos.reserve(childEntries.size());
|
||||||
std::vector<HypothesisDimension>::const_iterator iter;
|
std::vector<HypothesisDimension>::const_iterator iter;
|
||||||
for (iter = childEntries.begin(); iter != childEntries.end(); ++iter)
|
for (iter = childEntries.begin(); iter != childEntries.end(); ++iter) {
|
||||||
{
|
|
||||||
m_prevHypos.push_back(iter->GetHypothesis());
|
m_prevHypos.push_back(iter->GetHypothesis());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -85,17 +84,14 @@ ChartHypothesis::ChartHypothesis(const ChartHypothesis &pred,
|
|||||||
ChartHypothesis::~ChartHypothesis()
|
ChartHypothesis::~ChartHypothesis()
|
||||||
{
|
{
|
||||||
// delete feature function states
|
// delete feature function states
|
||||||
for (unsigned i = 0; i < m_ffStates.size(); ++i)
|
for (unsigned i = 0; i < m_ffStates.size(); ++i) {
|
||||||
{
|
|
||||||
delete m_ffStates[i];
|
delete m_ffStates[i];
|
||||||
}
|
}
|
||||||
|
|
||||||
// delete hypotheses that are not in the chart (recombined away)
|
// delete hypotheses that are not in the chart (recombined away)
|
||||||
if (m_arcList)
|
if (m_arcList) {
|
||||||
{
|
|
||||||
ChartArcList::iterator iter;
|
ChartArcList::iterator iter;
|
||||||
for (iter = m_arcList->begin() ; iter != m_arcList->end() ; ++iter)
|
for (iter = m_arcList->begin() ; iter != m_arcList->end() ; ++iter) {
|
||||||
{
|
|
||||||
ChartHypothesis *hypo = *iter;
|
ChartHypothesis *hypo = *iter;
|
||||||
Delete(hypo);
|
Delete(hypo);
|
||||||
}
|
}
|
||||||
@ -112,25 +108,19 @@ void ChartHypothesis::GetOutputPhrase(Phrase &outPhrase) const
|
|||||||
{
|
{
|
||||||
FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor();
|
FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor();
|
||||||
|
|
||||||
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos)
|
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos) {
|
||||||
{
|
|
||||||
const Word &word = GetCurrTargetPhrase().GetWord(pos);
|
const Word &word = GetCurrTargetPhrase().GetWord(pos);
|
||||||
if (word.IsNonTerminal())
|
if (word.IsNonTerminal()) {
|
||||||
{
|
|
||||||
// non-term. fill out with prev hypo
|
// non-term. fill out with prev hypo
|
||||||
size_t nonTermInd = GetCurrTargetPhrase().GetAlignNonTerm().GetNonTermIndexMap()[pos];
|
size_t nonTermInd = GetCurrTargetPhrase().GetAlignNonTerm().GetNonTermIndexMap()[pos];
|
||||||
const ChartHypothesis *prevHypo = m_prevHypos[nonTermInd];
|
const ChartHypothesis *prevHypo = m_prevHypos[nonTermInd];
|
||||||
prevHypo->GetOutputPhrase(outPhrase);
|
prevHypo->GetOutputPhrase(outPhrase);
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
outPhrase.AddWord(word);
|
outPhrase.AddWord(word);
|
||||||
|
|
||||||
if (placeholderFactor != NOT_FOUND)
|
if (placeholderFactor != NOT_FOUND) {
|
||||||
{
|
|
||||||
std::set<size_t> sourcePosSet = GetCurrTargetPhrase().GetAlignTerm().GetAlignmentsForTarget(pos);
|
std::set<size_t> sourcePosSet = GetCurrTargetPhrase().GetAlignTerm().GetAlignmentsForTarget(pos);
|
||||||
if (sourcePosSet.size() == 1)
|
if (sourcePosSet.size() == 1) {
|
||||||
{
|
|
||||||
const std::vector<const Word*> *ruleSourceFromInputPath = GetTranslationOption().GetSourceRuleFromInputPath();
|
const std::vector<const Word*> *ruleSourceFromInputPath = GetTranslationOption().GetSourceRuleFromInputPath();
|
||||||
UTIL_THROW_IF2(ruleSourceFromInputPath == NULL,
|
UTIL_THROW_IF2(ruleSourceFromInputPath == NULL,
|
||||||
"No source rule");
|
"No source rule");
|
||||||
@ -140,8 +130,7 @@ void ChartHypothesis::GetOutputPhrase(Phrase &outPhrase) const
|
|||||||
UTIL_THROW_IF2(sourceWord == NULL,
|
UTIL_THROW_IF2(sourceWord == NULL,
|
||||||
"No source word");
|
"No source word");
|
||||||
const Factor *factor = sourceWord->GetFactor(placeholderFactor);
|
const Factor *factor = sourceWord->GetFactor(placeholderFactor);
|
||||||
if (factor)
|
if (factor) {
|
||||||
{
|
|
||||||
outPhrase.Back()[0] = factor;
|
outPhrase.Back()[0] = factor;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -165,33 +154,24 @@ void ChartHypothesis::GetOutputPhrase(size_t leftRightMost, size_t numWords, Phr
|
|||||||
const TargetPhrase &tp = GetCurrTargetPhrase();
|
const TargetPhrase &tp = GetCurrTargetPhrase();
|
||||||
|
|
||||||
size_t targetSize = tp.GetSize();
|
size_t targetSize = tp.GetSize();
|
||||||
for (size_t i = 0; i < targetSize; ++i)
|
for (size_t i = 0; i < targetSize; ++i) {
|
||||||
{
|
|
||||||
size_t pos;
|
size_t pos;
|
||||||
if (leftRightMost == 1)
|
if (leftRightMost == 1) {
|
||||||
{
|
|
||||||
pos = i;
|
pos = i;
|
||||||
}
|
} else if (leftRightMost == 2) {
|
||||||
else if (leftRightMost == 2)
|
|
||||||
{
|
|
||||||
pos = targetSize - i - 1;
|
pos = targetSize - i - 1;
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
abort();
|
abort();
|
||||||
}
|
}
|
||||||
|
|
||||||
const Word &word = tp.GetWord(pos);
|
const Word &word = tp.GetWord(pos);
|
||||||
|
|
||||||
if (word.IsNonTerminal())
|
if (word.IsNonTerminal()) {
|
||||||
{
|
|
||||||
// non-term. fill out with prev hypo
|
// non-term. fill out with prev hypo
|
||||||
size_t nonTermInd = tp.GetAlignNonTerm().GetNonTermIndexMap()[pos];
|
size_t nonTermInd = tp.GetAlignNonTerm().GetNonTermIndexMap()[pos];
|
||||||
const ChartHypothesis *prevHypo = m_prevHypos[nonTermInd];
|
const ChartHypothesis *prevHypo = m_prevHypos[nonTermInd];
|
||||||
prevHypo->GetOutputPhrase(outPhrase);
|
prevHypo->GetOutputPhrase(outPhrase);
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
outPhrase.AddWord(word);
|
outPhrase.AddWord(word);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -236,20 +216,16 @@ void ChartHypothesis::EvaluateWhenApplied()
|
|||||||
// cached in the translation option-- there is no principled distinction
|
// cached in the translation option-- there is no principled distinction
|
||||||
const std::vector<const StatelessFeatureFunction*>& sfs =
|
const std::vector<const StatelessFeatureFunction*>& sfs =
|
||||||
StatelessFeatureFunction::GetStatelessFeatureFunctions();
|
StatelessFeatureFunction::GetStatelessFeatureFunctions();
|
||||||
for (unsigned i = 0; i < sfs.size(); ++i)
|
for (unsigned i = 0; i < sfs.size(); ++i) {
|
||||||
{
|
if (! staticData.IsFeatureFunctionIgnored( *sfs[i] )) {
|
||||||
if (! staticData.IsFeatureFunctionIgnored( *sfs[i] ))
|
|
||||||
{
|
|
||||||
sfs[i]->EvaluateWhenApplied(*this,&m_currScoreBreakdown);
|
sfs[i]->EvaluateWhenApplied(*this,&m_currScoreBreakdown);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const std::vector<const StatefulFeatureFunction*>& ffs =
|
const std::vector<const StatefulFeatureFunction*>& ffs =
|
||||||
StatefulFeatureFunction::GetStatefulFeatureFunctions();
|
StatefulFeatureFunction::GetStatefulFeatureFunctions();
|
||||||
for (unsigned i = 0; i < ffs.size(); ++i)
|
for (unsigned i = 0; i < ffs.size(); ++i) {
|
||||||
{
|
if (! staticData.IsFeatureFunctionIgnored( *ffs[i] )) {
|
||||||
if (! staticData.IsFeatureFunctionIgnored( *ffs[i] ))
|
|
||||||
{
|
|
||||||
m_ffStates[i] = ffs[i]->EvaluateWhenApplied(*this,i,&m_currScoreBreakdown);
|
m_ffStates[i] = ffs[i]->EvaluateWhenApplied(*this,i,&m_currScoreBreakdown);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -267,31 +243,25 @@ void ChartHypothesis::EvaluateWhenApplied()
|
|||||||
|
|
||||||
void ChartHypothesis::AddArc(ChartHypothesis *loserHypo)
|
void ChartHypothesis::AddArc(ChartHypothesis *loserHypo)
|
||||||
{
|
{
|
||||||
if (!m_arcList)
|
if (!m_arcList) {
|
||||||
{
|
if (loserHypo->m_arcList) {
|
||||||
if (loserHypo->m_arcList)
|
// we don't have an arcList, but loser does
|
||||||
{ // we don't have an arcList, but loser does
|
|
||||||
this->m_arcList = loserHypo->m_arcList; // take ownership, we'll delete
|
this->m_arcList = loserHypo->m_arcList; // take ownership, we'll delete
|
||||||
loserHypo->m_arcList = 0; // prevent a double deletion
|
loserHypo->m_arcList = 0; // prevent a double deletion
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
this->m_arcList = new ChartArcList();
|
this->m_arcList = new ChartArcList();
|
||||||
}
|
}
|
||||||
}
|
} else {
|
||||||
else
|
if (loserHypo->m_arcList) {
|
||||||
{
|
// both have an arc list: merge. delete loser
|
||||||
if (loserHypo->m_arcList)
|
|
||||||
{ // both have an arc list: merge. delete loser
|
|
||||||
size_t my_size = m_arcList->size();
|
size_t my_size = m_arcList->size();
|
||||||
size_t add_size = loserHypo->m_arcList->size();
|
size_t add_size = loserHypo->m_arcList->size();
|
||||||
this->m_arcList->resize(my_size + add_size, 0);
|
this->m_arcList->resize(my_size + add_size, 0);
|
||||||
std::memcpy(&(*m_arcList)[0] + my_size, &(*loserHypo->m_arcList)[0], add_size * sizeof(ChartHypothesis *));
|
std::memcpy(&(*m_arcList)[0] + my_size, &(*loserHypo->m_arcList)[0], add_size * sizeof(ChartHypothesis *));
|
||||||
delete loserHypo->m_arcList;
|
delete loserHypo->m_arcList;
|
||||||
loserHypo->m_arcList = 0;
|
loserHypo->m_arcList = 0;
|
||||||
}
|
} else {
|
||||||
else
|
// loserHypo doesn't have any arcs
|
||||||
{ // loserHypo doesn't have any arcs
|
|
||||||
// DO NOTHING
|
// DO NOTHING
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -299,10 +269,8 @@ void ChartHypothesis::AddArc(ChartHypothesis *loserHypo)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// sorting helper
|
// sorting helper
|
||||||
struct CompareChartHypothesisTotalScore
|
struct CompareChartHypothesisTotalScore {
|
||||||
{
|
bool operator()(const ChartHypothesis* hypo1, const ChartHypothesis* hypo2) const {
|
||||||
bool operator()(const ChartHypothesis* hypo1, const ChartHypothesis* hypo2) const
|
|
||||||
{
|
|
||||||
return hypo1->GetTotalScore() > hypo2->GetTotalScore();
|
return hypo1->GetTotalScore() > hypo2->GetTotalScore();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@ -322,8 +290,7 @@ void ChartHypothesis::CleanupArcList()
|
|||||||
size_t nBestSize = staticData.GetNBestSize();
|
size_t nBestSize = staticData.GetNBestSize();
|
||||||
bool distinctNBest = staticData.GetDistinctNBest() || staticData.UseMBR() || staticData.GetOutputSearchGraph() || staticData.GetOutputSearchGraphHypergraph();
|
bool distinctNBest = staticData.GetDistinctNBest() || staticData.UseMBR() || staticData.GetOutputSearchGraph() || staticData.GetOutputSearchGraphHypergraph();
|
||||||
|
|
||||||
if (!distinctNBest && m_arcList->size() > nBestSize)
|
if (!distinctNBest && m_arcList->size() > nBestSize) {
|
||||||
{
|
|
||||||
// prune arc list only if there too many arcs
|
// prune arc list only if there too many arcs
|
||||||
NTH_ELEMENT4(m_arcList->begin()
|
NTH_ELEMENT4(m_arcList->begin()
|
||||||
, m_arcList->begin() + nBestSize - 1
|
, m_arcList->begin() + nBestSize - 1
|
||||||
@ -332,8 +299,7 @@ void ChartHypothesis::CleanupArcList()
|
|||||||
|
|
||||||
// delete bad ones
|
// delete bad ones
|
||||||
ChartArcList::iterator iter;
|
ChartArcList::iterator iter;
|
||||||
for (iter = m_arcList->begin() + nBestSize ; iter != m_arcList->end() ; ++iter)
|
for (iter = m_arcList->begin() + nBestSize ; iter != m_arcList->end() ; ++iter) {
|
||||||
{
|
|
||||||
ChartHypothesis *arc = *iter;
|
ChartHypothesis *arc = *iter;
|
||||||
ChartHypothesis::Delete(arc);
|
ChartHypothesis::Delete(arc);
|
||||||
}
|
}
|
||||||
@ -343,8 +309,7 @@ void ChartHypothesis::CleanupArcList()
|
|||||||
|
|
||||||
// set all arc's main hypo variable to this hypo
|
// set all arc's main hypo variable to this hypo
|
||||||
ChartArcList::iterator iter = m_arcList->begin();
|
ChartArcList::iterator iter = m_arcList->begin();
|
||||||
for (; iter != m_arcList->end() ; ++iter)
|
for (; iter != m_arcList->end() ; ++iter) {
|
||||||
{
|
|
||||||
ChartHypothesis *arc = *iter;
|
ChartHypothesis *arc = *iter;
|
||||||
arc->SetWinningHypo(this);
|
arc->SetWinningHypo(this);
|
||||||
}
|
}
|
||||||
@ -367,13 +332,11 @@ std::ostream& operator<<(std::ostream& out, const ChartHypothesis& hypo)
|
|||||||
|
|
||||||
// recombination
|
// recombination
|
||||||
if (hypo.GetWinningHypothesis() != NULL &&
|
if (hypo.GetWinningHypothesis() != NULL &&
|
||||||
hypo.GetWinningHypothesis() != &hypo)
|
hypo.GetWinningHypothesis() != &hypo) {
|
||||||
{
|
|
||||||
out << "->" << hypo.GetWinningHypothesis()->GetId();
|
out << "->" << hypo.GetWinningHypothesis()->GetId();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (StaticData::Instance().GetIncludeLHSInSearchGraph())
|
if (StaticData::Instance().GetIncludeLHSInSearchGraph()) {
|
||||||
{
|
|
||||||
out << " " << hypo.GetTargetLHS() << "=>";
|
out << " " << hypo.GetTargetLHS() << "=>";
|
||||||
}
|
}
|
||||||
out << " " << hypo.GetCurrTargetPhrase()
|
out << " " << hypo.GetCurrTargetPhrase()
|
||||||
@ -381,8 +344,7 @@ std::ostream& operator<<(std::ostream& out, const ChartHypothesis& hypo)
|
|||||||
<< " " << hypo.GetCurrSourceRange();
|
<< " " << hypo.GetCurrSourceRange();
|
||||||
|
|
||||||
HypoList::const_iterator iter;
|
HypoList::const_iterator iter;
|
||||||
for (iter = hypo.GetPrevHypos().begin(); iter != hypo.GetPrevHypos().end(); ++iter)
|
for (iter = hypo.GetPrevHypos().begin(); iter != hypo.GetPrevHypos().end(); ++iter) {
|
||||||
{
|
|
||||||
const ChartHypothesis &prevHypo = **iter;
|
const ChartHypothesis &prevHypo = **iter;
|
||||||
out << " " << prevHypo.GetId();
|
out << " " << prevHypo.GetId();
|
||||||
}
|
}
|
||||||
|
@ -82,21 +82,18 @@ protected:
|
|||||||
|
|
||||||
public:
|
public:
|
||||||
#ifdef USE_HYPO_POOL
|
#ifdef USE_HYPO_POOL
|
||||||
void *operator new(size_t /* num_bytes */)
|
void *operator new(size_t /* num_bytes */) {
|
||||||
{
|
|
||||||
void *ptr = s_objectPool.getPtr();
|
void *ptr = s_objectPool.getPtr();
|
||||||
return ptr;
|
return ptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
//! delete \param hypo. Works with object pool too
|
//! delete \param hypo. Works with object pool too
|
||||||
static void Delete(ChartHypothesis *hypo)
|
static void Delete(ChartHypothesis *hypo) {
|
||||||
{
|
|
||||||
s_objectPool.freeObject(hypo);
|
s_objectPool.freeObject(hypo);
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
//! delete \param hypo. Works with object pool too
|
//! delete \param hypo. Works with object pool too
|
||||||
static void Delete(ChartHypothesis *hypo)
|
static void Delete(ChartHypothesis *hypo) {
|
||||||
{
|
|
||||||
delete hypo;
|
delete hypo;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
@ -109,43 +106,36 @@ public:
|
|||||||
|
|
||||||
~ChartHypothesis();
|
~ChartHypothesis();
|
||||||
|
|
||||||
unsigned GetId() const
|
unsigned GetId() const {
|
||||||
{
|
|
||||||
return m_id;
|
return m_id;
|
||||||
}
|
}
|
||||||
|
|
||||||
const ChartTranslationOption &GetTranslationOption() const
|
const ChartTranslationOption &GetTranslationOption() const {
|
||||||
{
|
|
||||||
return *m_transOpt;
|
return *m_transOpt;
|
||||||
}
|
}
|
||||||
|
|
||||||
//! Get the rule that created this hypothesis
|
//! Get the rule that created this hypothesis
|
||||||
const TargetPhrase &GetCurrTargetPhrase() const
|
const TargetPhrase &GetCurrTargetPhrase() const {
|
||||||
{
|
|
||||||
return m_transOpt->GetPhrase();
|
return m_transOpt->GetPhrase();
|
||||||
}
|
}
|
||||||
|
|
||||||
//! the source range that this hypothesis spans
|
//! the source range that this hypothesis spans
|
||||||
const WordsRange &GetCurrSourceRange() const
|
const WordsRange &GetCurrSourceRange() const {
|
||||||
{
|
|
||||||
return m_currSourceWordsRange;
|
return m_currSourceWordsRange;
|
||||||
}
|
}
|
||||||
|
|
||||||
//! the arc list when creating n-best lists
|
//! the arc list when creating n-best lists
|
||||||
inline const ChartArcList* GetArcList() const
|
inline const ChartArcList* GetArcList() const {
|
||||||
{
|
|
||||||
return m_arcList;
|
return m_arcList;
|
||||||
}
|
}
|
||||||
|
|
||||||
//! the feature function states for a particular feature \param featureID
|
//! the feature function states for a particular feature \param featureID
|
||||||
inline const FFState* GetFFState( size_t featureID ) const
|
inline const FFState* GetFFState( size_t featureID ) const {
|
||||||
{
|
|
||||||
return m_ffStates[ featureID ];
|
return m_ffStates[ featureID ];
|
||||||
}
|
}
|
||||||
|
|
||||||
//! reference back to the manager
|
//! reference back to the manager
|
||||||
inline const ChartManager& GetManager() const
|
inline const ChartManager& GetManager() const {
|
||||||
{
|
|
||||||
return m_manager;
|
return m_manager;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -165,21 +155,17 @@ public:
|
|||||||
void SetWinningHypo(const ChartHypothesis *hypo);
|
void SetWinningHypo(const ChartHypothesis *hypo);
|
||||||
|
|
||||||
//! get the unweighted score for each feature function
|
//! get the unweighted score for each feature function
|
||||||
const ScoreComponentCollection &GetScoreBreakdown() const
|
const ScoreComponentCollection &GetScoreBreakdown() const {
|
||||||
{
|
|
||||||
// Note: never call this method before m_currScoreBreakdown is fully computed
|
// Note: never call this method before m_currScoreBreakdown is fully computed
|
||||||
if (!m_scoreBreakdown.get())
|
if (!m_scoreBreakdown.get()) {
|
||||||
{
|
|
||||||
m_scoreBreakdown.reset(new ScoreComponentCollection());
|
m_scoreBreakdown.reset(new ScoreComponentCollection());
|
||||||
// score breakdown from current translation rule
|
// score breakdown from current translation rule
|
||||||
if (m_transOpt)
|
if (m_transOpt) {
|
||||||
{
|
|
||||||
m_scoreBreakdown->PlusEquals(GetTranslationOption().GetScores());
|
m_scoreBreakdown->PlusEquals(GetTranslationOption().GetScores());
|
||||||
}
|
}
|
||||||
m_scoreBreakdown->PlusEquals(m_currScoreBreakdown);
|
m_scoreBreakdown->PlusEquals(m_currScoreBreakdown);
|
||||||
// score breakdowns from prev hypos
|
// score breakdowns from prev hypos
|
||||||
for (std::vector<const ChartHypothesis*>::const_iterator iter = m_prevHypos.begin(); iter != m_prevHypos.end(); ++iter)
|
for (std::vector<const ChartHypothesis*>::const_iterator iter = m_prevHypos.begin(); iter != m_prevHypos.end(); ++iter) {
|
||||||
{
|
|
||||||
const ChartHypothesis &prevHypo = **iter;
|
const ChartHypothesis &prevHypo = **iter;
|
||||||
m_scoreBreakdown->PlusEquals(prevHypo.GetScoreBreakdown());
|
m_scoreBreakdown->PlusEquals(prevHypo.GetScoreBreakdown());
|
||||||
}
|
}
|
||||||
@ -188,15 +174,12 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
//! get the unweighted score delta for each feature function
|
//! get the unweighted score delta for each feature function
|
||||||
const ScoreComponentCollection &GetDeltaScoreBreakdown() const
|
const ScoreComponentCollection &GetDeltaScoreBreakdown() const {
|
||||||
{
|
|
||||||
// Note: never call this method before m_currScoreBreakdown is fully computed
|
// Note: never call this method before m_currScoreBreakdown is fully computed
|
||||||
if (!m_deltaScoreBreakdown.get())
|
if (!m_deltaScoreBreakdown.get()) {
|
||||||
{
|
|
||||||
m_deltaScoreBreakdown.reset(new ScoreComponentCollection());
|
m_deltaScoreBreakdown.reset(new ScoreComponentCollection());
|
||||||
// score breakdown from current translation rule
|
// score breakdown from current translation rule
|
||||||
if (m_transOpt)
|
if (m_transOpt) {
|
||||||
{
|
|
||||||
m_deltaScoreBreakdown->PlusEquals(GetTranslationOption().GetScores());
|
m_deltaScoreBreakdown->PlusEquals(GetTranslationOption().GetScores());
|
||||||
}
|
}
|
||||||
m_deltaScoreBreakdown->PlusEquals(m_currScoreBreakdown);
|
m_deltaScoreBreakdown->PlusEquals(m_currScoreBreakdown);
|
||||||
@ -206,33 +189,28 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
//! Get the weighted total score
|
//! Get the weighted total score
|
||||||
float GetTotalScore() const
|
float GetTotalScore() const {
|
||||||
{
|
|
||||||
// scores from current translation rule. eg. translation models & word penalty
|
// scores from current translation rule. eg. translation models & word penalty
|
||||||
return m_totalScore;
|
return m_totalScore;
|
||||||
}
|
}
|
||||||
|
|
||||||
//! vector of previous hypotheses this hypo is built on
|
//! vector of previous hypotheses this hypo is built on
|
||||||
const std::vector<const ChartHypothesis*> &GetPrevHypos() const
|
const std::vector<const ChartHypothesis*> &GetPrevHypos() const {
|
||||||
{
|
|
||||||
return m_prevHypos;
|
return m_prevHypos;
|
||||||
}
|
}
|
||||||
|
|
||||||
//! get a particular previous hypos
|
//! get a particular previous hypos
|
||||||
const ChartHypothesis* GetPrevHypo(size_t pos) const
|
const ChartHypothesis* GetPrevHypo(size_t pos) const {
|
||||||
{
|
|
||||||
return m_prevHypos[pos];
|
return m_prevHypos[pos];
|
||||||
}
|
}
|
||||||
|
|
||||||
//! get the constituency label that covers this hypo
|
//! get the constituency label that covers this hypo
|
||||||
const Word &GetTargetLHS() const
|
const Word &GetTargetLHS() const {
|
||||||
{
|
|
||||||
return GetCurrTargetPhrase().GetTargetLHS();
|
return GetCurrTargetPhrase().GetTargetLHS();
|
||||||
}
|
}
|
||||||
|
|
||||||
//! get the best hypo in the arc list when doing n-best list creation. It's either this hypothesis, or the best hypo is this hypo is in the arc list
|
//! get the best hypo in the arc list when doing n-best list creation. It's either this hypothesis, or the best hypo is this hypo is in the arc list
|
||||||
const ChartHypothesis* GetWinningHypothesis() const
|
const ChartHypothesis* GetWinningHypothesis() const {
|
||||||
{
|
|
||||||
return m_winningHypo;
|
return m_winningHypo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -169,8 +169,7 @@ TreePointer ChartKBestExtractor::GetOutputTree(const Derivation &d)
|
|||||||
|
|
||||||
mytree->Combine(previous_trees);
|
mytree->Combine(previous_trees);
|
||||||
return mytree;
|
return mytree;
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
UTIL_THROW2("Error: TreeStructureFeature active, but no internal tree structure found");
|
UTIL_THROW2("Error: TreeStructureFeature active, but no internal tree structure found");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -290,12 +290,14 @@ void ChartManager::FindReachableHypotheses(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ChartManager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const {
|
void ChartManager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const
|
||||||
|
{
|
||||||
ChartSearchGraphWriterHypergraph writer(&outputSearchGraphStream);
|
ChartSearchGraphWriterHypergraph writer(&outputSearchGraphStream);
|
||||||
WriteSearchGraph(writer);
|
WriteSearchGraph(writer);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ChartManager::OutputSearchGraphMoses(std::ostream &outputSearchGraphStream) const {
|
void ChartManager::OutputSearchGraphMoses(std::ostream &outputSearchGraphStream) const
|
||||||
|
{
|
||||||
ChartSearchGraphWriterMoses writer(&outputSearchGraphStream, m_source.GetTranslationId());
|
ChartSearchGraphWriterMoses writer(&outputSearchGraphStream, m_source.GetTranslationId());
|
||||||
WriteSearchGraph(writer);
|
WriteSearchGraph(writer);
|
||||||
}
|
}
|
||||||
|
@ -126,8 +126,8 @@ public:
|
|||||||
return m_hypoStackColl;
|
return m_hypoStackColl;
|
||||||
}
|
}
|
||||||
|
|
||||||
void CalcDecoderStatistics() const
|
void CalcDecoderStatistics() const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void ResetSentenceStats(const InputType& source) {
|
void ResetSentenceStats(const InputType& source) {
|
||||||
m_sentenceStats = std::auto_ptr<SentenceStats>(new SentenceStats(source));
|
m_sentenceStats = std::auto_ptr<SentenceStats>(new SentenceStats(source));
|
||||||
@ -138,22 +138,24 @@ public:
|
|||||||
return m_hypothesisId++;
|
return m_hypothesisId++;
|
||||||
}
|
}
|
||||||
|
|
||||||
const ChartParser &GetParser() const { return m_parser; }
|
const ChartParser &GetParser() const {
|
||||||
|
return m_parser;
|
||||||
|
}
|
||||||
|
|
||||||
// outputs
|
// outputs
|
||||||
void OutputBest(OutputCollector *collector) const;
|
void OutputBest(OutputCollector *collector) const;
|
||||||
void OutputNBest(OutputCollector *collector) const;
|
void OutputNBest(OutputCollector *collector) const;
|
||||||
void OutputLatticeSamples(OutputCollector *collector) const
|
void OutputLatticeSamples(OutputCollector *collector) const {
|
||||||
{}
|
}
|
||||||
void OutputAlignment(OutputCollector *collector) const;
|
void OutputAlignment(OutputCollector *collector) const;
|
||||||
void OutputDetailedTranslationReport(OutputCollector *collector) const;
|
void OutputDetailedTranslationReport(OutputCollector *collector) const;
|
||||||
void OutputUnknowns(OutputCollector *collector) const;
|
void OutputUnknowns(OutputCollector *collector) const;
|
||||||
void OutputDetailedTreeFragmentsTranslationReport(OutputCollector *collector) const;
|
void OutputDetailedTreeFragmentsTranslationReport(OutputCollector *collector) const;
|
||||||
void OutputWordGraph(OutputCollector *collector) const
|
void OutputWordGraph(OutputCollector *collector) const {
|
||||||
{}
|
}
|
||||||
void OutputSearchGraph(OutputCollector *collector) const;
|
void OutputSearchGraph(OutputCollector *collector) const;
|
||||||
void OutputSearchGraphSLF() const
|
void OutputSearchGraphSLF() const {
|
||||||
{}
|
}
|
||||||
void OutputSearchGraphHypergraph() const;
|
void OutputSearchGraphHypergraph() const;
|
||||||
|
|
||||||
};
|
};
|
||||||
|
@ -72,8 +72,7 @@ void ChartTranslationOptions::EvaluateWithSourceContext(const InputType &input,
|
|||||||
|
|
||||||
if (transOpt->GetScores().GetWeightedScore() == - std::numeric_limits<float>::infinity()) {
|
if (transOpt->GetScores().GetWeightedScore() == - std::numeric_limits<float>::infinity()) {
|
||||||
++numDiscard;
|
++numDiscard;
|
||||||
}
|
} else if (numDiscard) {
|
||||||
else if (numDiscard) {
|
|
||||||
m_collection[i - numDiscard] = m_collection[i];
|
m_collection[i - numDiscard] = m_collection[i];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -49,8 +49,8 @@ public:
|
|||||||
DecodeGraph(size_t id)
|
DecodeGraph(size_t id)
|
||||||
: m_id(id)
|
: m_id(id)
|
||||||
, m_maxChartSpan(NOT_FOUND)
|
, m_maxChartSpan(NOT_FOUND)
|
||||||
, m_backoff(0)
|
, m_backoff(0) {
|
||||||
{}
|
}
|
||||||
|
|
||||||
// for chart decoding
|
// for chart decoding
|
||||||
DecodeGraph(size_t id, size_t maxChartSpan)
|
DecodeGraph(size_t id, size_t maxChartSpan)
|
||||||
|
@ -126,17 +126,17 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
bool Enabled() const {
|
bool Enabled() const {
|
||||||
return m_enabled;
|
return m_enabled;
|
||||||
|
@ -11,8 +11,8 @@ namespace Moses
|
|||||||
class ConstrainedDecodingState : public FFState
|
class ConstrainedDecodingState : public FFState
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
ConstrainedDecodingState()
|
ConstrainedDecodingState() {
|
||||||
{}
|
}
|
||||||
|
|
||||||
ConstrainedDecodingState(const Hypothesis &hypo);
|
ConstrainedDecodingState(const Hypothesis &hypo);
|
||||||
ConstrainedDecodingState(const ChartHypothesis &hypo);
|
ConstrainedDecodingState(const ChartHypothesis &hypo);
|
||||||
@ -44,20 +44,20 @@ public:
|
|||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWithSourceContext(const InputType &input
|
void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
FFState* EvaluateWhenApplied(
|
FFState* EvaluateWhenApplied(
|
||||||
const Hypothesis& cur_hypo,
|
const Hypothesis& cur_hypo,
|
||||||
|
@ -20,8 +20,8 @@ class ControlRecombinationState : public FFState
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
ControlRecombinationState(const ControlRecombination &ff)
|
ControlRecombinationState(const ControlRecombination &ff)
|
||||||
:m_ff(ff)
|
:m_ff(ff) {
|
||||||
{}
|
}
|
||||||
|
|
||||||
ControlRecombinationState(const Hypothesis &hypo, const ControlRecombination &ff);
|
ControlRecombinationState(const Hypothesis &hypo, const ControlRecombination &ff);
|
||||||
ControlRecombinationState(const ChartHypothesis &hypo, const ControlRecombination &ff);
|
ControlRecombinationState(const ChartHypothesis &hypo, const ControlRecombination &ff);
|
||||||
@ -60,19 +60,19 @@ public:
|
|||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWithSourceContext(const InputType &input
|
void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
FFState* EvaluateWhenApplied(
|
FFState* EvaluateWhenApplied(
|
||||||
const Hypothesis& cur_hypo,
|
const Hypothesis& cur_hypo,
|
||||||
|
@ -23,21 +23,21 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(
|
void EvaluateWhenApplied(
|
||||||
const ChartHypothesis& hypo,
|
const ChartHypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
|
@ -67,7 +67,8 @@ void CoveredReferenceFeature::EvaluateWithSourceContext(const InputType &input
|
|||||||
estimatedFutureScore->Assign(this, scores);
|
estimatedFutureScore->Assign(this, scores);
|
||||||
}
|
}
|
||||||
|
|
||||||
void CoveredReferenceFeature::Load() {
|
void CoveredReferenceFeature::Load()
|
||||||
|
{
|
||||||
InputFileStream refFile(m_path);
|
InputFileStream refFile(m_path);
|
||||||
std::string line;
|
std::string line;
|
||||||
const StaticData &staticData = StaticData::Instance();
|
const StaticData &staticData = StaticData::Instance();
|
||||||
|
@ -63,8 +63,8 @@ public:
|
|||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
FFState* EvaluateWhenApplied(
|
FFState* EvaluateWhenApplied(
|
||||||
const Hypothesis& cur_hypo,
|
const Hypothesis& cur_hypo,
|
||||||
|
@ -63,30 +63,30 @@ public:
|
|||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWhenApplied(const Syntax::SHyperedge &hyperedge,
|
void EvaluateWhenApplied(const Syntax::SHyperedge &hyperedge,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWithSourceContext(const InputType &input
|
void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void SetContainer(const DecodeStep *container) {
|
void SetContainer(const DecodeStep *container) {
|
||||||
m_container = container;
|
m_container = container;
|
||||||
|
@ -52,18 +52,18 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -93,12 +93,16 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
static const DynamicCacheBasedLanguageModel* Instance(const std::string& name) {
|
static const DynamicCacheBasedLanguageModel* Instance(const std::string& name) {
|
||||||
if (s_instance_map.find(name) == s_instance_map.end()){ return NULL; }
|
if (s_instance_map.find(name) == s_instance_map.end()) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
return s_instance_map[name];
|
return s_instance_map[name];
|
||||||
}
|
}
|
||||||
|
|
||||||
static DynamicCacheBasedLanguageModel* InstanceNonConst(const std::string& name) {
|
static DynamicCacheBasedLanguageModel* InstanceNonConst(const std::string& name) {
|
||||||
if (s_instance_map.find(name) == s_instance_map.end()){ return NULL; }
|
if (s_instance_map.find(name) == s_instance_map.end()) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
return s_instance_map[name];
|
return s_instance_map[name];
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -135,20 +139,20 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void SetQueryType(size_t type);
|
void SetQueryType(size_t type);
|
||||||
void SetScoreType(size_t type);
|
void SetScoreType(size_t type);
|
||||||
|
@ -18,8 +18,8 @@ protected:
|
|||||||
public:
|
public:
|
||||||
ExternalFeatureState(int stateSize)
|
ExternalFeatureState(int stateSize)
|
||||||
:m_stateSize(stateSize)
|
:m_stateSize(stateSize)
|
||||||
,m_data(NULL)
|
,m_data(NULL) {
|
||||||
{}
|
}
|
||||||
ExternalFeatureState(int stateSize, void *data);
|
ExternalFeatureState(int stateSize, void *data);
|
||||||
|
|
||||||
~ExternalFeatureState() {
|
~ExternalFeatureState() {
|
||||||
@ -54,19 +54,19 @@ public:
|
|||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWithSourceContext(const InputType &input
|
void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
FFState* EvaluateWhenApplied(
|
FFState* EvaluateWhenApplied(
|
||||||
const Hypothesis& cur_hypo,
|
const Hypothesis& cur_hypo,
|
||||||
|
@ -116,8 +116,8 @@ public:
|
|||||||
, ScoreComponentCollection &estimatedFutureScore) const = 0;
|
, ScoreComponentCollection &estimatedFutureScore) const = 0;
|
||||||
|
|
||||||
// override this method if you want to change the input before decoding
|
// override this method if you want to change the input before decoding
|
||||||
virtual void ChangeSource(InputType *&input) const
|
virtual void ChangeSource(InputType *&input) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
// This method is called once all the translation options are retrieved from the phrase table, and
|
// This method is called once all the translation options are retrieved from the phrase table, and
|
||||||
// just before search.
|
// just before search.
|
||||||
|
@ -76,23 +76,23 @@ public:
|
|||||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWithSourceContext(const InputType &input
|
void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -95,18 +95,18 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void AddFeature(ScoreComponentCollection* accumulator,
|
void AddFeature(ScoreComponentCollection* accumulator,
|
||||||
StringPiece sourceTrigger, StringPiece sourceWord, StringPiece targetTrigger,
|
StringPiece sourceTrigger, StringPiece sourceWord, StringPiece targetTrigger,
|
||||||
|
@ -21,31 +21,31 @@ public:
|
|||||||
virtual void EvaluateInIsolation(const Phrase &source
|
virtual void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
virtual void EvaluateWithSourceContext(const InputType &input
|
virtual void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
virtual void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
virtual void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
|
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Same for chart-based features.
|
* Same for chart-based features.
|
||||||
**/
|
**/
|
||||||
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -44,8 +44,8 @@ public:
|
|||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWithSourceContext(const InputType &input
|
void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
@ -55,15 +55,15 @@ public:
|
|||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
|
|
||||||
};
|
};
|
||||||
|
@ -22,19 +22,18 @@ InternalTree::InternalTree(const std::string & line, const bool terminal):
|
|||||||
|
|
||||||
if (found == line.npos) {
|
if (found == line.npos) {
|
||||||
m_value = line;
|
m_value = line;
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
AddSubTree(line, 0);
|
AddSubTree(line, 0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t InternalTree::AddSubTree(const std::string & line, size_t pos) {
|
size_t InternalTree::AddSubTree(const std::string & line, size_t pos)
|
||||||
|
{
|
||||||
|
|
||||||
char token = 0;
|
char token = 0;
|
||||||
size_t len = 0;
|
size_t len = 0;
|
||||||
|
|
||||||
while (token != ']' && pos != std::string::npos)
|
while (token != ']' && pos != std::string::npos) {
|
||||||
{
|
|
||||||
size_t oldpos = pos;
|
size_t oldpos = pos;
|
||||||
pos = line.find_first_of("[] ", pos);
|
pos = line.find_first_of("[] ", pos);
|
||||||
if (pos == std::string::npos) break;
|
if (pos == std::string::npos) break;
|
||||||
@ -45,19 +44,16 @@ size_t InternalTree::AddSubTree(const std::string & line, size_t pos) {
|
|||||||
if (!m_value.empty()) {
|
if (!m_value.empty()) {
|
||||||
m_children.push_back(boost::make_shared<InternalTree>(line, oldpos, len, false));
|
m_children.push_back(boost::make_shared<InternalTree>(line, oldpos, len, false));
|
||||||
pos = m_children.back()->AddSubTree(line, pos+1);
|
pos = m_children.back()->AddSubTree(line, pos+1);
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
if (len > 0) {
|
if (len > 0) {
|
||||||
m_value.assign(line, oldpos, len);
|
m_value.assign(line, oldpos, len);
|
||||||
}
|
}
|
||||||
pos = AddSubTree(line, pos+1);
|
pos = AddSubTree(line, pos+1);
|
||||||
}
|
}
|
||||||
}
|
} else if (token == ' ' || token == ']') {
|
||||||
else if (token == ' ' || token == ']') {
|
|
||||||
if (len > 0 && m_value.empty()) {
|
if (len > 0 && m_value.empty()) {
|
||||||
m_value.assign(line, oldpos, len);
|
m_value.assign(line, oldpos, len);
|
||||||
}
|
} else if (len > 0) {
|
||||||
else if (len > 0) {
|
|
||||||
m_isTerminal = false;
|
m_isTerminal = false;
|
||||||
m_children.push_back(boost::make_shared<InternalTree>(line, oldpos, len, true));
|
m_children.push_back(boost::make_shared<InternalTree>(line, oldpos, len, true));
|
||||||
}
|
}
|
||||||
@ -78,7 +74,8 @@ size_t InternalTree::AddSubTree(const std::string & line, size_t pos) {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string InternalTree::GetString(bool start) const {
|
std::string InternalTree::GetString(bool start) const
|
||||||
|
{
|
||||||
|
|
||||||
std::string ret = "";
|
std::string ret = "";
|
||||||
if (!start) {
|
if (!start) {
|
||||||
@ -90,8 +87,7 @@ std::string InternalTree::GetString(bool start) const {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ret += m_value;
|
ret += m_value;
|
||||||
for (std::vector<TreePointer>::const_iterator it = m_children.begin(); it != m_children.end(); ++it)
|
for (std::vector<TreePointer>::const_iterator it = m_children.begin(); it != m_children.end(); ++it) {
|
||||||
{
|
|
||||||
ret += (*it)->GetString(false);
|
ret += (*it)->GetString(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -103,7 +99,8 @@ std::string InternalTree::GetString(bool start) const {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void InternalTree::Combine(const std::vector<TreePointer> &previous) {
|
void InternalTree::Combine(const std::vector<TreePointer> &previous)
|
||||||
|
{
|
||||||
|
|
||||||
std::vector<TreePointer>::iterator it;
|
std::vector<TreePointer>::iterator it;
|
||||||
bool found = false;
|
bool found = false;
|
||||||
@ -112,15 +109,15 @@ void InternalTree::Combine(const std::vector<TreePointer> &previous) {
|
|||||||
found = next_leafNT(it);
|
found = next_leafNT(it);
|
||||||
if (found) {
|
if (found) {
|
||||||
*it = *it_prev;
|
*it = *it_prev;
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
std::cerr << "Warning: leaf nonterminal not found in rule; why did this happen?\n";
|
std::cerr << "Warning: leaf nonterminal not found in rule; why did this happen?\n";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool InternalTree::FlatSearch(const std::string & label, std::vector<TreePointer>::const_iterator & it) const {
|
bool InternalTree::FlatSearch(const std::string & label, std::vector<TreePointer>::const_iterator & it) const
|
||||||
|
{
|
||||||
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
||||||
if ((*it)->GetLabel() == label) {
|
if ((*it)->GetLabel() == label) {
|
||||||
return true;
|
return true;
|
||||||
@ -129,7 +126,8 @@ bool InternalTree::FlatSearch(const std::string & label, std::vector<TreePointer
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool InternalTree::RecursiveSearch(const std::string & label, std::vector<TreePointer>::const_iterator & it) const {
|
bool InternalTree::RecursiveSearch(const std::string & label, std::vector<TreePointer>::const_iterator & it) const
|
||||||
|
{
|
||||||
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
||||||
if ((*it)->GetLabel() == label) {
|
if ((*it)->GetLabel() == label) {
|
||||||
return true;
|
return true;
|
||||||
@ -143,7 +141,8 @@ bool InternalTree::RecursiveSearch(const std::string & label, std::vector<TreePo
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool InternalTree::RecursiveSearch(const std::string & label, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const {
|
bool InternalTree::RecursiveSearch(const std::string & label, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const
|
||||||
|
{
|
||||||
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
||||||
if ((*it)->GetLabel() == label) {
|
if ((*it)->GetLabel() == label) {
|
||||||
parent = this;
|
parent = this;
|
||||||
@ -159,7 +158,8 @@ bool InternalTree::RecursiveSearch(const std::string & label, std::vector<TreePo
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool InternalTree::FlatSearch(const NTLabel & label, std::vector<TreePointer>::const_iterator & it) const {
|
bool InternalTree::FlatSearch(const NTLabel & label, std::vector<TreePointer>::const_iterator & it) const
|
||||||
|
{
|
||||||
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
||||||
if ((*it)->GetNTLabel() == label) {
|
if ((*it)->GetNTLabel() == label) {
|
||||||
return true;
|
return true;
|
||||||
@ -168,7 +168,8 @@ bool InternalTree::FlatSearch(const NTLabel & label, std::vector<TreePointer>::c
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool InternalTree::RecursiveSearch(const NTLabel & label, std::vector<TreePointer>::const_iterator & it) const {
|
bool InternalTree::RecursiveSearch(const NTLabel & label, std::vector<TreePointer>::const_iterator & it) const
|
||||||
|
{
|
||||||
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
||||||
if ((*it)->GetNTLabel() == label) {
|
if ((*it)->GetNTLabel() == label) {
|
||||||
return true;
|
return true;
|
||||||
@ -182,7 +183,8 @@ bool InternalTree::RecursiveSearch(const NTLabel & label, std::vector<TreePointe
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool InternalTree::RecursiveSearch(const NTLabel & label, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const {
|
bool InternalTree::RecursiveSearch(const NTLabel & label, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const
|
||||||
|
{
|
||||||
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
||||||
if ((*it)->GetNTLabel() == label) {
|
if ((*it)->GetNTLabel() == label) {
|
||||||
parent = this;
|
parent = this;
|
||||||
@ -198,7 +200,8 @@ bool InternalTree::RecursiveSearch(const NTLabel & label, std::vector<TreePointe
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool InternalTree::FlatSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it) const {
|
bool InternalTree::FlatSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it) const
|
||||||
|
{
|
||||||
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
||||||
if (std::binary_search(labels.begin(), labels.end(), (*it)->GetNTLabel())) {
|
if (std::binary_search(labels.begin(), labels.end(), (*it)->GetNTLabel())) {
|
||||||
return true;
|
return true;
|
||||||
@ -207,7 +210,8 @@ bool InternalTree::FlatSearch(const std::vector<NTLabel> & labels, std::vector<T
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool InternalTree::RecursiveSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it) const {
|
bool InternalTree::RecursiveSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it) const
|
||||||
|
{
|
||||||
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
||||||
if (std::binary_search(labels.begin(), labels.end(), (*it)->GetNTLabel())) {
|
if (std::binary_search(labels.begin(), labels.end(), (*it)->GetNTLabel())) {
|
||||||
return true;
|
return true;
|
||||||
@ -221,7 +225,8 @@ bool InternalTree::RecursiveSearch(const std::vector<NTLabel> & labels, std::vec
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool InternalTree::RecursiveSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const {
|
bool InternalTree::RecursiveSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const
|
||||||
|
{
|
||||||
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
for (it = m_children.begin(); it != m_children.end(); ++it) {
|
||||||
if (std::binary_search(labels.begin(), labels.end(), (*it)->GetNTLabel())) {
|
if (std::binary_search(labels.begin(), labels.end(), (*it)->GetNTLabel())) {
|
||||||
parent = this;
|
parent = this;
|
||||||
|
@ -101,18 +101,21 @@ class TreeState : public FFState
|
|||||||
TreePointer m_tree;
|
TreePointer m_tree;
|
||||||
public:
|
public:
|
||||||
TreeState(TreePointer tree)
|
TreeState(TreePointer tree)
|
||||||
:m_tree(tree)
|
:m_tree(tree) {
|
||||||
{}
|
}
|
||||||
|
|
||||||
TreePointer GetTree() const {
|
TreePointer GetTree() const {
|
||||||
return m_tree;
|
return m_tree;
|
||||||
}
|
}
|
||||||
|
|
||||||
int Compare(const FFState& other) const {return 0;};
|
int Compare(const FFState& other) const {
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
// Python-like generator that yields next nonterminal leaf on every call
|
// Python-like generator that yields next nonterminal leaf on every call
|
||||||
$generator(leafNT) {
|
$generator(leafNT)
|
||||||
|
{
|
||||||
std::vector<TreePointer>::iterator it;
|
std::vector<TreePointer>::iterator it;
|
||||||
InternalTree* tree;
|
InternalTree* tree;
|
||||||
leafNT(InternalTree* root = 0): tree(root) {}
|
leafNT(InternalTree* root = 0): tree(root) {}
|
||||||
@ -120,8 +123,7 @@ $generator(leafNT) {
|
|||||||
for (it = tree->GetChildren().begin(); it !=tree->GetChildren().end(); ++it) {
|
for (it = tree->GetChildren().begin(); it !=tree->GetChildren().end(); ++it) {
|
||||||
if (!(*it)->IsTerminal() && (*it)->GetLength() == 0) {
|
if (!(*it)->IsTerminal() && (*it)->GetLength() == 0) {
|
||||||
$yield(it);
|
$yield(it);
|
||||||
}
|
} else if ((*it)->GetLength() > 0) {
|
||||||
else if ((*it)->GetLength() > 0) {
|
|
||||||
if ((*it).get()) { // normal pointer to same object that TreePointer points to
|
if ((*it).get()) { // normal pointer to same object that TreePointer points to
|
||||||
$restart(tree = (*it).get());
|
$restart(tree = (*it).get());
|
||||||
}
|
}
|
||||||
@ -132,7 +134,8 @@ $generator(leafNT) {
|
|||||||
|
|
||||||
|
|
||||||
// Python-like generator that yields the parent of the next nonterminal leaf on every call
|
// Python-like generator that yields the parent of the next nonterminal leaf on every call
|
||||||
$generator(leafNTParent) {
|
$generator(leafNTParent)
|
||||||
|
{
|
||||||
std::vector<TreePointer>::iterator it;
|
std::vector<TreePointer>::iterator it;
|
||||||
InternalTree* tree;
|
InternalTree* tree;
|
||||||
leafNTParent(InternalTree* root = 0): tree(root) {}
|
leafNTParent(InternalTree* root = 0): tree(root) {}
|
||||||
@ -140,8 +143,7 @@ $generator(leafNTParent) {
|
|||||||
for (it = tree->GetChildren().begin(); it !=tree->GetChildren().end(); ++it) {
|
for (it = tree->GetChildren().begin(); it !=tree->GetChildren().end(); ++it) {
|
||||||
if (!(*it)->IsTerminal() && (*it)->GetLength() == 0) {
|
if (!(*it)->IsTerminal() && (*it)->GetLength() == 0) {
|
||||||
$yield(tree);
|
$yield(tree);
|
||||||
}
|
} else if ((*it)->GetLength() > 0) {
|
||||||
else if ((*it)->GetLength() > 0) {
|
|
||||||
if ((*it).get()) {
|
if ((*it).get()) {
|
||||||
$restart(tree = (*it).get());
|
$restart(tree = (*it).get());
|
||||||
}
|
}
|
||||||
@ -151,7 +153,8 @@ $generator(leafNTParent) {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Python-like generator that yields the next nonterminal leaf on every call, and also stores the path from the root of the tree to the nonterminal
|
// Python-like generator that yields the next nonterminal leaf on every call, and also stores the path from the root of the tree to the nonterminal
|
||||||
$generator(leafNTPath) {
|
$generator(leafNTPath)
|
||||||
|
{
|
||||||
std::vector<TreePointer>::iterator it;
|
std::vector<TreePointer>::iterator it;
|
||||||
InternalTree* tree;
|
InternalTree* tree;
|
||||||
std::vector<InternalTree*> * path;
|
std::vector<InternalTree*> * path;
|
||||||
@ -163,8 +166,7 @@ $generator(leafNTPath) {
|
|||||||
path->push_back((*it).get());
|
path->push_back((*it).get());
|
||||||
$yield(it);
|
$yield(it);
|
||||||
path->pop_back();
|
path->pop_back();
|
||||||
}
|
} else if ((*it)->GetLength() > 0) {
|
||||||
else if ((*it)->GetLength() > 0) {
|
|
||||||
if ((*it).get()) {
|
if ((*it).get()) {
|
||||||
$restart(tree = (*it).get());
|
$restart(tree = (*it).get());
|
||||||
}
|
}
|
||||||
|
@ -59,20 +59,24 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
bool GetHaveDefaultScores() { return m_haveDefaultScores; }
|
bool GetHaveDefaultScores() {
|
||||||
float GetDefaultScore( size_t i ) { return m_defaultScores[i]; }
|
return m_haveDefaultScores;
|
||||||
|
}
|
||||||
|
float GetDefaultScore( size_t i ) {
|
||||||
|
return m_defaultScores[i];
|
||||||
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
bool DecodeCondition(std::string s);
|
bool DecodeCondition(std::string s);
|
||||||
|
@ -146,8 +146,7 @@ void LexicalReorderingState::CopyScores(ScoreComponentCollection* accum, const
|
|||||||
const Scores &scoreSet = *cachedScores;
|
const Scores &scoreSet = *cachedScores;
|
||||||
if(m_configuration.CollapseScores()) {
|
if(m_configuration.CollapseScores()) {
|
||||||
scores[m_offset] = scoreSet[m_offset + reoType];
|
scores[m_offset] = scoreSet[m_offset + reoType];
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
std::fill(scores.begin() + m_offset, scores.begin() + m_offset + m_configuration.GetNumberOfTypes(), 0);
|
std::fill(scores.begin() + m_offset, scores.begin() + m_offset + m_configuration.GetNumberOfTypes(), 0);
|
||||||
scores[m_offset + reoType] = scoreSet[m_offset + reoType];
|
scores[m_offset + reoType] = scoreSet[m_offset + reoType];
|
||||||
}
|
}
|
||||||
@ -158,8 +157,7 @@ void LexicalReorderingState::CopyScores(ScoreComponentCollection* accum, const
|
|||||||
Scores scores(m_configuration.GetScoreProducer()->GetNumScoreComponents(),0);
|
Scores scores(m_configuration.GetScoreProducer()->GetNumScoreComponents(),0);
|
||||||
if(m_configuration.CollapseScores()) {
|
if(m_configuration.CollapseScores()) {
|
||||||
scores[m_offset] = m_configuration.GetScoreProducer()->GetDefaultScore(m_offset + reoType);
|
scores[m_offset] = m_configuration.GetScoreProducer()->GetDefaultScore(m_offset + reoType);
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
scores[m_offset + reoType] = m_configuration.GetScoreProducer()->GetDefaultScore(m_offset + reoType);
|
scores[m_offset + reoType] = m_configuration.GetScoreProducer()->GetDefaultScore(m_offset + reoType);
|
||||||
}
|
}
|
||||||
accum->PlusEquals(m_configuration.GetScoreProducer(), scores);
|
accum->PlusEquals(m_configuration.GetScoreProducer(), scores);
|
||||||
|
@ -19,7 +19,8 @@ using namespace std;
|
|||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
|
|
||||||
const std::string& SparseReorderingFeatureKey::Name (const string& wordListId) {
|
const std::string& SparseReorderingFeatureKey::Name (const string& wordListId)
|
||||||
|
{
|
||||||
static string kSep = "-";
|
static string kSep = "-";
|
||||||
static string name;
|
static string name;
|
||||||
ostringstream buf;
|
ostringstream buf;
|
||||||
@ -93,7 +94,8 @@ SparseReordering::SparseReordering(const map<string,string>& config, const Lexic
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void SparseReordering::PreCalculateFeatureNames(size_t index, const string& id, SparseReorderingFeatureKey::Side side, const Factor* factor, bool isCluster) {
|
void SparseReordering::PreCalculateFeatureNames(size_t index, const string& id, SparseReorderingFeatureKey::Side side, const Factor* factor, bool isCluster)
|
||||||
|
{
|
||||||
for (size_t type = SparseReorderingFeatureKey::Stack;
|
for (size_t type = SparseReorderingFeatureKey::Stack;
|
||||||
type <= SparseReorderingFeatureKey::Between; ++type) {
|
type <= SparseReorderingFeatureKey::Between; ++type) {
|
||||||
for (size_t position = SparseReorderingFeatureKey::First;
|
for (size_t position = SparseReorderingFeatureKey::First;
|
||||||
@ -108,7 +110,8 @@ void SparseReordering::PreCalculateFeatureNames(size_t index, const string& id,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void SparseReordering::ReadWordList(const string& filename, const string& id, SparseReorderingFeatureKey::Side side, vector<WordList>* pWordLists) {
|
void SparseReordering::ReadWordList(const string& filename, const string& id, SparseReorderingFeatureKey::Side side, vector<WordList>* pWordLists)
|
||||||
|
{
|
||||||
ifstream fh(filename.c_str());
|
ifstream fh(filename.c_str());
|
||||||
UTIL_THROW_IF(!fh, util::Exception, "Unable to open: " << filename);
|
UTIL_THROW_IF(!fh, util::Exception, "Unable to open: " << filename);
|
||||||
string line;
|
string line;
|
||||||
@ -123,7 +126,8 @@ void SparseReordering::ReadWordList(const string& filename, const string& id, Sp
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void SparseReordering::ReadClusterMap(const string& filename, const string& id, SparseReorderingFeatureKey::Side side, vector<ClusterMap>* pClusterMaps) {
|
void SparseReordering::ReadClusterMap(const string& filename, const string& id, SparseReorderingFeatureKey::Side side, vector<ClusterMap>* pClusterMaps)
|
||||||
|
{
|
||||||
pClusterMaps->push_back(ClusterMap());
|
pClusterMaps->push_back(ClusterMap());
|
||||||
pClusterMaps->back().first = id;
|
pClusterMaps->back().first = id;
|
||||||
util::FilePiece file(filename.c_str());
|
util::FilePiece file(filename.c_str());
|
||||||
@ -149,7 +153,8 @@ void SparseReordering::AddFeatures(
|
|||||||
SparseReorderingFeatureKey::Type type, SparseReorderingFeatureKey::Side side,
|
SparseReorderingFeatureKey::Type type, SparseReorderingFeatureKey::Side side,
|
||||||
const Word& word, SparseReorderingFeatureKey::Position position,
|
const Word& word, SparseReorderingFeatureKey::Position position,
|
||||||
LexicalReorderingState::ReorderingType reoType,
|
LexicalReorderingState::ReorderingType reoType,
|
||||||
ScoreComponentCollection* scores) const {
|
ScoreComponentCollection* scores) const
|
||||||
|
{
|
||||||
|
|
||||||
const Factor* wordFactor = word.GetFactor(0);
|
const Factor* wordFactor = word.GetFactor(0);
|
||||||
|
|
||||||
|
@ -53,8 +53,8 @@ struct SparseReorderingFeatureKey {
|
|||||||
SparseReorderingFeatureKey(size_t id_, Type type_, const Factor* word_, bool isCluster_,
|
SparseReorderingFeatureKey(size_t id_, Type type_, const Factor* word_, bool isCluster_,
|
||||||
Position position_, Side side_, LexicalReorderingState::ReorderingType reoType_)
|
Position position_, Side side_, LexicalReorderingState::ReorderingType reoType_)
|
||||||
: id(id_), type(type_), word(word_), isCluster(isCluster_),
|
: id(id_), type(type_), word(word_), isCluster(isCluster_),
|
||||||
position(position_), side(side_), reoType(reoType_)
|
position(position_), side(side_), reoType(reoType_) {
|
||||||
{}
|
}
|
||||||
|
|
||||||
const std::string& Name(const std::string& wordListId) ;
|
const std::string& Name(const std::string& wordListId) ;
|
||||||
};
|
};
|
||||||
|
@ -12,8 +12,9 @@ class MaxSpanFreeNonTermSource : public StatelessFeatureFunction
|
|||||||
public:
|
public:
|
||||||
MaxSpanFreeNonTermSource(const std::string &line);
|
MaxSpanFreeNonTermSource(const std::string &line);
|
||||||
|
|
||||||
virtual bool IsUseable(const FactorMask &mask) const
|
virtual bool IsUseable(const FactorMask &mask) const {
|
||||||
{ return true; }
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
virtual void EvaluateInIsolation(const Phrase &source
|
virtual void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
@ -28,16 +29,16 @@ public:
|
|||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
|
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
std::vector<float> DefaultWeights() const;
|
std::vector<float> DefaultWeights() const;
|
||||||
|
@ -31,8 +31,8 @@ public:
|
|||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const;
|
ScoreComponentCollection* accumulator) const;
|
||||||
|
@ -7,7 +7,8 @@
|
|||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
|
|
||||||
class KenOSMBase {
|
class KenOSMBase
|
||||||
|
{
|
||||||
public:
|
public:
|
||||||
virtual float Score(const lm::ngram::State&, const std::string&,
|
virtual float Score(const lm::ngram::State&, const std::string&,
|
||||||
lm::ngram::State&) const = 0;
|
lm::ngram::State&) const = 0;
|
||||||
@ -18,7 +19,8 @@ class KenOSMBase {
|
|||||||
};
|
};
|
||||||
|
|
||||||
template <class KenModel>
|
template <class KenModel>
|
||||||
class KenOSM : public KenOSMBase {
|
class KenOSM : public KenOSMBase
|
||||||
|
{
|
||||||
public:
|
public:
|
||||||
KenOSM(const std::string& file)
|
KenOSM(const std::string& file)
|
||||||
: m_kenlm(new KenModel(file.c_str())) {}
|
: m_kenlm(new KenModel(file.c_str())) {}
|
||||||
|
@ -19,7 +19,8 @@ OpSequenceModel::OpSequenceModel(const std::string &line)
|
|||||||
ReadParameters();
|
ReadParameters();
|
||||||
}
|
}
|
||||||
|
|
||||||
OpSequenceModel::~OpSequenceModel() {
|
OpSequenceModel::~OpSequenceModel()
|
||||||
|
{
|
||||||
delete OSM;
|
delete OSM;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -42,12 +42,12 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
|
@ -58,17 +58,17 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
|
@ -25,24 +25,24 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const ChartHypothesis& hypo,
|
void EvaluateWhenApplied(const ChartHypothesis& hypo,
|
||||||
ScoreComponentCollection*) const
|
ScoreComponentCollection*) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWithSourceContext(const InputType &input
|
void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
virtual void EvaluateInIsolation(const Phrase &source
|
virtual void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
|
@ -40,12 +40,9 @@ PhraseOrientationFeature::PhraseOrientationFeature(const std::string &line)
|
|||||||
|
|
||||||
void PhraseOrientationFeature::SetParameter(const std::string& key, const std::string& value)
|
void PhraseOrientationFeature::SetParameter(const std::string& key, const std::string& value)
|
||||||
{
|
{
|
||||||
if (key == "glueTargetLHS")
|
if (key == "glueTargetLHS") {
|
||||||
{
|
|
||||||
m_glueTargetLHSStr = value;
|
m_glueTargetLHSStr = value;
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
StatefulFeatureFunction::SetParameter(key, value);
|
StatefulFeatureFunction::SetParameter(key, value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -69,20 +66,17 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
// const Factor* targetLHS = currTarPhr.GetTargetLHS()[0];
|
// const Factor* targetLHS = currTarPhr.GetTargetLHS()[0];
|
||||||
// bool isGlueGrammarRule = false;
|
// bool isGlueGrammarRule = false;
|
||||||
|
|
||||||
IFFEATUREVERBOSE(2)
|
IFFEATUREVERBOSE(2) {
|
||||||
{
|
|
||||||
FEATUREVERBOSE(2, *currSrcPhr << std::endl);
|
FEATUREVERBOSE(2, *currSrcPhr << std::endl);
|
||||||
FEATUREVERBOSE(2, currTarPhr << std::endl);
|
FEATUREVERBOSE(2, currTarPhr << std::endl);
|
||||||
|
|
||||||
for (AlignmentInfo::const_iterator it=currTarPhr.GetAlignTerm().begin();
|
for (AlignmentInfo::const_iterator it=currTarPhr.GetAlignTerm().begin();
|
||||||
it!=currTarPhr.GetAlignTerm().end(); ++it)
|
it!=currTarPhr.GetAlignTerm().end(); ++it) {
|
||||||
{
|
|
||||||
FEATUREVERBOSE(2, "alignTerm " << it->first << " " << it->second << std::endl);
|
FEATUREVERBOSE(2, "alignTerm " << it->first << " " << it->second << std::endl);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (AlignmentInfo::const_iterator it=currTarPhr.GetAlignNonTerm().begin();
|
for (AlignmentInfo::const_iterator it=currTarPhr.GetAlignNonTerm().begin();
|
||||||
it!=currTarPhr.GetAlignNonTerm().end(); ++it)
|
it!=currTarPhr.GetAlignNonTerm().end(); ++it) {
|
||||||
{
|
|
||||||
FEATUREVERBOSE(2, "alignNonTerm " << it->first << " " << it->second << std::endl);
|
FEATUREVERBOSE(2, "alignNonTerm " << it->first << " " << it->second << std::endl);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -98,8 +92,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
// Determine & score orientations
|
// Determine & score orientations
|
||||||
|
|
||||||
for (AlignmentInfo::const_iterator it=currTarPhr.GetAlignNonTerm().begin();
|
for (AlignmentInfo::const_iterator it=currTarPhr.GetAlignNonTerm().begin();
|
||||||
it!=currTarPhr.GetAlignNonTerm().end(); ++it)
|
it!=currTarPhr.GetAlignNonTerm().end(); ++it) {
|
||||||
{
|
|
||||||
size_t sourceIndex = it->first;
|
size_t sourceIndex = it->first;
|
||||||
size_t targetIndex = it->second;
|
size_t targetIndex = it->second;
|
||||||
size_t nonTermIndex = nonTermIndexMap[targetIndex];
|
size_t nonTermIndex = nonTermIndexMap[targetIndex];
|
||||||
@ -110,8 +103,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
const ChartHypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndex);
|
const ChartHypothesis *prevHypo = hypo.GetPrevHypo(nonTermIndex);
|
||||||
const TargetPhrase &prevTarPhr = prevHypo->GetCurrTargetPhrase();
|
const TargetPhrase &prevTarPhr = prevHypo->GetCurrTargetPhrase();
|
||||||
|
|
||||||
if (const PhraseProperty *property = prevTarPhr.GetProperty("Orientation"))
|
if (const PhraseProperty *property = prevTarPhr.GetProperty("Orientation")) {
|
||||||
{
|
|
||||||
const OrientationPhraseProperty *orientationPhraseProperty = static_cast<const OrientationPhraseProperty*>(property);
|
const OrientationPhraseProperty *orientationPhraseProperty = static_cast<const OrientationPhraseProperty*>(property);
|
||||||
|
|
||||||
FEATUREVERBOSE(5, "orientationPhraseProperty: "
|
FEATUREVERBOSE(5, "orientationPhraseProperty: "
|
||||||
@ -133,11 +125,9 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
|
|
||||||
Moses::GHKM::PhraseOrientation::REO_CLASS l2rOrientation = phraseOrientation.GetOrientationInfo(sourceIndex,sourceIndex,Moses::GHKM::PhraseOrientation::REO_DIR_L2R);
|
Moses::GHKM::PhraseOrientation::REO_CLASS l2rOrientation = phraseOrientation.GetOrientationInfo(sourceIndex,sourceIndex,Moses::GHKM::PhraseOrientation::REO_DIR_L2R);
|
||||||
|
|
||||||
IFFEATUREVERBOSE(2)
|
IFFEATUREVERBOSE(2) {
|
||||||
{
|
|
||||||
FEATUREVERBOSE(2, "l2rOrientation ");
|
FEATUREVERBOSE(2, "l2rOrientation ");
|
||||||
switch (l2rOrientation)
|
switch (l2rOrientation) {
|
||||||
{
|
|
||||||
case Moses::GHKM::PhraseOrientation::REO_CLASS_LEFT:
|
case Moses::GHKM::PhraseOrientation::REO_CLASS_LEFT:
|
||||||
FEATUREVERBOSE2(2, "mono" << std::endl);
|
FEATUREVERBOSE2(2, "mono" << std::endl);
|
||||||
break;
|
break;
|
||||||
@ -164,8 +154,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
bool delayedScoringL2R = false;
|
bool delayedScoringL2R = false;
|
||||||
|
|
||||||
if ( ((targetIndex == 0) || !phraseOrientation.TargetSpanIsAligned(0,targetIndex)) // boundary non-terminal in rule-initial position (left boundary)
|
if ( ((targetIndex == 0) || !phraseOrientation.TargetSpanIsAligned(0,targetIndex)) // boundary non-terminal in rule-initial position (left boundary)
|
||||||
&& (currTarPhrLHS != m_glueTargetLHS) ) // and not glue rule
|
&& (currTarPhrLHS != m_glueTargetLHS) ) { // and not glue rule
|
||||||
{
|
|
||||||
// delay left-to-right scoring
|
// delay left-to-right scoring
|
||||||
|
|
||||||
FEATUREVERBOSE(3, "Left boundary");
|
FEATUREVERBOSE(3, "Left boundary");
|
||||||
@ -180,13 +169,10 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
FEATUREVERBOSE(4, "previousSourceSpanIsAligned = " << previousSourceSpanIsAligned << std::endl);
|
FEATUREVERBOSE(4, "previousSourceSpanIsAligned = " << previousSourceSpanIsAligned << std::endl);
|
||||||
FEATUREVERBOSE(4, "followingSourceSpanIsAligned = " << followingSourceSpanIsAligned << std::endl;);
|
FEATUREVERBOSE(4, "followingSourceSpanIsAligned = " << followingSourceSpanIsAligned << std::endl;);
|
||||||
|
|
||||||
if (previousSourceSpanIsAligned && followingSourceSpanIsAligned)
|
if (previousSourceSpanIsAligned && followingSourceSpanIsAligned) {
|
||||||
{
|
|
||||||
// discontinuous
|
// discontinuous
|
||||||
l2rOrientation = Moses::GHKM::PhraseOrientation::REO_CLASS_DLEFT;
|
l2rOrientation = Moses::GHKM::PhraseOrientation::REO_CLASS_DLEFT;
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
FEATUREVERBOSE(3, "Delaying left-to-right scoring" << std::endl);
|
FEATUREVERBOSE(3, "Delaying left-to-right scoring" << std::endl);
|
||||||
|
|
||||||
delayedScoringL2R = true;
|
delayedScoringL2R = true;
|
||||||
@ -202,25 +188,20 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
scoresL2R.push_back( std::log(orientationPhraseProperty->GetLeftToRightProbabilitySwap()) );
|
scoresL2R.push_back( std::log(orientationPhraseProperty->GetLeftToRightProbabilitySwap()) );
|
||||||
scoresL2R.push_back( std::log(orientationPhraseProperty->GetLeftToRightProbabilityDiscontinuous()) );
|
scoresL2R.push_back( std::log(orientationPhraseProperty->GetLeftToRightProbabilityDiscontinuous()) );
|
||||||
std::vector<float> weightedScoresL2R;
|
std::vector<float> weightedScoresL2R;
|
||||||
for ( size_t i=0; i<3;++i )
|
for ( size_t i=0; i<3; ++i ) {
|
||||||
{
|
|
||||||
weightedScoresL2R.push_back( weightsVector[i] * scoresL2R[i] );
|
weightedScoresL2R.push_back( weightsVector[i] * scoresL2R[i] );
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t heuristicScoreIndex = 0;
|
size_t heuristicScoreIndex = 0;
|
||||||
for (size_t i=1; i<3; ++i)
|
for (size_t i=1; i<3; ++i) {
|
||||||
{
|
if (possibleFutureOrientationsL2R[i]) {
|
||||||
if (possibleFutureOrientationsL2R[i])
|
if (weightedScoresL2R[i] > weightedScoresL2R[heuristicScoreIndex]) {
|
||||||
{
|
|
||||||
if (weightedScoresL2R[i] > weightedScoresL2R[heuristicScoreIndex])
|
|
||||||
{
|
|
||||||
heuristicScoreIndex = i;
|
heuristicScoreIndex = i;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
IFFEATUREVERBOSE(5)
|
IFFEATUREVERBOSE(5) {
|
||||||
{
|
|
||||||
FEATUREVERBOSE(5, "Heuristic score computation (L2R): "
|
FEATUREVERBOSE(5, "Heuristic score computation (L2R): "
|
||||||
<< "heuristicScoreIndex= " << heuristicScoreIndex);
|
<< "heuristicScoreIndex= " << heuristicScoreIndex);
|
||||||
for (size_t i=0; i<3; ++i)
|
for (size_t i=0; i<3; ++i)
|
||||||
@ -231,8 +212,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
FEATUREVERBOSE2(5, " weightedScoresL2R[" << i << "]= " << weightedScoresL2R[i]);
|
FEATUREVERBOSE2(5, " weightedScoresL2R[" << i << "]= " << weightedScoresL2R[i]);
|
||||||
for (size_t i=0; i<3; ++i)
|
for (size_t i=0; i<3; ++i)
|
||||||
FEATUREVERBOSE2(5, " possibleFutureOrientationsL2R[" << i << "]= " << possibleFutureOrientationsL2R[i]);
|
FEATUREVERBOSE2(5, " possibleFutureOrientationsL2R[" << i << "]= " << possibleFutureOrientationsL2R[i]);
|
||||||
if ( possibleFutureOrientationsL2R == 0x7 )
|
if ( possibleFutureOrientationsL2R == 0x7 ) {
|
||||||
{
|
|
||||||
FEATUREVERBOSE2(5, " (all orientations possible)");
|
FEATUREVERBOSE2(5, " (all orientations possible)");
|
||||||
}
|
}
|
||||||
FEATUREVERBOSE2(5, std::endl);
|
FEATUREVERBOSE2(5, std::endl);
|
||||||
@ -241,8 +221,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
newScores[heuristicScoreIndex] += scoresL2R[heuristicScoreIndex];
|
newScores[heuristicScoreIndex] += scoresL2R[heuristicScoreIndex];
|
||||||
state->SetLeftBoundaryL2R(scoresL2R, heuristicScoreIndex, possibleFutureOrientationsL2R, prevState);
|
state->SetLeftBoundaryL2R(scoresL2R, heuristicScoreIndex, possibleFutureOrientationsL2R, prevState);
|
||||||
|
|
||||||
if ( (possibleFutureOrientationsL2R & prevState->m_leftBoundaryNonTerminalL2RPossibleFutureOrientations) == 0x4 )
|
if ( (possibleFutureOrientationsL2R & prevState->m_leftBoundaryNonTerminalL2RPossibleFutureOrientations) == 0x4 ) {
|
||||||
{
|
|
||||||
// recursive: discontinuous orientation
|
// recursive: discontinuous orientation
|
||||||
FEATUREVERBOSE(5, "previous state: L2R discontinuous orientation "
|
FEATUREVERBOSE(5, "previous state: L2R discontinuous orientation "
|
||||||
<< possibleFutureOrientationsL2R << " & " << prevState->m_leftBoundaryNonTerminalL2RPossibleFutureOrientations
|
<< possibleFutureOrientationsL2R << " & " << prevState->m_leftBoundaryNonTerminalL2RPossibleFutureOrientations
|
||||||
@ -254,10 +233,8 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!delayedScoringL2R)
|
if (!delayedScoringL2R) {
|
||||||
{
|
switch (l2rOrientation) {
|
||||||
switch (l2rOrientation)
|
|
||||||
{
|
|
||||||
case Moses::GHKM::PhraseOrientation::REO_CLASS_LEFT:
|
case Moses::GHKM::PhraseOrientation::REO_CLASS_LEFT:
|
||||||
newScores[0] += std::log(orientationPhraseProperty->GetLeftToRightProbabilityMono());
|
newScores[0] += std::log(orientationPhraseProperty->GetLeftToRightProbabilityMono());
|
||||||
// if sub-derivation has left-boundary non-terminal:
|
// if sub-derivation has left-boundary non-terminal:
|
||||||
@ -301,11 +278,9 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
|
|
||||||
Moses::GHKM::PhraseOrientation::REO_CLASS r2lOrientation = phraseOrientation.GetOrientationInfo(sourceIndex,sourceIndex,Moses::GHKM::PhraseOrientation::REO_DIR_R2L);
|
Moses::GHKM::PhraseOrientation::REO_CLASS r2lOrientation = phraseOrientation.GetOrientationInfo(sourceIndex,sourceIndex,Moses::GHKM::PhraseOrientation::REO_DIR_R2L);
|
||||||
|
|
||||||
IFFEATUREVERBOSE(2)
|
IFFEATUREVERBOSE(2) {
|
||||||
{
|
|
||||||
FEATUREVERBOSE(2, "r2lOrientation ");
|
FEATUREVERBOSE(2, "r2lOrientation ");
|
||||||
switch (r2lOrientation)
|
switch (r2lOrientation) {
|
||||||
{
|
|
||||||
case Moses::GHKM::PhraseOrientation::REO_CLASS_LEFT:
|
case Moses::GHKM::PhraseOrientation::REO_CLASS_LEFT:
|
||||||
FEATUREVERBOSE2(2, "mono" << std::endl);
|
FEATUREVERBOSE2(2, "mono" << std::endl);
|
||||||
break;
|
break;
|
||||||
@ -332,8 +307,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
bool delayedScoringR2L = false;
|
bool delayedScoringR2L = false;
|
||||||
|
|
||||||
if ( ((targetIndex == currTarPhr.GetSize()-1) || !phraseOrientation.TargetSpanIsAligned(targetIndex,currTarPhr.GetSize()-1)) // boundary non-terminal in rule-final position (right boundary)
|
if ( ((targetIndex == currTarPhr.GetSize()-1) || !phraseOrientation.TargetSpanIsAligned(targetIndex,currTarPhr.GetSize()-1)) // boundary non-terminal in rule-final position (right boundary)
|
||||||
&& (currTarPhrLHS != m_glueTargetLHS) ) // and not glue rule
|
&& (currTarPhrLHS != m_glueTargetLHS) ) { // and not glue rule
|
||||||
{
|
|
||||||
// delay right-to-left scoring
|
// delay right-to-left scoring
|
||||||
|
|
||||||
FEATUREVERBOSE(3, "Right boundary");
|
FEATUREVERBOSE(3, "Right boundary");
|
||||||
@ -348,13 +322,10 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
FEATUREVERBOSE(4, "previousSourceSpanIsAligned = " << previousSourceSpanIsAligned << std::endl);
|
FEATUREVERBOSE(4, "previousSourceSpanIsAligned = " << previousSourceSpanIsAligned << std::endl);
|
||||||
FEATUREVERBOSE(4, "followingSourceSpanIsAligned = " << followingSourceSpanIsAligned << std::endl;);
|
FEATUREVERBOSE(4, "followingSourceSpanIsAligned = " << followingSourceSpanIsAligned << std::endl;);
|
||||||
|
|
||||||
if (previousSourceSpanIsAligned && followingSourceSpanIsAligned)
|
if (previousSourceSpanIsAligned && followingSourceSpanIsAligned) {
|
||||||
{
|
|
||||||
// discontinuous
|
// discontinuous
|
||||||
r2lOrientation = Moses::GHKM::PhraseOrientation::REO_CLASS_DLEFT;
|
r2lOrientation = Moses::GHKM::PhraseOrientation::REO_CLASS_DLEFT;
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
FEATUREVERBOSE(3, "Delaying right-to-left scoring" << std::endl);
|
FEATUREVERBOSE(3, "Delaying right-to-left scoring" << std::endl);
|
||||||
|
|
||||||
delayedScoringR2L = true;
|
delayedScoringR2L = true;
|
||||||
@ -370,25 +341,20 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
scoresR2L.push_back( std::log(orientationPhraseProperty->GetRightToLeftProbabilitySwap()) );
|
scoresR2L.push_back( std::log(orientationPhraseProperty->GetRightToLeftProbabilitySwap()) );
|
||||||
scoresR2L.push_back( std::log(orientationPhraseProperty->GetRightToLeftProbabilityDiscontinuous()) );
|
scoresR2L.push_back( std::log(orientationPhraseProperty->GetRightToLeftProbabilityDiscontinuous()) );
|
||||||
std::vector<float> weightedScoresR2L;
|
std::vector<float> weightedScoresR2L;
|
||||||
for ( size_t i=0; i<3;++i )
|
for ( size_t i=0; i<3; ++i ) {
|
||||||
{
|
|
||||||
weightedScoresR2L.push_back( weightsVector[m_offsetR2LScores+i] * scoresR2L[i] );
|
weightedScoresR2L.push_back( weightsVector[m_offsetR2LScores+i] * scoresR2L[i] );
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t heuristicScoreIndex = 0;
|
size_t heuristicScoreIndex = 0;
|
||||||
for (size_t i=1; i<3; ++i)
|
for (size_t i=1; i<3; ++i) {
|
||||||
{
|
if (possibleFutureOrientationsR2L[i]) {
|
||||||
if (possibleFutureOrientationsR2L[i])
|
if (weightedScoresR2L[i] > weightedScoresR2L[heuristicScoreIndex]) {
|
||||||
{
|
|
||||||
if (weightedScoresR2L[i] > weightedScoresR2L[heuristicScoreIndex])
|
|
||||||
{
|
|
||||||
heuristicScoreIndex = i;
|
heuristicScoreIndex = i;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
IFFEATUREVERBOSE(5)
|
IFFEATUREVERBOSE(5) {
|
||||||
{
|
|
||||||
FEATUREVERBOSE(5, "Heuristic score computation (R2L): "
|
FEATUREVERBOSE(5, "Heuristic score computation (R2L): "
|
||||||
<< "heuristicScoreIndex= " << heuristicScoreIndex);
|
<< "heuristicScoreIndex= " << heuristicScoreIndex);
|
||||||
for (size_t i=0; i<3; ++i)
|
for (size_t i=0; i<3; ++i)
|
||||||
@ -399,8 +365,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
FEATUREVERBOSE2(5, " weightedScoresR2L[" << i << "]= " << weightedScoresR2L[i]);
|
FEATUREVERBOSE2(5, " weightedScoresR2L[" << i << "]= " << weightedScoresR2L[i]);
|
||||||
for (size_t i=0; i<3; ++i)
|
for (size_t i=0; i<3; ++i)
|
||||||
FEATUREVERBOSE2(5, " possibleFutureOrientationsR2L[" << i << "]= " << possibleFutureOrientationsR2L[i]);
|
FEATUREVERBOSE2(5, " possibleFutureOrientationsR2L[" << i << "]= " << possibleFutureOrientationsR2L[i]);
|
||||||
if ( possibleFutureOrientationsR2L == 0x7 )
|
if ( possibleFutureOrientationsR2L == 0x7 ) {
|
||||||
{
|
|
||||||
FEATUREVERBOSE2(5, " (all orientations possible)");
|
FEATUREVERBOSE2(5, " (all orientations possible)");
|
||||||
}
|
}
|
||||||
FEATUREVERBOSE2(5, std::endl);
|
FEATUREVERBOSE2(5, std::endl);
|
||||||
@ -409,8 +374,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
newScores[m_offsetR2LScores+heuristicScoreIndex] += scoresR2L[heuristicScoreIndex];
|
newScores[m_offsetR2LScores+heuristicScoreIndex] += scoresR2L[heuristicScoreIndex];
|
||||||
state->SetRightBoundaryR2L(scoresR2L, heuristicScoreIndex, possibleFutureOrientationsR2L, prevState);
|
state->SetRightBoundaryR2L(scoresR2L, heuristicScoreIndex, possibleFutureOrientationsR2L, prevState);
|
||||||
|
|
||||||
if ( (possibleFutureOrientationsR2L & prevState->m_rightBoundaryNonTerminalR2LPossibleFutureOrientations) == 0x4 )
|
if ( (possibleFutureOrientationsR2L & prevState->m_rightBoundaryNonTerminalR2LPossibleFutureOrientations) == 0x4 ) {
|
||||||
{
|
|
||||||
// recursive: discontinuous orientation
|
// recursive: discontinuous orientation
|
||||||
FEATUREVERBOSE(5, "previous state: R2L discontinuous orientation "
|
FEATUREVERBOSE(5, "previous state: R2L discontinuous orientation "
|
||||||
<< possibleFutureOrientationsR2L << " & " << prevState->m_rightBoundaryNonTerminalR2LPossibleFutureOrientations
|
<< possibleFutureOrientationsR2L << " & " << prevState->m_rightBoundaryNonTerminalR2LPossibleFutureOrientations
|
||||||
@ -422,10 +386,8 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!delayedScoringR2L)
|
if (!delayedScoringR2L) {
|
||||||
{
|
switch (r2lOrientation) {
|
||||||
switch (r2lOrientation)
|
|
||||||
{
|
|
||||||
case Moses::GHKM::PhraseOrientation::REO_CLASS_LEFT:
|
case Moses::GHKM::PhraseOrientation::REO_CLASS_LEFT:
|
||||||
newScores[m_offsetR2LScores+0] += std::log(orientationPhraseProperty->GetRightToLeftProbabilityMono());
|
newScores[m_offsetR2LScores+0] += std::log(orientationPhraseProperty->GetRightToLeftProbabilityMono());
|
||||||
// if sub-derivation has right-boundary non-terminal:
|
// if sub-derivation has right-boundary non-terminal:
|
||||||
@ -463,9 +425,7 @@ FFState* PhraseOrientationFeature::EvaluateWhenApplied(
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
// abort with error message if the phrase does not translate an unknown word
|
// abort with error message if the phrase does not translate an unknown word
|
||||||
UTIL_THROW_IF2(!prevTarPhr.GetWord(0).IsOOV(), GetScoreProducerDescription()
|
UTIL_THROW_IF2(!prevTarPhr.GetWord(0).IsOOV(), GetScoreProducerDescription()
|
||||||
<< ": Missing Orientation property. "
|
<< ": Missing Orientation property. "
|
||||||
@ -483,43 +443,30 @@ void PhraseOrientationFeature::LeftBoundaryL2RScoreRecursive(int featureID,
|
|||||||
const std::bitset<3> orientation,
|
const std::bitset<3> orientation,
|
||||||
std::vector<float>& newScores) const
|
std::vector<float>& newScores) const
|
||||||
{
|
{
|
||||||
if (state->m_leftBoundaryIsSet)
|
if (state->m_leftBoundaryIsSet) {
|
||||||
{
|
|
||||||
// subtract heuristic score from subderivation
|
// subtract heuristic score from subderivation
|
||||||
newScores[state->m_leftBoundaryNonTerminalL2RHeuristicScoreIndex] -= state->m_leftBoundaryNonTerminalL2RScores[state->m_leftBoundaryNonTerminalL2RHeuristicScoreIndex];
|
newScores[state->m_leftBoundaryNonTerminalL2RHeuristicScoreIndex] -= state->m_leftBoundaryNonTerminalL2RScores[state->m_leftBoundaryNonTerminalL2RHeuristicScoreIndex];
|
||||||
|
|
||||||
// add actual score
|
// add actual score
|
||||||
std::bitset<3> recursiveOrientation = orientation;
|
std::bitset<3> recursiveOrientation = orientation;
|
||||||
if ( (orientation == 0x4) || (orientation == 0x0) )
|
if ( (orientation == 0x4) || (orientation == 0x0) ) {
|
||||||
{
|
|
||||||
// discontinuous
|
// discontinuous
|
||||||
newScores[2] += state->GetLeftBoundaryL2RScoreDiscontinuous();
|
newScores[2] += state->GetLeftBoundaryL2RScoreDiscontinuous();
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
recursiveOrientation &= state->m_leftBoundaryNonTerminalL2RPossibleFutureOrientations;
|
recursiveOrientation &= state->m_leftBoundaryNonTerminalL2RPossibleFutureOrientations;
|
||||||
if ( recursiveOrientation == 0x1 )
|
if ( recursiveOrientation == 0x1 ) {
|
||||||
{
|
|
||||||
// monotone
|
// monotone
|
||||||
newScores[0] += state->GetLeftBoundaryL2RScoreMono();
|
newScores[0] += state->GetLeftBoundaryL2RScoreMono();
|
||||||
}
|
} else if ( recursiveOrientation == 0x2 ) {
|
||||||
else if ( recursiveOrientation == 0x2 )
|
|
||||||
{
|
|
||||||
// swap
|
// swap
|
||||||
newScores[1] += state->GetLeftBoundaryL2RScoreSwap();
|
newScores[1] += state->GetLeftBoundaryL2RScoreSwap();
|
||||||
}
|
} else if ( recursiveOrientation == 0x4 ) {
|
||||||
else if ( recursiveOrientation == 0x4 )
|
|
||||||
{
|
|
||||||
// discontinuous
|
// discontinuous
|
||||||
newScores[2] += state->GetLeftBoundaryL2RScoreDiscontinuous();
|
newScores[2] += state->GetLeftBoundaryL2RScoreDiscontinuous();
|
||||||
}
|
} else if ( recursiveOrientation == 0x0 ) {
|
||||||
else if ( recursiveOrientation == 0x0 )
|
|
||||||
{
|
|
||||||
// discontinuous
|
// discontinuous
|
||||||
newScores[2] += state->GetLeftBoundaryL2RScoreDiscontinuous();
|
newScores[2] += state->GetLeftBoundaryL2RScoreDiscontinuous();
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
UTIL_THROW2(GetScoreProducerDescription()
|
UTIL_THROW2(GetScoreProducerDescription()
|
||||||
<< ": Error in recursive scoring.");
|
<< ": Error in recursive scoring.");
|
||||||
}
|
}
|
||||||
@ -528,14 +475,11 @@ void PhraseOrientationFeature::LeftBoundaryL2RScoreRecursive(int featureID,
|
|||||||
FEATUREVERBOSE(6, "Left boundary recursion: " << orientation << " & " << state->m_leftBoundaryNonTerminalL2RPossibleFutureOrientations << " = " << recursiveOrientation
|
FEATUREVERBOSE(6, "Left boundary recursion: " << orientation << " & " << state->m_leftBoundaryNonTerminalL2RPossibleFutureOrientations << " = " << recursiveOrientation
|
||||||
<< " --- Subtracted heuristic score: " << state->m_leftBoundaryNonTerminalL2RScores[state->m_leftBoundaryNonTerminalL2RHeuristicScoreIndex] << std::endl);
|
<< " --- Subtracted heuristic score: " << state->m_leftBoundaryNonTerminalL2RScores[state->m_leftBoundaryNonTerminalL2RHeuristicScoreIndex] << std::endl);
|
||||||
|
|
||||||
if (!state->m_leftBoundaryRecursionGuard)
|
if (!state->m_leftBoundaryRecursionGuard) {
|
||||||
{
|
|
||||||
// recursive call
|
// recursive call
|
||||||
const PhraseOrientationFeatureState* prevState = state->m_leftBoundaryPrevState;
|
const PhraseOrientationFeatureState* prevState = state->m_leftBoundaryPrevState;
|
||||||
LeftBoundaryL2RScoreRecursive(featureID, prevState, recursiveOrientation, newScores);
|
LeftBoundaryL2RScoreRecursive(featureID, prevState, recursiveOrientation, newScores);
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
FEATUREVERBOSE(6, "m_leftBoundaryRecursionGuard" << std::endl);
|
FEATUREVERBOSE(6, "m_leftBoundaryRecursionGuard" << std::endl);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -546,43 +490,30 @@ void PhraseOrientationFeature::RightBoundaryR2LScoreRecursive(int featureID,
|
|||||||
const std::bitset<3> orientation,
|
const std::bitset<3> orientation,
|
||||||
std::vector<float>& newScores) const
|
std::vector<float>& newScores) const
|
||||||
{
|
{
|
||||||
if (state->m_rightBoundaryIsSet)
|
if (state->m_rightBoundaryIsSet) {
|
||||||
{
|
|
||||||
// subtract heuristic score from subderivation
|
// subtract heuristic score from subderivation
|
||||||
newScores[m_offsetR2LScores+state->m_rightBoundaryNonTerminalR2LHeuristicScoreIndex] -= state->m_rightBoundaryNonTerminalR2LScores[state->m_rightBoundaryNonTerminalR2LHeuristicScoreIndex];
|
newScores[m_offsetR2LScores+state->m_rightBoundaryNonTerminalR2LHeuristicScoreIndex] -= state->m_rightBoundaryNonTerminalR2LScores[state->m_rightBoundaryNonTerminalR2LHeuristicScoreIndex];
|
||||||
|
|
||||||
// add actual score
|
// add actual score
|
||||||
std::bitset<3> recursiveOrientation = orientation;
|
std::bitset<3> recursiveOrientation = orientation;
|
||||||
if ( (orientation == 0x4) || (orientation == 0x0) )
|
if ( (orientation == 0x4) || (orientation == 0x0) ) {
|
||||||
{
|
|
||||||
// discontinuous
|
// discontinuous
|
||||||
newScores[m_offsetR2LScores+2] += state->GetRightBoundaryR2LScoreDiscontinuous();
|
newScores[m_offsetR2LScores+2] += state->GetRightBoundaryR2LScoreDiscontinuous();
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
recursiveOrientation &= state->m_rightBoundaryNonTerminalR2LPossibleFutureOrientations;
|
recursiveOrientation &= state->m_rightBoundaryNonTerminalR2LPossibleFutureOrientations;
|
||||||
if ( recursiveOrientation == 0x1 )
|
if ( recursiveOrientation == 0x1 ) {
|
||||||
{
|
|
||||||
// monotone
|
// monotone
|
||||||
newScores[m_offsetR2LScores+0] += state->GetRightBoundaryR2LScoreMono();
|
newScores[m_offsetR2LScores+0] += state->GetRightBoundaryR2LScoreMono();
|
||||||
}
|
} else if ( recursiveOrientation == 0x2 ) {
|
||||||
else if ( recursiveOrientation == 0x2 )
|
|
||||||
{
|
|
||||||
// swap
|
// swap
|
||||||
newScores[m_offsetR2LScores+1] += state->GetRightBoundaryR2LScoreSwap();
|
newScores[m_offsetR2LScores+1] += state->GetRightBoundaryR2LScoreSwap();
|
||||||
}
|
} else if ( recursiveOrientation == 0x4 ) {
|
||||||
else if ( recursiveOrientation == 0x4 )
|
|
||||||
{
|
|
||||||
// discontinuous
|
// discontinuous
|
||||||
newScores[m_offsetR2LScores+2] += state->GetRightBoundaryR2LScoreDiscontinuous();
|
newScores[m_offsetR2LScores+2] += state->GetRightBoundaryR2LScoreDiscontinuous();
|
||||||
}
|
} else if ( recursiveOrientation == 0x0 ) {
|
||||||
else if ( recursiveOrientation == 0x0 )
|
|
||||||
{
|
|
||||||
// discontinuous
|
// discontinuous
|
||||||
newScores[m_offsetR2LScores+2] += state->GetRightBoundaryR2LScoreDiscontinuous();
|
newScores[m_offsetR2LScores+2] += state->GetRightBoundaryR2LScoreDiscontinuous();
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
UTIL_THROW2(GetScoreProducerDescription()
|
UTIL_THROW2(GetScoreProducerDescription()
|
||||||
<< ": Error in recursive scoring.");
|
<< ": Error in recursive scoring.");
|
||||||
}
|
}
|
||||||
@ -591,14 +522,11 @@ void PhraseOrientationFeature::RightBoundaryR2LScoreRecursive(int featureID,
|
|||||||
FEATUREVERBOSE(6, "Right boundary recursion: " << orientation << " & " << state->m_rightBoundaryNonTerminalR2LPossibleFutureOrientations << " = " << recursiveOrientation
|
FEATUREVERBOSE(6, "Right boundary recursion: " << orientation << " & " << state->m_rightBoundaryNonTerminalR2LPossibleFutureOrientations << " = " << recursiveOrientation
|
||||||
<< " --- Subtracted heuristic score: " << state->m_rightBoundaryNonTerminalR2LScores[state->m_rightBoundaryNonTerminalR2LHeuristicScoreIndex] << std::endl);
|
<< " --- Subtracted heuristic score: " << state->m_rightBoundaryNonTerminalR2LScores[state->m_rightBoundaryNonTerminalR2LHeuristicScoreIndex] << std::endl);
|
||||||
|
|
||||||
if (!state->m_rightBoundaryRecursionGuard)
|
if (!state->m_rightBoundaryRecursionGuard) {
|
||||||
{
|
|
||||||
// recursive call
|
// recursive call
|
||||||
const PhraseOrientationFeatureState* prevState = state->m_rightBoundaryPrevState;
|
const PhraseOrientationFeatureState* prevState = state->m_rightBoundaryPrevState;
|
||||||
RightBoundaryR2LScoreRecursive(featureID, prevState, recursiveOrientation, newScores);
|
RightBoundaryR2LScoreRecursive(featureID, prevState, recursiveOrientation, newScores);
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
FEATUREVERBOSE(6, "m_rightBoundaryRecursionGuard" << std::endl);
|
FEATUREVERBOSE(6, "m_rightBoundaryRecursionGuard" << std::endl);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -36,16 +36,14 @@ public:
|
|||||||
, m_leftBoundaryRecursionGuard(false)
|
, m_leftBoundaryRecursionGuard(false)
|
||||||
, m_rightBoundaryRecursionGuard(false)
|
, m_rightBoundaryRecursionGuard(false)
|
||||||
, m_leftBoundaryIsSet(false)
|
, m_leftBoundaryIsSet(false)
|
||||||
, m_rightBoundaryIsSet(false)
|
, m_rightBoundaryIsSet(false) {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void SetLeftBoundaryL2R(const std::vector<float> &scores,
|
void SetLeftBoundaryL2R(const std::vector<float> &scores,
|
||||||
size_t heuristicScoreIndex,
|
size_t heuristicScoreIndex,
|
||||||
std::bitset<3> &possibleFutureOrientations,
|
std::bitset<3> &possibleFutureOrientations,
|
||||||
const PhraseOrientationFeatureState* prevState)
|
const PhraseOrientationFeatureState* prevState) {
|
||||||
{
|
for (size_t i=0; i<3; ++i) {
|
||||||
for (size_t i=0; i<3; ++i)
|
|
||||||
{
|
|
||||||
m_leftBoundaryNonTerminalL2RScores[i] = scores[i];
|
m_leftBoundaryNonTerminalL2RScores[i] = scores[i];
|
||||||
m_leftBoundaryNonTerminalL2RPossibleFutureOrientations[i] = possibleFutureOrientations[i];
|
m_leftBoundaryNonTerminalL2RPossibleFutureOrientations[i] = possibleFutureOrientations[i];
|
||||||
}
|
}
|
||||||
@ -57,10 +55,8 @@ public:
|
|||||||
void SetRightBoundaryR2L(const std::vector<float> &scores,
|
void SetRightBoundaryR2L(const std::vector<float> &scores,
|
||||||
size_t heuristicScoreIndex,
|
size_t heuristicScoreIndex,
|
||||||
std::bitset<3> &possibleFutureOrientations,
|
std::bitset<3> &possibleFutureOrientations,
|
||||||
const PhraseOrientationFeatureState* prevState)
|
const PhraseOrientationFeatureState* prevState) {
|
||||||
{
|
for (size_t i=0; i<3; ++i) {
|
||||||
for (size_t i=0; i<3; ++i)
|
|
||||||
{
|
|
||||||
m_rightBoundaryNonTerminalR2LScores[i] = scores[i];
|
m_rightBoundaryNonTerminalR2LScores[i] = scores[i];
|
||||||
m_rightBoundaryNonTerminalR2LPossibleFutureOrientations[i] = possibleFutureOrientations[i];
|
m_rightBoundaryNonTerminalR2LPossibleFutureOrientations[i] = possibleFutureOrientations[i];
|
||||||
}
|
}
|
||||||
@ -70,77 +66,61 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
float GetLeftBoundaryL2RScoreMono() const
|
float GetLeftBoundaryL2RScoreMono() const {
|
||||||
{
|
|
||||||
return m_leftBoundaryNonTerminalL2RScores[0];
|
return m_leftBoundaryNonTerminalL2RScores[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
float GetLeftBoundaryL2RScoreSwap() const
|
float GetLeftBoundaryL2RScoreSwap() const {
|
||||||
{
|
|
||||||
return m_leftBoundaryNonTerminalL2RScores[1];
|
return m_leftBoundaryNonTerminalL2RScores[1];
|
||||||
}
|
}
|
||||||
|
|
||||||
float GetLeftBoundaryL2RScoreDiscontinuous() const
|
float GetLeftBoundaryL2RScoreDiscontinuous() const {
|
||||||
{
|
|
||||||
return m_leftBoundaryNonTerminalL2RScores[2];
|
return m_leftBoundaryNonTerminalL2RScores[2];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
float GetRightBoundaryR2LScoreMono() const
|
float GetRightBoundaryR2LScoreMono() const {
|
||||||
{
|
|
||||||
return m_rightBoundaryNonTerminalR2LScores[0];
|
return m_rightBoundaryNonTerminalR2LScores[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
float GetRightBoundaryR2LScoreSwap() const
|
float GetRightBoundaryR2LScoreSwap() const {
|
||||||
{
|
|
||||||
return m_rightBoundaryNonTerminalR2LScores[1];
|
return m_rightBoundaryNonTerminalR2LScores[1];
|
||||||
}
|
}
|
||||||
|
|
||||||
float GetRightBoundaryR2LScoreDiscontinuous() const
|
float GetRightBoundaryR2LScoreDiscontinuous() const {
|
||||||
{
|
|
||||||
return m_rightBoundaryNonTerminalR2LScores[2];
|
return m_rightBoundaryNonTerminalR2LScores[2];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
int Compare(const FFState& other) const
|
int Compare(const FFState& other) const {
|
||||||
{
|
|
||||||
const PhraseOrientationFeatureState &otherState = static_cast<const PhraseOrientationFeatureState&>(other);
|
const PhraseOrientationFeatureState &otherState = static_cast<const PhraseOrientationFeatureState&>(other);
|
||||||
|
|
||||||
if (!m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet &&
|
if (!m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet &&
|
||||||
!m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet)
|
!m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) {
|
||||||
{
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
if (m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet)
|
if (m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if (!m_leftBoundaryIsSet && otherState.m_leftBoundaryIsSet)
|
if (!m_leftBoundaryIsSet && otherState.m_leftBoundaryIsSet) {
|
||||||
{
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
if (m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet)
|
if (m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if (!m_rightBoundaryIsSet && otherState.m_rightBoundaryIsSet)
|
if (!m_rightBoundaryIsSet && otherState.m_rightBoundaryIsSet) {
|
||||||
{
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (m_leftBoundaryIsSet)
|
if (m_leftBoundaryIsSet) {
|
||||||
{
|
|
||||||
int compareLeft = CompareLeftBoundaryRecursive(*this, otherState);
|
int compareLeft = CompareLeftBoundaryRecursive(*this, otherState);
|
||||||
if (compareLeft != 0)
|
if (compareLeft != 0) {
|
||||||
{
|
|
||||||
return compareLeft;
|
return compareLeft;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (m_rightBoundaryIsSet)
|
if (m_rightBoundaryIsSet) {
|
||||||
{
|
|
||||||
int compareRight = CompareRightBoundaryRecursive(*this, otherState);
|
int compareRight = CompareRightBoundaryRecursive(*this, otherState);
|
||||||
if (compareRight != 0)
|
if (compareRight != 0) {
|
||||||
{
|
|
||||||
return compareRight;
|
return compareRight;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -150,59 +130,45 @@ public:
|
|||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
static int CompareLeftBoundaryRecursive(const PhraseOrientationFeatureState& state, const PhraseOrientationFeatureState& otherState)
|
static int CompareLeftBoundaryRecursive(const PhraseOrientationFeatureState& state, const PhraseOrientationFeatureState& otherState) {
|
||||||
{
|
if (!state.m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet) {
|
||||||
if (!state.m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet)
|
|
||||||
{
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
if (state.m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet)
|
if (state.m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if (!state.m_leftBoundaryIsSet && otherState.m_leftBoundaryIsSet)
|
if (!state.m_leftBoundaryIsSet && otherState.m_leftBoundaryIsSet) {
|
||||||
{
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( otherState.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex < state.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex )
|
if ( otherState.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex < state.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex ) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if ( state.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex < otherState.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex )
|
if ( state.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex < otherState.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex ) {
|
||||||
{
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
if ( Smaller(otherState.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations, state.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations) )
|
if ( Smaller(otherState.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations, state.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations) ) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if ( Smaller(state.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations, otherState.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations) )
|
if ( Smaller(state.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations, otherState.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations) ) {
|
||||||
{
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
for (size_t i=0; i<state.m_leftBoundaryNonTerminalL2RScores.size(); ++i)
|
for (size_t i=0; i<state.m_leftBoundaryNonTerminalL2RScores.size(); ++i) {
|
||||||
{
|
if (state.m_leftBoundaryNonTerminalL2RScores[i] > otherState.m_leftBoundaryNonTerminalL2RScores[i]) {
|
||||||
if (state.m_leftBoundaryNonTerminalL2RScores[i] > otherState.m_leftBoundaryNonTerminalL2RScores[i])
|
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if (state.m_leftBoundaryNonTerminalL2RScores[i] < otherState.m_leftBoundaryNonTerminalL2RScores[i])
|
if (state.m_leftBoundaryNonTerminalL2RScores[i] < otherState.m_leftBoundaryNonTerminalL2RScores[i]) {
|
||||||
{
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (state.m_leftBoundaryRecursionGuard && otherState.m_leftBoundaryRecursionGuard)
|
if (state.m_leftBoundaryRecursionGuard && otherState.m_leftBoundaryRecursionGuard) {
|
||||||
{
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
if (state.m_leftBoundaryRecursionGuard && !otherState.m_leftBoundaryRecursionGuard)
|
if (state.m_leftBoundaryRecursionGuard && !otherState.m_leftBoundaryRecursionGuard) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if (!state.m_leftBoundaryRecursionGuard && otherState.m_leftBoundaryRecursionGuard)
|
if (!state.m_leftBoundaryRecursionGuard && otherState.m_leftBoundaryRecursionGuard) {
|
||||||
{
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -212,59 +178,45 @@ private:
|
|||||||
return CompareLeftBoundaryRecursive(*prevState, *otherPrevState);
|
return CompareLeftBoundaryRecursive(*prevState, *otherPrevState);
|
||||||
};
|
};
|
||||||
|
|
||||||
static int CompareRightBoundaryRecursive(const PhraseOrientationFeatureState& state, const PhraseOrientationFeatureState& otherState)
|
static int CompareRightBoundaryRecursive(const PhraseOrientationFeatureState& state, const PhraseOrientationFeatureState& otherState) {
|
||||||
{
|
if (!state.m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) {
|
||||||
if (!state.m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet)
|
|
||||||
{
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
if (state.m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet)
|
if (state.m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if (!state.m_rightBoundaryIsSet && otherState.m_rightBoundaryIsSet)
|
if (!state.m_rightBoundaryIsSet && otherState.m_rightBoundaryIsSet) {
|
||||||
{
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( otherState.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex < state.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex )
|
if ( otherState.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex < state.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex ) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if ( state.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex < otherState.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex )
|
if ( state.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex < otherState.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex ) {
|
||||||
{
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
if ( Smaller(otherState.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations, state.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations) )
|
if ( Smaller(otherState.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations, state.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations) ) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if ( Smaller(state.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations, otherState.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations) )
|
if ( Smaller(state.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations, otherState.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations) ) {
|
||||||
{
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
for (size_t i=0; i<state.m_rightBoundaryNonTerminalR2LScores.size(); ++i)
|
for (size_t i=0; i<state.m_rightBoundaryNonTerminalR2LScores.size(); ++i) {
|
||||||
{
|
if (state.m_rightBoundaryNonTerminalR2LScores[i] > otherState.m_rightBoundaryNonTerminalR2LScores[i]) {
|
||||||
if (state.m_rightBoundaryNonTerminalR2LScores[i] > otherState.m_rightBoundaryNonTerminalR2LScores[i])
|
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if (state.m_rightBoundaryNonTerminalR2LScores[i] < otherState.m_rightBoundaryNonTerminalR2LScores[i])
|
if (state.m_rightBoundaryNonTerminalR2LScores[i] < otherState.m_rightBoundaryNonTerminalR2LScores[i]) {
|
||||||
{
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (state.m_rightBoundaryRecursionGuard && otherState.m_rightBoundaryRecursionGuard)
|
if (state.m_rightBoundaryRecursionGuard && otherState.m_rightBoundaryRecursionGuard) {
|
||||||
{
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
if (state.m_rightBoundaryRecursionGuard && !otherState.m_rightBoundaryRecursionGuard)
|
if (state.m_rightBoundaryRecursionGuard && !otherState.m_rightBoundaryRecursionGuard) {
|
||||||
{
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if (!state.m_rightBoundaryRecursionGuard && otherState.m_rightBoundaryRecursionGuard)
|
if (!state.m_rightBoundaryRecursionGuard && otherState.m_rightBoundaryRecursionGuard) {
|
||||||
{
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -274,10 +226,8 @@ private:
|
|||||||
return CompareRightBoundaryRecursive(*prevState, *otherPrevState);
|
return CompareRightBoundaryRecursive(*prevState, *otherPrevState);
|
||||||
};
|
};
|
||||||
|
|
||||||
template<std::size_t N> static bool Smaller(const std::bitset<N>& x, const std::bitset<N>& y)
|
template<std::size_t N> static bool Smaller(const std::bitset<N>& x, const std::bitset<N>& y) {
|
||||||
{
|
for (size_t i=0; i<N; ++i) {
|
||||||
for (size_t i=0; i<N; ++i)
|
|
||||||
{
|
|
||||||
if (x[i] ^ y[i])
|
if (x[i] ^ y[i])
|
||||||
return y[i];
|
return y[i];
|
||||||
}
|
}
|
||||||
@ -325,8 +275,7 @@ public:
|
|||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{
|
|
||||||
targetPhrase.SetRuleSource(source);
|
targetPhrase.SetRuleSource(source);
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -335,17 +284,16 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{};
|
};
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
FFState* EvaluateWhenApplied(
|
FFState* EvaluateWhenApplied(
|
||||||
const Hypothesis& cur_hypo,
|
const Hypothesis& cur_hypo,
|
||||||
const FFState* prev_state,
|
const FFState* prev_state,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{
|
|
||||||
return new PhraseOrientationFeatureState();
|
return new PhraseOrientationFeatureState();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -43,12 +43,12 @@ public:
|
|||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWithSourceContext(const InputType &input
|
void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
@ -57,12 +57,12 @@ public:
|
|||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const ChartHypothesis& hypo,
|
void EvaluateWhenApplied(const ChartHypothesis& hypo,
|
||||||
ScoreComponentCollection*) const
|
ScoreComponentCollection*) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
|
|
||||||
};
|
};
|
||||||
|
@ -32,8 +32,7 @@ void PhrasePenalty::EvaluateInIsolation(const Phrase &source
|
|||||||
scoreBreakdown.Assign(this, scores);
|
scoreBreakdown.Assign(this, scores);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
scoreBreakdown.Assign(this, 1.0f);
|
scoreBreakdown.Assign(this, 1.0f);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -42,8 +41,7 @@ void PhrasePenalty::SetParameter(const std::string& key, const std::string& valu
|
|||||||
{
|
{
|
||||||
if (key == "per-phrase-table") {
|
if (key == "per-phrase-table") {
|
||||||
m_perPhraseTable =Scan<bool>(value);
|
m_perPhraseTable =Scan<bool>(value);
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
StatelessFeatureFunction::SetParameter(key, value);
|
StatelessFeatureFunction::SetParameter(key, value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -20,26 +20,26 @@ public:
|
|||||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWhenApplied(const Syntax::SHyperedge &hyperedge,
|
void EvaluateWhenApplied(const Syntax::SHyperedge &hyperedge,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWithSourceContext(const InputType &input
|
void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
@ -12,37 +12,39 @@ class ReferenceComparison : public StatelessFeatureFunction
|
|||||||
public:
|
public:
|
||||||
ReferenceComparison(const std::string &line);
|
ReferenceComparison(const std::string &line);
|
||||||
|
|
||||||
virtual bool IsUseable(const FactorMask &mask) const
|
virtual bool IsUseable(const FactorMask &mask) const {
|
||||||
{ return true; }
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
virtual void EvaluateInIsolation(const Phrase &source
|
virtual void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
virtual void EvaluateWithSourceContext(const InputType &input
|
virtual void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
|
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
std::vector<float> DefaultWeights() const
|
std::vector<float> DefaultWeights() const {
|
||||||
{ return std::vector<float>(); }
|
return std::vector<float>();
|
||||||
|
}
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
|
||||||
|
@ -31,8 +31,7 @@ void RuleScope::EvaluateInIsolation(const Phrase &source
|
|||||||
bool ambiguous = IsAmbiguous(word, m_sourceSyntax);
|
bool ambiguous = IsAmbiguous(word, m_sourceSyntax);
|
||||||
if (ambiguous) {
|
if (ambiguous) {
|
||||||
++count;
|
++count;
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
if (count > 0) {
|
if (count > 0) {
|
||||||
score += count;
|
score += count;
|
||||||
}
|
}
|
||||||
|
@ -11,8 +11,9 @@ class RuleScope : public StatelessFeatureFunction
|
|||||||
public:
|
public:
|
||||||
RuleScope(const std::string &line);
|
RuleScope(const std::string &line);
|
||||||
|
|
||||||
virtual bool IsUseable(const FactorMask &mask) const
|
virtual bool IsUseable(const FactorMask &mask) const {
|
||||||
{ return true; }
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
virtual void EvaluateInIsolation(const Phrase &source
|
virtual void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
@ -24,21 +25,21 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
|
|
||||||
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
|
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
|
@ -11,8 +11,9 @@ class SetSourcePhrase : public StatelessFeatureFunction
|
|||||||
public:
|
public:
|
||||||
SetSourcePhrase(const std::string &line);
|
SetSourcePhrase(const std::string &line);
|
||||||
|
|
||||||
virtual bool IsUseable(const FactorMask &mask) const
|
virtual bool IsUseable(const FactorMask &mask) const {
|
||||||
{ return true; }
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
virtual void EvaluateInIsolation(const Phrase &source
|
virtual void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
@ -24,23 +25,24 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
|
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
std::vector<float> DefaultWeights() const
|
std::vector<float> DefaultWeights() const {
|
||||||
{ return std::vector<float>(); }
|
return std::vector<float>();
|
||||||
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -12,8 +12,8 @@ class SkeletonState : public FFState
|
|||||||
int m_targetLen;
|
int m_targetLen;
|
||||||
public:
|
public:
|
||||||
SkeletonState(int targetLen)
|
SkeletonState(int targetLen)
|
||||||
:m_targetLen(targetLen)
|
:m_targetLen(targetLen) {
|
||||||
{}
|
}
|
||||||
|
|
||||||
int Compare(const FFState& other) const;
|
int Compare(const FFState& other) const;
|
||||||
};
|
};
|
||||||
|
@ -10,8 +10,7 @@ class SkeletonTranslationOptionListFeature : public StatelessFeatureFunction
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
SkeletonTranslationOptionListFeature(const std::string &line)
|
SkeletonTranslationOptionListFeature(const std::string &line)
|
||||||
:StatelessFeatureFunction(1, line)
|
:StatelessFeatureFunction(1, line) {
|
||||||
{
|
|
||||||
ReadParameters();
|
ReadParameters();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -22,20 +21,19 @@ public:
|
|||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWithSourceContext(const InputType &input
|
void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{
|
|
||||||
vector<float> newScores(m_numScoreComponents);
|
vector<float> newScores(m_numScoreComponents);
|
||||||
newScores[0] = translationOptionList.size();
|
newScores[0] = translationOptionList.size();
|
||||||
|
|
||||||
@ -52,16 +50,16 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value)
|
void SetParameter(const std::string& key, const std::string& value) {
|
||||||
{}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -87,7 +87,8 @@ void SoftMatchingFeature::EvaluateWhenApplied(const ChartHypothesis& hypo,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// when loading, or when we notice that non-terminals have been added after loading, we resize vectors
|
// when loading, or when we notice that non-terminals have been added after loading, we resize vectors
|
||||||
void SoftMatchingFeature::ResizeCache() const {
|
void SoftMatchingFeature::ResizeCache() const
|
||||||
|
{
|
||||||
FactorCollection& fc = FactorCollection::Instance();
|
FactorCollection& fc = FactorCollection::Instance();
|
||||||
size_t numNonTerminals = fc.GetNumNonTerminals();
|
size_t numNonTerminals = fc.GetNumNonTerminals();
|
||||||
|
|
||||||
@ -98,7 +99,8 @@ void SoftMatchingFeature::ResizeCache() const {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
const std::string& SoftMatchingFeature::GetOrSetFeatureName(const Word& RHS, const Word& LHS) const {
|
const std::string& SoftMatchingFeature::GetOrSetFeatureName(const Word& RHS, const Word& LHS) const
|
||||||
|
{
|
||||||
try {
|
try {
|
||||||
#ifdef WITH_THREADS //try read-only lock
|
#ifdef WITH_THREADS //try read-only lock
|
||||||
boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
|
boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
|
||||||
@ -107,8 +109,7 @@ const std::string& SoftMatchingFeature::GetOrSetFeatureName(const Word& RHS, con
|
|||||||
if (!name.empty()) {
|
if (!name.empty()) {
|
||||||
return name;
|
return name;
|
||||||
}
|
}
|
||||||
}
|
} catch (const std::out_of_range& oor) {
|
||||||
catch (const std::out_of_range& oor) {
|
|
||||||
#ifdef WITH_THREADS //need to resize cache; write lock
|
#ifdef WITH_THREADS //need to resize cache; write lock
|
||||||
boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
|
boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
|
||||||
#endif
|
#endif
|
||||||
|
@ -34,8 +34,8 @@ public:
|
|||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const {};
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {};
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const {};
|
ScoreComponentCollection* accumulator) const {};
|
||||||
|
@ -238,20 +238,17 @@ void SoftSourceSyntacticConstraintsFeature::EvaluateWithSourceContext(const Inpu
|
|||||||
{
|
{
|
||||||
assert(stackVec);
|
assert(stackVec);
|
||||||
|
|
||||||
IFFEATUREVERBOSE(2)
|
IFFEATUREVERBOSE(2) {
|
||||||
{
|
|
||||||
FEATUREVERBOSE(2, targetPhrase << std::endl);
|
FEATUREVERBOSE(2, targetPhrase << std::endl);
|
||||||
FEATUREVERBOSE(2, inputPath << std::endl);
|
FEATUREVERBOSE(2, inputPath << std::endl);
|
||||||
for (size_t i = 0; i < stackVec->size(); ++i)
|
for (size_t i = 0; i < stackVec->size(); ++i) {
|
||||||
{
|
|
||||||
const ChartCellLabel &cell = *stackVec->at(i);
|
const ChartCellLabel &cell = *stackVec->at(i);
|
||||||
const WordsRange &ntRange = cell.GetCoverage();
|
const WordsRange &ntRange = cell.GetCoverage();
|
||||||
FEATUREVERBOSE(2, "stackVec[ " << i << " ] : " << ntRange.GetStartPos() << " - " << ntRange.GetEndPos() << std::endl);
|
FEATUREVERBOSE(2, "stackVec[ " << i << " ] : " << ntRange.GetStartPos() << " - " << ntRange.GetEndPos() << std::endl);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (AlignmentInfo::const_iterator it=targetPhrase.GetAlignNonTerm().begin();
|
for (AlignmentInfo::const_iterator it=targetPhrase.GetAlignNonTerm().begin();
|
||||||
it!=targetPhrase.GetAlignNonTerm().end(); ++it)
|
it!=targetPhrase.GetAlignNonTerm().end(); ++it) {
|
||||||
{
|
|
||||||
FEATUREVERBOSE(2, "alignNonTerm " << it->first << " " << it->second << std::endl);
|
FEATUREVERBOSE(2, "alignNonTerm " << it->first << " " << it->second << std::endl);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -311,14 +308,11 @@ void SoftSourceSyntacticConstraintsFeature::EvaluateWithSourceContext(const Inpu
|
|||||||
const NonTerminalSet& prevTreeInputLabels = treeInput.GetLabelSet(prevStartPos,prevEndPos);
|
const NonTerminalSet& prevTreeInputLabels = treeInput.GetLabelSet(prevStartPos,prevEndPos);
|
||||||
|
|
||||||
for (NonTerminalSet::const_iterator prevTreeInputLabelsIt = prevTreeInputLabels.begin();
|
for (NonTerminalSet::const_iterator prevTreeInputLabelsIt = prevTreeInputLabels.begin();
|
||||||
prevTreeInputLabelsIt != prevTreeInputLabels.end(); ++prevTreeInputLabelsIt)
|
prevTreeInputLabelsIt != prevTreeInputLabels.end(); ++prevTreeInputLabelsIt) {
|
||||||
{
|
if (*prevTreeInputLabelsIt != outputDefaultNonTerminal) {
|
||||||
if (*prevTreeInputLabelsIt != outputDefaultNonTerminal)
|
|
||||||
{
|
|
||||||
boost::unordered_map<const Factor*,size_t>::const_iterator foundPrevTreeInputLabel
|
boost::unordered_map<const Factor*,size_t>::const_iterator foundPrevTreeInputLabel
|
||||||
= m_sourceLabelIndexesByFactor.find((*prevTreeInputLabelsIt)[0]);
|
= m_sourceLabelIndexesByFactor.find((*prevTreeInputLabelsIt)[0]);
|
||||||
if (foundPrevTreeInputLabel != m_sourceLabelIndexesByFactor.end())
|
if (foundPrevTreeInputLabel != m_sourceLabelIndexesByFactor.end()) {
|
||||||
{
|
|
||||||
size_t prevTreeInputLabelIndex = foundPrevTreeInputLabel->second;
|
size_t prevTreeInputLabelIndex = foundPrevTreeInputLabel->second;
|
||||||
treeInputLabelsRHS[nonTerminalNumber].insert(prevTreeInputLabelIndex);
|
treeInputLabelsRHS[nonTerminalNumber].insert(prevTreeInputLabelIndex);
|
||||||
}
|
}
|
||||||
|
@ -32,8 +32,7 @@ public:
|
|||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{
|
|
||||||
targetPhrase.SetRuleSource(source);
|
targetPhrase.SetRuleSource(source);
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -45,19 +44,19 @@ public:
|
|||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
|
|
||||||
void EvaluateWhenApplied(
|
void EvaluateWhenApplied(
|
||||||
const Hypothesis& cur_hypo,
|
const Hypothesis& cur_hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{};
|
};
|
||||||
|
|
||||||
void EvaluateWhenApplied(
|
void EvaluateWhenApplied(
|
||||||
const ChartHypothesis& cur_hypo,
|
const ChartHypothesis& cur_hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{};
|
};
|
||||||
|
|
||||||
private:
|
private:
|
||||||
std::string m_sourceLabelSetFile;
|
std::string m_sourceLabelSetFile;
|
||||||
|
@ -30,8 +30,8 @@ public:
|
|||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
|
@ -37,20 +37,20 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void ComputeFeatures(const Phrase &source,
|
void ComputeFeatures(const Phrase &source,
|
||||||
const TargetPhrase& targetPhrase,
|
const TargetPhrase& targetPhrase,
|
||||||
|
@ -73,18 +73,14 @@ void SpanLength::SetParameter(const std::string& key, const std::string& value)
|
|||||||
if (key == "smoothing") {
|
if (key == "smoothing") {
|
||||||
if (value == "plus-constant") {
|
if (value == "plus-constant") {
|
||||||
m_smoothingMethod = PlusConst;
|
m_smoothingMethod = PlusConst;
|
||||||
}
|
} else if (value == "none") {
|
||||||
else if (value == "none") {
|
|
||||||
m_smoothingMethod = None;
|
m_smoothingMethod = None;
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
UTIL_THROW(util::Exception, "Unknown smoothing type " << value);
|
UTIL_THROW(util::Exception, "Unknown smoothing type " << value);
|
||||||
}
|
}
|
||||||
}
|
} else if (key == "constant") {
|
||||||
else if (key == "constant") {
|
|
||||||
m_const = Scan<float>(value);
|
m_const = Scan<float>(value);
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
StatelessFeatureFunction::SetParameter(key, value);
|
StatelessFeatureFunction::SetParameter(key, value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -11,8 +11,9 @@ class SpanLength : public StatelessFeatureFunction
|
|||||||
public:
|
public:
|
||||||
SpanLength(const std::string &line);
|
SpanLength(const std::string &line);
|
||||||
|
|
||||||
virtual bool IsUseable(const FactorMask &mask) const
|
virtual bool IsUseable(const FactorMask &mask) const {
|
||||||
{ return true; }
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
virtual void EvaluateInIsolation(const Phrase &source
|
virtual void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
@ -27,23 +28,22 @@ public:
|
|||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
|
|
||||||
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
|
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
enum SmoothingMethod
|
enum SmoothingMethod {
|
||||||
{
|
|
||||||
None,
|
None,
|
||||||
PlusConst,
|
PlusConst,
|
||||||
};
|
};
|
||||||
|
@ -39,7 +39,8 @@ SparseHieroReorderingFeature::SparseHieroReorderingFeature(const std::string &li
|
|||||||
LoadVocabulary(m_targetVocabFile, m_targetVocab);
|
LoadVocabulary(m_targetVocabFile, m_targetVocab);
|
||||||
}
|
}
|
||||||
|
|
||||||
void SparseHieroReorderingFeature::SetParameter(const std::string& key, const std::string& value) {
|
void SparseHieroReorderingFeature::SetParameter(const std::string& key, const std::string& value)
|
||||||
|
{
|
||||||
if (key == "input-factor") {
|
if (key == "input-factor") {
|
||||||
m_sourceFactor = Scan<FactorType>(value);
|
m_sourceFactor = Scan<FactorType>(value);
|
||||||
} else if (key == "output-factor") {
|
} else if (key == "output-factor") {
|
||||||
@ -75,7 +76,8 @@ void SparseHieroReorderingFeature::LoadVocabulary(const std::string& filename, V
|
|||||||
in.close();
|
in.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
const Factor* SparseHieroReorderingFeature::GetFactor(const Word& word, const Vocab& vocab, FactorType factorType) const {
|
const Factor* SparseHieroReorderingFeature::GetFactor(const Word& word, const Vocab& vocab, FactorType factorType) const
|
||||||
|
{
|
||||||
const Factor* factor = word.GetFactor(factorType);
|
const Factor* factor = word.GetFactor(factorType);
|
||||||
if (vocab.size() && vocab.find(factor) == vocab.end()) return m_otherFactor;
|
if (vocab.size() && vocab.find(factor) == vocab.end()) return m_otherFactor;
|
||||||
return factor;
|
return factor;
|
||||||
@ -197,8 +199,7 @@ void SparseHieroReorderingFeature::EvaluateWhenApplied(
|
|||||||
if ((sourceLeftBoundaryPos < sourceRightBoundaryPos &&
|
if ((sourceLeftBoundaryPos < sourceRightBoundaryPos &&
|
||||||
targetLeftRulePos > targetRightRulePos) ||
|
targetLeftRulePos > targetRightRulePos) ||
|
||||||
((sourceLeftBoundaryPos > sourceRightBoundaryPos &&
|
((sourceLeftBoundaryPos > sourceRightBoundaryPos &&
|
||||||
targetLeftRulePos < targetRightRulePos)))
|
targetLeftRulePos < targetRightRulePos))) {
|
||||||
{
|
|
||||||
isMonotone = false;
|
isMonotone = false;
|
||||||
}
|
}
|
||||||
stringstream buf;
|
stringstream buf;
|
||||||
|
@ -26,31 +26,32 @@ public:
|
|||||||
|
|
||||||
SparseHieroReorderingFeature(const std::string &line);
|
SparseHieroReorderingFeature(const std::string &line);
|
||||||
|
|
||||||
bool IsUseable(const FactorMask &mask) const
|
bool IsUseable(const FactorMask &mask) const {
|
||||||
{ return true; }
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
virtual void EvaluateWithSourceContext(const InputType &input
|
virtual void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
|
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const;
|
ScoreComponentCollection* accumulator) const;
|
||||||
|
|
||||||
|
@ -44,7 +44,10 @@ public:
|
|||||||
virtual FFState* EvaluateWhenApplied(
|
virtual FFState* EvaluateWhenApplied(
|
||||||
const Syntax::SHyperedge& /* cur_hypo */,
|
const Syntax::SHyperedge& /* cur_hypo */,
|
||||||
int /* featureID - used to index the state in the previous hypotheses */,
|
int /* featureID - used to index the state in the previous hypotheses */,
|
||||||
ScoreComponentCollection* accumulator) const { assert(false); return 0; /* FIXME */ }
|
ScoreComponentCollection* accumulator) const {
|
||||||
|
assert(false);
|
||||||
|
return 0; /* FIXME */
|
||||||
|
}
|
||||||
|
|
||||||
//! return the state associated with the empty hypothesis for a given sentence
|
//! return the state associated with the empty hypothesis for a given sentence
|
||||||
virtual const FFState* EmptyHypothesisState(const InputType &input) const = 0;
|
virtual const FFState* EmptyHypothesisState(const InputType &input) const = 0;
|
||||||
|
@ -36,7 +36,9 @@ public:
|
|||||||
ScoreComponentCollection* accumulator) const = 0;
|
ScoreComponentCollection* accumulator) const = 0;
|
||||||
|
|
||||||
virtual void EvaluateWhenApplied(const Syntax::SHyperedge &,
|
virtual void EvaluateWhenApplied(const Syntax::SHyperedge &,
|
||||||
ScoreComponentCollection*) const { assert(false); }
|
ScoreComponentCollection*) const {
|
||||||
|
assert(false);
|
||||||
|
}
|
||||||
|
|
||||||
virtual bool IsStateless() const {
|
virtual bool IsStateless() const {
|
||||||
return true;
|
return true;
|
||||||
|
@ -27,16 +27,16 @@ public:
|
|||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -52,17 +52,17 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
|
@ -197,18 +197,18 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
|
@ -37,18 +37,18 @@ public:
|
|||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void ComputeFeatures(const Phrase &source,
|
void ComputeFeatures(const Phrase &source,
|
||||||
const TargetPhrase& targetPhrase,
|
const TargetPhrase& targetPhrase,
|
||||||
|
@ -8,7 +8,8 @@
|
|||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
|
|
||||||
void TreeStructureFeature::Load() {
|
void TreeStructureFeature::Load()
|
||||||
|
{
|
||||||
|
|
||||||
// syntactic constraints can be hooked in here.
|
// syntactic constraints can be hooked in here.
|
||||||
m_constraints = NULL;
|
m_constraints = NULL;
|
||||||
@ -20,7 +21,8 @@ void TreeStructureFeature::Load() {
|
|||||||
|
|
||||||
|
|
||||||
// define NT labels (ints) that are mapped from strings for quicker comparison.
|
// define NT labels (ints) that are mapped from strings for quicker comparison.
|
||||||
void TreeStructureFeature::AddNTLabels(TreePointer root) const {
|
void TreeStructureFeature::AddNTLabels(TreePointer root) const
|
||||||
|
{
|
||||||
std::string label = root->GetLabel();
|
std::string label = root->GetLabel();
|
||||||
|
|
||||||
if (root->IsTerminal()) {
|
if (root->IsTerminal()) {
|
||||||
@ -69,8 +71,7 @@ FFState* TreeStructureFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hy
|
|||||||
mytree->Combine(previous_trees);
|
mytree->Combine(previous_trees);
|
||||||
|
|
||||||
return new TreeState(mytree);
|
return new TreeState(mytree);
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
UTIL_THROW2("Error: TreeStructureFeature active, but no internal tree structure found");
|
UTIL_THROW2("Error: TreeStructureFeature active, but no internal tree structure found");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -14,8 +14,7 @@ typedef int NTLabel;
|
|||||||
|
|
||||||
// mapping from string nonterminal label to int representation.
|
// mapping from string nonterminal label to int representation.
|
||||||
// allows abstraction if multiple nonterminal strings should map to same label.
|
// allows abstraction if multiple nonterminal strings should map to same label.
|
||||||
struct LabelSet
|
struct LabelSet {
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
std::map<std::string, NTLabel> string_to_label;
|
std::map<std::string, NTLabel> string_to_label;
|
||||||
};
|
};
|
||||||
@ -40,7 +39,9 @@ public:
|
|||||||
:StatefulFeatureFunction(0, line) {
|
:StatefulFeatureFunction(0, line) {
|
||||||
ReadParameters();
|
ReadParameters();
|
||||||
}
|
}
|
||||||
~TreeStructureFeature() {delete m_constraints;};
|
~TreeStructureFeature() {
|
||||||
|
delete m_constraints;
|
||||||
|
};
|
||||||
|
|
||||||
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
|
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
|
||||||
return new TreeState(TreePointer());
|
return new TreeState(TreePointer());
|
||||||
@ -64,14 +65,16 @@ public:
|
|||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const {};
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {};
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
|
|
||||||
FFState* EvaluateWhenApplied(
|
FFState* EvaluateWhenApplied(
|
||||||
const Hypothesis& cur_hypo,
|
const Hypothesis& cur_hypo,
|
||||||
const FFState* prev_state,
|
const FFState* prev_state,
|
||||||
ScoreComponentCollection* accumulator) const {UTIL_THROW(util::Exception, "Not implemented");};
|
ScoreComponentCollection* accumulator) const {
|
||||||
|
UTIL_THROW(util::Exception, "Not implemented");
|
||||||
|
};
|
||||||
FFState* EvaluateWhenApplied(
|
FFState* EvaluateWhenApplied(
|
||||||
const ChartHypothesis& /* cur_hypo */,
|
const ChartHypothesis& /* cur_hypo */,
|
||||||
int /* featureID - used to index the state in the previous hypotheses */,
|
int /* featureID - used to index the state in the previous hypotheses */,
|
||||||
|
@ -32,31 +32,31 @@ public:
|
|||||||
std::vector<float> DefaultWeights() const;
|
std::vector<float> DefaultWeights() const;
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWhenApplied(const Syntax::SHyperedge &hyperedge,
|
void EvaluateWhenApplied(const Syntax::SHyperedge &hyperedge,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
void EvaluateWithSourceContext(const InputType &input
|
void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -49,8 +49,7 @@ class ThreadLocalByFeatureStorage
|
|||||||
std::make_pair(m_ff->GetScoreProducerDescription(), m_factory()));
|
std::make_pair(m_ff->GetScoreProducerDescription(), m_factory()));
|
||||||
|
|
||||||
return ret.first->second;
|
return ret.first->second;
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
return it->second;
|
return it->second;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -29,14 +29,12 @@ const std::string VW_DUMMY_LABEL = "1111"; // VW does not use the actual label,
|
|||||||
struct VWTargetSentence {
|
struct VWTargetSentence {
|
||||||
VWTargetSentence() : m_sentence(NULL), m_alignment(NULL) {}
|
VWTargetSentence() : m_sentence(NULL), m_alignment(NULL) {}
|
||||||
|
|
||||||
void Clear()
|
void Clear() {
|
||||||
{
|
|
||||||
if (m_sentence) delete m_sentence;
|
if (m_sentence) delete m_sentence;
|
||||||
if (m_alignment) delete m_alignment;
|
if (m_alignment) delete m_alignment;
|
||||||
}
|
}
|
||||||
|
|
||||||
~VWTargetSentence()
|
~VWTargetSentence() {
|
||||||
{
|
|
||||||
Clear();
|
Clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -53,8 +51,7 @@ public:
|
|||||||
VW(const std::string &line)
|
VW(const std::string &line)
|
||||||
: StatelessFeatureFunction(1, line)
|
: StatelessFeatureFunction(1, line)
|
||||||
, TLSTargetSentence(this)
|
, TLSTargetSentence(this)
|
||||||
, m_train(false)
|
, m_train(false) {
|
||||||
{
|
|
||||||
ReadParameters();
|
ReadParameters();
|
||||||
Discriminative::ClassifierFactory *classifierFactory = m_train
|
Discriminative::ClassifierFactory *classifierFactory = m_train
|
||||||
? new Discriminative::ClassifierFactory(m_modelPath)
|
? new Discriminative::ClassifierFactory(m_modelPath)
|
||||||
@ -68,8 +65,7 @@ public:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
virtual ~VW()
|
virtual ~VW() {
|
||||||
{
|
|
||||||
delete m_tlsClassifier;
|
delete m_tlsClassifier;
|
||||||
delete m_normalizer;
|
delete m_normalizer;
|
||||||
}
|
}
|
||||||
@ -81,20 +77,19 @@ public:
|
|||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedFutureScore) const
|
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWithSourceContext(const InputType &input
|
void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, const StackVec *stackVec
|
, const StackVec *stackVec
|
||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection *estimatedFutureScore = NULL) const
|
, ScoreComponentCollection *estimatedFutureScore = NULL) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
||||||
, const TranslationOptionList &translationOptionList) const
|
, const TranslationOptionList &translationOptionList) const {
|
||||||
{
|
|
||||||
Discriminative::Classifier &classifier = *m_tlsClassifier->GetStored();
|
Discriminative::Classifier &classifier = *m_tlsClassifier->GetStored();
|
||||||
|
|
||||||
if (translationOptionList.size() == 0)
|
if (translationOptionList.size() == 0)
|
||||||
@ -148,16 +143,15 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const Hypothesis& hypo,
|
void EvaluateWhenApplied(const Hypothesis& hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const
|
ScoreComponentCollection* accumulator) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value)
|
void SetParameter(const std::string& key, const std::string& value) {
|
||||||
{
|
|
||||||
if (key == "train") {
|
if (key == "train") {
|
||||||
m_train = Scan<bool>(value);
|
m_train = Scan<bool>(value);
|
||||||
} else if (key == "path") {
|
} else if (key == "path") {
|
||||||
@ -204,8 +198,7 @@ public:
|
|||||||
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
std::string MakeTargetLabel(const TargetPhrase &targetPhrase) const
|
std::string MakeTargetLabel(const TargetPhrase &targetPhrase) const {
|
||||||
{
|
|
||||||
return VW_DUMMY_LABEL;
|
return VW_DUMMY_LABEL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -15,8 +15,7 @@ class VWFeatureBase : public StatelessFeatureFunction
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
VWFeatureBase(const std::string &line, bool isSource = true)
|
VWFeatureBase(const std::string &line, bool isSource = true)
|
||||||
: StatelessFeatureFunction(0, line), m_usedBy(1, "VW0"), m_isSource(isSource)
|
: StatelessFeatureFunction(0, line), m_usedBy(1, "VW0"), m_isSource(isSource) {
|
||||||
{
|
|
||||||
// defaults
|
// defaults
|
||||||
m_sourceFactors.push_back(0);
|
m_sourceFactors.push_back(0);
|
||||||
m_targetFactors.push_back(0);
|
m_targetFactors.push_back(0);
|
||||||
@ -47,8 +46,7 @@ class VWFeatureBase : public StatelessFeatureFunction
|
|||||||
|
|
||||||
|
|
||||||
// Common parameters for classifier features, both source and target features
|
// Common parameters for classifier features, both source and target features
|
||||||
virtual void SetParameter(const std::string& key, const std::string& value)
|
virtual void SetParameter(const std::string& key, const std::string& value) {
|
||||||
{
|
|
||||||
if (key == "used-by") {
|
if (key == "used-by") {
|
||||||
ParseUsedBy(value);
|
ParseUsedBy(value);
|
||||||
} else if (key == "source-factors") {
|
} else if (key == "source-factors") {
|
||||||
|
@ -19,16 +19,16 @@ class VWFeatureSource : public VWFeatureBase
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
VWFeatureSource(const std::string &line)
|
VWFeatureSource(const std::string &line)
|
||||||
: VWFeatureBase(line, true)
|
: VWFeatureBase(line, true) {
|
||||||
{}
|
}
|
||||||
|
|
||||||
// Gets its pure virtual functions from VWFeatureBase
|
// Gets its pure virtual functions from VWFeatureBase
|
||||||
|
|
||||||
virtual void operator()(const InputType &input
|
virtual void operator()(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
, Discriminative::Classifier &classifier) const
|
, Discriminative::Classifier &classifier) const {
|
||||||
{}
|
}
|
||||||
|
|
||||||
virtual void SetParameter(const std::string& key, const std::string& value) {
|
virtual void SetParameter(const std::string& key, const std::string& value) {
|
||||||
VWFeatureBase::SetParameter(key, value);
|
VWFeatureBase::SetParameter(key, value);
|
||||||
|
@ -10,8 +10,7 @@ class VWFeatureSourceBagOfWords : public VWFeatureSource
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
VWFeatureSourceBagOfWords(const std::string &line)
|
VWFeatureSourceBagOfWords(const std::string &line)
|
||||||
: VWFeatureSource(line)
|
: VWFeatureSource(line) {
|
||||||
{
|
|
||||||
ReadParameters();
|
ReadParameters();
|
||||||
|
|
||||||
// Call this last
|
// Call this last
|
||||||
@ -21,8 +20,7 @@ class VWFeatureSourceBagOfWords : public VWFeatureSource
|
|||||||
void operator()(const InputType &input
|
void operator()(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const WordsRange &sourceRange
|
, const WordsRange &sourceRange
|
||||||
, Discriminative::Classifier &classifier) const
|
, Discriminative::Classifier &classifier) const {
|
||||||
{
|
|
||||||
for (size_t i = 0; i < input.GetSize(); i++) {
|
for (size_t i = 0; i < input.GetSize(); i++) {
|
||||||
classifier.AddLabelIndependentFeature("bow^" + GetWord(input, i));
|
classifier.AddLabelIndependentFeature("bow^" + GetWord(input, i));
|
||||||
}
|
}
|
||||||
|
@ -15,8 +15,7 @@ class VWFeatureSourceExternalFeatures : public VWFeatureSource
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
VWFeatureSourceExternalFeatures(const std::string &line)
|
VWFeatureSourceExternalFeatures(const std::string &line)
|
||||||
: VWFeatureSource(line), m_tls(this), m_column(0)
|
: VWFeatureSource(line), m_tls(this), m_column(0) {
|
||||||
{
|
|
||||||
ReadParameters();
|
ReadParameters();
|
||||||
|
|
||||||
// Call this last
|
// Call this last
|
||||||
@ -26,8 +25,7 @@ class VWFeatureSourceExternalFeatures : public VWFeatureSource
|
|||||||
void operator()(const InputType &input
|
void operator()(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
, const WordsRange &sourceRange
|
, const WordsRange &sourceRange
|
||||||
, Discriminative::Classifier &classifier) const
|
, Discriminative::Classifier &classifier) const {
|
||||||
{
|
|
||||||
const Features& features = *m_tls.GetStored();
|
const Features& features = *m_tls.GetStored();
|
||||||
for (size_t i = 0; i < features.size(); i++) {
|
for (size_t i = 0; i < features.size(); i++) {
|
||||||
classifier.AddLabelIndependentFeature("srcext^" + features[i]);
|
classifier.AddLabelIndependentFeature("srcext^" + features[i]);
|
||||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user