mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
Merge branch 'master' of http://github.com/moses-smt/mosesdecoder into ranked-sampling
This commit is contained in:
commit
b8b3d2111a
@ -95,6 +95,16 @@
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/BitmapContainer.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>Bitmaps.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/Bitmaps.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>Bitmaps.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/Bitmaps.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>BookkeepingOptions.cpp</name>
|
||||
<type>1</type>
|
||||
|
25
moses/Bitmaps.cpp
Normal file
25
moses/Bitmaps.cpp
Normal file
@ -0,0 +1,25 @@
|
||||
#include "Bitmaps.h"
|
||||
#include "Util.h"
|
||||
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
Bitmaps::~Bitmaps()
|
||||
{
|
||||
RemoveAllInColl(m_coll);
|
||||
}
|
||||
|
||||
const WordsBitmap &Bitmaps::GetBitmap(const WordsBitmap &bm)
|
||||
{
|
||||
Coll::const_iterator iter = m_coll.find(&bm);
|
||||
if (iter == m_coll.end()) {
|
||||
WordsBitmap *newBM = new WordsBitmap(bm);
|
||||
m_coll.insert(newBM);
|
||||
return *newBM;
|
||||
} else {
|
||||
return **iter;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
21
moses/Bitmaps.h
Normal file
21
moses/Bitmaps.h
Normal file
@ -0,0 +1,21 @@
|
||||
#pragma once
|
||||
|
||||
#include <boost/unordered_set.hpp>
|
||||
#include "WordsBitmap.h"
|
||||
#include "Util.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
class Bitmaps
|
||||
{
|
||||
typedef boost::unordered_set<const WordsBitmap*, UnorderedComparer<WordsBitmap>, UnorderedComparer<WordsBitmap> > Coll;
|
||||
Coll m_coll;
|
||||
|
||||
public:
|
||||
virtual ~Bitmaps();
|
||||
const WordsBitmap &GetBitmap(const WordsBitmap &bm);
|
||||
|
||||
};
|
||||
|
||||
}
|
@ -120,11 +120,6 @@ public:
|
||||
void OutputSizes(std::ostream &out) const;
|
||||
size_t GetSize() const;
|
||||
|
||||
//! transitive comparison used for adding objects into set
|
||||
inline bool operator<(const ChartCell &compare) const {
|
||||
return m_coverage < compare.m_coverage;
|
||||
}
|
||||
|
||||
void WriteSearchGraph(const ChartSearchGraphWriter& writer, const std::map<unsigned,bool> &reachable) const;
|
||||
|
||||
};
|
||||
|
@ -78,15 +78,6 @@ public:
|
||||
return m_bestScore;
|
||||
}
|
||||
|
||||
bool operator<(const ChartCellLabel &other) const {
|
||||
// m_coverage and m_label uniquely identify a ChartCellLabel, so don't
|
||||
// need to compare m_stack.
|
||||
if (m_coverage == other.m_coverage) {
|
||||
return m_label < other.m_label;
|
||||
}
|
||||
return m_coverage < other.m_coverage;
|
||||
}
|
||||
|
||||
private:
|
||||
const WordsRange &m_coverage;
|
||||
const Word &m_label;
|
||||
|
@ -39,22 +39,6 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
/** functor to compare (chart) hypotheses by feature function states.
|
||||
* If 2 hypos are equal, according to this functor, then they can be recombined.
|
||||
*/
|
||||
class ChartHypothesisRecombinationUnordered
|
||||
{
|
||||
public:
|
||||
size_t operator()(const ChartHypothesis* hypo) const {
|
||||
return hypo->hash();
|
||||
}
|
||||
|
||||
bool operator()(const ChartHypothesis* hypoA, const ChartHypothesis* hypoB) const {
|
||||
return (*hypoA) == (*hypoB);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
/** Contains a set of unique hypos that have the same HS non-term.
|
||||
* ie. 1 of these for each target LHS in each cell
|
||||
*/
|
||||
@ -64,7 +48,7 @@ class ChartHypothesisCollection
|
||||
|
||||
protected:
|
||||
//typedef std::set<ChartHypothesis*, ChartHypothesisRecombinationOrderer> HCType;
|
||||
typedef boost::unordered_set< ChartHypothesis*, ChartHypothesisRecombinationUnordered, ChartHypothesisRecombinationUnordered > HCType;
|
||||
typedef boost::unordered_set< ChartHypothesis*, UnorderedComparer<ChartHypothesis>, UnorderedComparer<ChartHypothesis> > HCType;
|
||||
HCType m_hypos;
|
||||
HypoList m_hyposOrdered;
|
||||
|
||||
|
@ -619,7 +619,7 @@ void ChartManager::OutputDetailedTranslationReport(
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
|
||||
if (staticData.IsDetailedAllTranslationReportingEnabled()) {
|
||||
const Sentence &sentence = dynamic_cast<const Sentence &>(m_source);
|
||||
const Sentence &sentence = static_cast<const Sentence &>(m_source);
|
||||
size_t nBestSize = staticData.options().nbest.nbest_size;
|
||||
std::vector<boost::shared_ptr<ChartKBestExtractor::Derivation> > nBestList;
|
||||
CalcNBest(nBestSize, nBestList, staticData.options().nbest.nbest_size);
|
||||
@ -720,7 +720,7 @@ void ChartManager::OutputDetailedTreeFragmentsTranslationReport(OutputCollector
|
||||
std::ostringstream out;
|
||||
ApplicationContext applicationContext;
|
||||
|
||||
const Sentence &sentence = dynamic_cast<const Sentence &>(m_source);
|
||||
const Sentence &sentence = static_cast<const Sentence &>(m_source);
|
||||
const size_t translationId = m_source.GetTranslationId();
|
||||
|
||||
OutputTreeFragmentsTranslationOptions(out, applicationContext, hypo, sentence, translationId);
|
||||
@ -731,7 +731,7 @@ void ChartManager::OutputDetailedTreeFragmentsTranslationReport(OutputCollector
|
||||
const vector<const StatefulFeatureFunction*>& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions();
|
||||
for( size_t i=0; i<sff.size(); i++ ) {
|
||||
if (sff[i] == treeStructure) {
|
||||
const TreeState* tree = dynamic_cast<const TreeState*>(hypo->GetFFState(i));
|
||||
const TreeState* tree = static_cast<const TreeState*>(hypo->GetFFState(i));
|
||||
out << "Full Tree " << translationId << ": " << tree->GetTree()->GetString() << "\n";
|
||||
break;
|
||||
}
|
||||
|
@ -22,33 +22,6 @@ BleuScoreState::BleuScoreState(): m_words(1),
|
||||
{
|
||||
}
|
||||
|
||||
int BleuScoreState::Compare(const FFState& o) const
|
||||
{
|
||||
if (&o == this)
|
||||
return 0;
|
||||
|
||||
if (StaticData::Instance().IsSyntax())
|
||||
return 0;
|
||||
|
||||
const BleuScoreState& other = dynamic_cast<const BleuScoreState&>(o);
|
||||
int c = m_words.Compare(other.m_words);
|
||||
if (c != 0)
|
||||
return c;
|
||||
|
||||
/*for(size_t i = 0; i < m_ngram_counts.size(); i++) {
|
||||
if (m_ngram_counts[i] < other.m_ngram_counts[i])
|
||||
return -1;
|
||||
if (m_ngram_counts[i] > other.m_ngram_counts[i])
|
||||
return 1;
|
||||
if (m_ngram_matches[i] < other.m_ngram_matches[i])
|
||||
return -1;
|
||||
if (m_ngram_matches[i] > other.m_ngram_matches[i])
|
||||
return 1;
|
||||
}*/
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t BleuScoreState::hash() const
|
||||
{
|
||||
if (StaticData::Instance().IsSyntax())
|
||||
@ -67,11 +40,7 @@ bool BleuScoreState::operator==(const FFState& o) const
|
||||
return true;
|
||||
|
||||
const BleuScoreState& other = static_cast<const BleuScoreState&>(o);
|
||||
int c = m_words.Compare(other.m_words);
|
||||
if (c == 0)
|
||||
return true;
|
||||
|
||||
return false;
|
||||
return m_words == other.m_words;
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream& out, const BleuScoreState& state)
|
||||
@ -534,7 +503,7 @@ FFState* BleuScoreFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo,
|
||||
if (!m_enabled) return new BleuScoreState();
|
||||
|
||||
NGrams::const_iterator reference_ngrams_iter;
|
||||
const BleuScoreState& ps = dynamic_cast<const BleuScoreState&>(*prev_state);
|
||||
const BleuScoreState& ps = static_cast<const BleuScoreState&>(*prev_state);
|
||||
BleuScoreState* new_state = new BleuScoreState(ps);
|
||||
|
||||
float old_bleu, new_bleu;
|
||||
@ -610,13 +579,13 @@ FFState* BleuScoreFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo,
|
||||
new_state = new BleuScoreState();
|
||||
else {
|
||||
const FFState* prev_state_zero = cur_hypo.GetPrevHypo(0)->GetFFState(featureID);
|
||||
const BleuScoreState& ps_zero = dynamic_cast<const BleuScoreState&>(*prev_state_zero);
|
||||
const BleuScoreState& ps_zero = static_cast<const BleuScoreState&>(*prev_state_zero);
|
||||
new_state = new BleuScoreState(ps_zero);
|
||||
num_words_first_prev = ps_zero.m_target_length;
|
||||
|
||||
for (size_t i = 0; i < cur_hypo.GetPrevHypos().size(); ++i) {
|
||||
const FFState* prev_state = cur_hypo.GetPrevHypo(i)->GetFFState(featureID);
|
||||
const BleuScoreState* ps = dynamic_cast<const BleuScoreState*>(prev_state);
|
||||
const BleuScoreState* ps = static_cast<const BleuScoreState*>(prev_state);
|
||||
BleuScoreState* ps_nonConst = const_cast<BleuScoreState*>(ps);
|
||||
// cerr << "prev phrase: " << cur_hypo.GetPrevHypo(i)->GetOutputPhrase()
|
||||
// << " ( " << cur_hypo.GetPrevHypo(i)->GetTargetLHS() << ")" << endl;
|
||||
|
@ -25,7 +25,6 @@ public:
|
||||
static size_t bleu_order;
|
||||
|
||||
BleuScoreState();
|
||||
virtual int Compare(const FFState& other) const;
|
||||
size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
|
@ -22,13 +22,6 @@ ConstrainedDecodingState::ConstrainedDecodingState(const ChartHypothesis &hypo)
|
||||
hypo.GetOutputPhrase(m_outputPhrase);
|
||||
}
|
||||
|
||||
int ConstrainedDecodingState::Compare(const FFState& other) const
|
||||
{
|
||||
const ConstrainedDecodingState &otherFF = static_cast<const ConstrainedDecodingState&>(other);
|
||||
int ret = m_outputPhrase.Compare(otherFF.m_outputPhrase);
|
||||
return ret;
|
||||
}
|
||||
|
||||
size_t ConstrainedDecodingState::hash() const
|
||||
{
|
||||
size_t ret = hash_value(m_outputPhrase);
|
||||
|
@ -17,7 +17,6 @@ public:
|
||||
ConstrainedDecodingState(const Hypothesis &hypo);
|
||||
ConstrainedDecodingState(const ChartHypothesis &hypo);
|
||||
|
||||
int Compare(const FFState& other) const;
|
||||
virtual size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
|
@ -33,21 +33,6 @@ ControlRecombinationState::ControlRecombinationState(const ChartHypothesis &hypo
|
||||
}
|
||||
}
|
||||
|
||||
int ControlRecombinationState::Compare(const FFState& other) const
|
||||
{
|
||||
const ControlRecombinationState &otherFF = static_cast<const ControlRecombinationState&>(other);
|
||||
|
||||
if (m_ff.GetType() == SameOutput) {
|
||||
int ret = m_outputPhrase.Compare(otherFF.m_outputPhrase);
|
||||
return ret;
|
||||
} else {
|
||||
// compare hypo address. Won't be equal unless they're actually the same hypo
|
||||
if (m_hypo == otherFF.m_hypo)
|
||||
return 0;
|
||||
return (m_hypo < otherFF.m_hypo) ? -1 : +1;
|
||||
}
|
||||
}
|
||||
|
||||
size_t ControlRecombinationState::hash() const
|
||||
{
|
||||
size_t ret;
|
||||
@ -65,8 +50,7 @@ bool ControlRecombinationState::operator==(const FFState& other) const
|
||||
const ControlRecombinationState &otherFF = static_cast<const ControlRecombinationState&>(other);
|
||||
|
||||
if (m_ff.GetType() == SameOutput) {
|
||||
int ret = m_outputPhrase.Compare(otherFF.m_outputPhrase);
|
||||
return ret == 0;
|
||||
return m_outputPhrase == otherFF.m_outputPhrase;
|
||||
} else {
|
||||
// compare hypo address. Won't be equal unless they're actually the same hypo
|
||||
if (m_hypo == otherFF.m_hypo)
|
||||
|
@ -26,7 +26,6 @@ public:
|
||||
ControlRecombinationState(const Hypothesis &hypo, const ControlRecombination &ff);
|
||||
ControlRecombinationState(const ChartHypothesis &hypo, const ControlRecombination &ff);
|
||||
|
||||
int Compare(const FFState& other) const;
|
||||
virtual size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
|
@ -19,29 +19,6 @@ using namespace std;
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
int CoveredReferenceState::Compare(const FFState& other) const
|
||||
{
|
||||
const CoveredReferenceState &otherState = static_cast<const CoveredReferenceState&>(other);
|
||||
|
||||
if (m_coveredRef.size() != otherState.m_coveredRef.size()) {
|
||||
return (m_coveredRef.size() < otherState.m_coveredRef.size()) ? -1 : +1;
|
||||
} else {
|
||||
multiset<string>::const_iterator thisIt, otherIt;
|
||||
for (thisIt = m_coveredRef.begin(), otherIt = otherState.m_coveredRef.begin();
|
||||
thisIt != m_coveredRef.end();
|
||||
thisIt++, otherIt++) {
|
||||
if (*thisIt != *otherIt) return thisIt->compare(*otherIt);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
|
||||
// return m_coveredRef == otherState.m_coveredRef;
|
||||
|
||||
// if (m_coveredRef == otherState.m_coveredRef)
|
||||
// return 0;
|
||||
// return (m_coveredRef.size() < otherState.m_coveredRef.size()) ? -1 : +1;
|
||||
}
|
||||
|
||||
size_t CoveredReferenceState::hash() const
|
||||
{
|
||||
UTIL_THROW2("TODO:Haven't figure this out yet");
|
||||
|
@ -19,7 +19,6 @@ class CoveredReferenceState : public FFState
|
||||
public:
|
||||
std::multiset<std::string> m_coveredRef;
|
||||
|
||||
int Compare(const FFState& other) const;
|
||||
virtual size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
|
@ -13,13 +13,6 @@ struct DistortionState_traditional : public FFState {
|
||||
WordsRange range;
|
||||
int first_gap;
|
||||
DistortionState_traditional(const WordsRange& wr, int fg) : range(wr), first_gap(fg) {}
|
||||
int Compare(const FFState& other) const {
|
||||
const DistortionState_traditional& o =
|
||||
static_cast<const DistortionState_traditional&>(other);
|
||||
if (range.GetEndPos() < o.range.GetEndPos()) return -1;
|
||||
if (range.GetEndPos() > o.range.GetEndPos()) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t hash() const {
|
||||
return range.GetEndPos();
|
||||
|
@ -12,7 +12,6 @@ class FFState
|
||||
{
|
||||
public:
|
||||
virtual ~FFState();
|
||||
//virtual int Compare(const FFState& other) const = 0;
|
||||
virtual size_t hash() const = 0;
|
||||
virtual bool operator==(const FFState& other) const = 0;
|
||||
|
||||
@ -25,9 +24,6 @@ class DummyState : public FFState
|
||||
{
|
||||
public:
|
||||
DummyState() {}
|
||||
int Compare(const FFState& other) const {
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual size_t hash() const {
|
||||
return 0;
|
||||
|
@ -43,7 +43,7 @@ GlobalLexicalModel::~GlobalLexicalModel()
|
||||
// delete words in the hash data structure
|
||||
DoubleHash::const_iterator iter;
|
||||
for(iter = m_hash.begin(); iter != m_hash.end(); iter++ ) {
|
||||
map< const Word*, float, WordComparer >::const_iterator iter2;
|
||||
boost::unordered_map< const Word*, float, UnorderedComparer<Word>, UnorderedComparer<Word> >::const_iterator iter2;
|
||||
for(iter2 = iter->second.begin(); iter2 != iter->second.end(); iter2++ ) {
|
||||
delete iter2->first; // delete input word
|
||||
}
|
||||
@ -134,7 +134,7 @@ float GlobalLexicalModel::ScorePhrase( const TargetPhrase& targetPhrase ) const
|
||||
sum += inputWordHash->second;
|
||||
}
|
||||
|
||||
set< const Word*, WordComparer > alreadyScored; // do not score a word twice
|
||||
boost::unordered_set< const Word*, UnorderedComparer<Word>, UnorderedComparer<Word> > alreadyScored; // do not score a word twice
|
||||
for(size_t inputIndex = 0; inputIndex < input.GetSize(); inputIndex++ ) {
|
||||
const Word& inputWord = input.GetWord( inputIndex );
|
||||
if ( alreadyScored.find( &inputWord ) == alreadyScored.end() ) {
|
||||
|
@ -33,8 +33,10 @@ class InputType;
|
||||
*/
|
||||
class GlobalLexicalModel : public StatelessFeatureFunction
|
||||
{
|
||||
typedef std::map< const Word*, std::map< const Word*, float, WordComparer >, WordComparer > DoubleHash;
|
||||
typedef std::map< const Word*, float, WordComparer > SingleHash;
|
||||
typedef boost::unordered_map< const Word*,
|
||||
boost::unordered_map< const Word*, float, UnorderedComparer<Word> , UnorderedComparer<Word> >,
|
||||
UnorderedComparer<Word>, UnorderedComparer<Word> > DoubleHash;
|
||||
typedef boost::unordered_map< const Word*, float, UnorderedComparer<Word>, UnorderedComparer<Word> > SingleHash;
|
||||
typedef std::map< const TargetPhrase*, float > LexiconCache;
|
||||
|
||||
struct ThreadLocalStorage {
|
||||
|
@ -141,10 +141,6 @@ public:
|
||||
return m_tree;
|
||||
}
|
||||
|
||||
int Compare(const FFState& other) const {
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual size_t hash() const {
|
||||
return 0;
|
||||
}
|
||||
|
@ -312,25 +312,6 @@ PhraseBasedReorderingState(const LRModel &config,
|
||||
{ }
|
||||
|
||||
|
||||
int
|
||||
PhraseBasedReorderingState::
|
||||
Compare(const FFState& o) const
|
||||
{
|
||||
if (&o == this) return 0;
|
||||
|
||||
const PhraseBasedReorderingState* other = static_cast<const PhraseBasedReorderingState*>(&o);
|
||||
if (m_prevRange == other->m_prevRange) {
|
||||
if (m_direction == LRModel::Forward) {
|
||||
return ComparePrevScores(other->m_prevOption);
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
} else if (m_prevRange < other->m_prevRange) {
|
||||
return -1;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
size_t PhraseBasedReorderingState::hash() const
|
||||
{
|
||||
size_t ret;
|
||||
@ -378,21 +359,6 @@ Expand(const TranslationOption& topt, const InputType& input,
|
||||
///////////////////////////
|
||||
//BidirectionalReorderingState
|
||||
|
||||
int
|
||||
BidirectionalReorderingState::
|
||||
Compare(FFState const& o) const
|
||||
{
|
||||
/*
|
||||
if (&o == this) return 0;
|
||||
|
||||
BidirectionalReorderingState const &other
|
||||
= static_cast<BidirectionalReorderingState const&>(o);
|
||||
|
||||
int cmp = m_backward->Compare(*other.m_backward);
|
||||
return (cmp < 0) ? -1 : cmp ? 1 : m_forward->Compare(*other.m_forward);
|
||||
*/
|
||||
}
|
||||
|
||||
size_t BidirectionalReorderingState::hash() const
|
||||
{
|
||||
size_t ret = m_backward->hash();
|
||||
@ -436,16 +402,6 @@ HReorderingBackwardState(const LRModel &config, size_t offset)
|
||||
: LRState(config, LRModel::Backward, offset)
|
||||
{ }
|
||||
|
||||
|
||||
int
|
||||
HReorderingBackwardState::
|
||||
Compare(const FFState& o) const
|
||||
{
|
||||
const HReorderingBackwardState& other
|
||||
= static_cast<const HReorderingBackwardState&>(o);
|
||||
return m_reoStack.Compare(other.m_reoStack);
|
||||
}
|
||||
|
||||
size_t HReorderingBackwardState::hash() const
|
||||
{
|
||||
size_t ret = m_reoStack.hash();
|
||||
@ -497,20 +453,6 @@ HReorderingForwardState(const HReorderingForwardState *prev,
|
||||
m_coverage.SetValue(topt.GetSourceWordsRange(), true);
|
||||
}
|
||||
|
||||
int
|
||||
HReorderingForwardState::
|
||||
Compare(const FFState& o) const
|
||||
{
|
||||
if (&o == this) return 0;
|
||||
|
||||
HReorderingForwardState const& other
|
||||
= static_cast<HReorderingForwardState const&>(o);
|
||||
|
||||
return ((m_prevRange == other.m_prevRange)
|
||||
? ComparePrevScores(other.m_prevOption)
|
||||
: (m_prevRange < other.m_prevRange) ? -1 : 1);
|
||||
}
|
||||
|
||||
size_t HReorderingForwardState::hash() const
|
||||
{
|
||||
size_t ret;
|
||||
|
@ -218,10 +218,6 @@ public:
|
||||
delete m_forward;
|
||||
}
|
||||
|
||||
virtual
|
||||
int
|
||||
Compare(const FFState& o) const;
|
||||
|
||||
virtual size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
@ -247,10 +243,6 @@ public:
|
||||
PhraseBasedReorderingState(const PhraseBasedReorderingState *prev,
|
||||
const TranslationOption &topt);
|
||||
|
||||
virtual
|
||||
int
|
||||
Compare(const FFState& o) const;
|
||||
|
||||
virtual size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
@ -277,9 +269,6 @@ public:
|
||||
HReorderingBackwardState(const HReorderingBackwardState *prev,
|
||||
const TranslationOption &topt,
|
||||
ReorderingStack reoStack);
|
||||
|
||||
virtual int Compare(const FFState& o) const;
|
||||
|
||||
virtual size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
@ -308,8 +297,6 @@ public:
|
||||
HReorderingForwardState(const HReorderingForwardState *prev,
|
||||
const TranslationOption &topt);
|
||||
|
||||
virtual int Compare(const FFState& o) const;
|
||||
|
||||
virtual size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
|
@ -9,17 +9,6 @@
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
int ReorderingStack::Compare(const ReorderingStack& o) const
|
||||
{
|
||||
const ReorderingStack& other = static_cast<const ReorderingStack&>(o);
|
||||
if (other.m_stack > m_stack) {
|
||||
return 1;
|
||||
} else if (other.m_stack < m_stack) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t ReorderingStack::hash() const
|
||||
{
|
||||
std::size_t ret = boost::hash_range(m_stack.begin(), m_stack.end());
|
||||
|
@ -27,7 +27,6 @@ private:
|
||||
|
||||
public:
|
||||
|
||||
int Compare(const ReorderingStack& o) const;
|
||||
size_t hash() const;
|
||||
bool operator==(const ReorderingStack& other) const;
|
||||
|
||||
|
@ -1,5 +1,4 @@
|
||||
#include <vector>
|
||||
#include <set>
|
||||
#include "NieceTerminal.h"
|
||||
#include "moses/ScoreComponentCollection.h"
|
||||
#include "moses/TargetPhrase.h"
|
||||
@ -45,7 +44,7 @@ void NieceTerminal::EvaluateWithSourceContext(const InputType &input
|
||||
const Phrase *ruleSource = targetPhrase.GetRuleSource();
|
||||
assert(ruleSource);
|
||||
|
||||
std::set<Word> terms;
|
||||
boost::unordered_set<Word> terms;
|
||||
for (size_t i = 0; i < ruleSource->GetSize(); ++i) {
|
||||
const Word &word = ruleSource->GetWord(i);
|
||||
if (!word.IsNonTerminal()) {
|
||||
@ -81,9 +80,9 @@ void NieceTerminal::EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||
|
||||
bool NieceTerminal::ContainTerm(const InputType &input,
|
||||
const WordsRange &ntRange,
|
||||
const std::set<Word> &terms) const
|
||||
const boost::unordered_set<Word> &terms) const
|
||||
{
|
||||
std::set<Word>::const_iterator iter;
|
||||
boost::unordered_set<Word>::const_iterator iter;
|
||||
|
||||
for (size_t pos = ntRange.GetStartPos(); pos <= ntRange.GetEndPos(); ++pos) {
|
||||
const Word &word = input.GetWord(pos);
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <set>
|
||||
#include <boost/unordered_set.hpp>
|
||||
#include <string>
|
||||
#include "StatelessFeatureFunction.h"
|
||||
|
||||
@ -46,7 +46,7 @@ protected:
|
||||
bool m_hardConstraint;
|
||||
bool ContainTerm(const InputType &input,
|
||||
const WordsRange &ntRange,
|
||||
const std::set<Word> &terms) const;
|
||||
const boost::unordered_set<Word> &terms) const;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -22,23 +22,6 @@ void osmState::saveState(int jVal, int eVal, map <int , string> & gapVal)
|
||||
E = eVal;
|
||||
}
|
||||
|
||||
int osmState::Compare(const FFState& otherBase) const
|
||||
{
|
||||
const osmState &other = static_cast<const osmState&>(otherBase);
|
||||
if (j != other.j)
|
||||
return (j < other.j) ? -1 : +1;
|
||||
if (E != other.E)
|
||||
return (E < other.E) ? -1 : +1;
|
||||
if (gap != other.gap)
|
||||
return (gap < other.gap) ? -1 : +1;
|
||||
|
||||
if (lmState.length < other.lmState.length) return -1;
|
||||
|
||||
if (lmState.length > other.lmState.length) return 1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t osmState::hash() const
|
||||
{
|
||||
size_t ret = j;
|
||||
|
@ -16,7 +16,6 @@ class osmState : public FFState
|
||||
{
|
||||
public:
|
||||
osmState(const lm::ngram::State & val);
|
||||
int Compare(const FFState& other) const;
|
||||
virtual size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
|
@ -10,14 +10,6 @@ using namespace std;
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
int PhraseBoundaryState::Compare(const FFState& other) const
|
||||
{
|
||||
const PhraseBoundaryState& rhs = dynamic_cast<const PhraseBoundaryState&>(other);
|
||||
int tgt = Word::Compare(*m_targetWord,*(rhs.m_targetWord));
|
||||
if (tgt) return tgt;
|
||||
return Word::Compare(*m_sourceWord,*(rhs.m_sourceWord));
|
||||
}
|
||||
|
||||
size_t PhraseBoundaryState::hash() const
|
||||
{
|
||||
size_t ret = hash_value(*m_targetWord);
|
||||
@ -27,7 +19,7 @@ size_t PhraseBoundaryState::hash() const
|
||||
}
|
||||
bool PhraseBoundaryState::operator==(const FFState& other) const
|
||||
{
|
||||
const PhraseBoundaryState& rhs = dynamic_cast<const PhraseBoundaryState&>(other);
|
||||
const PhraseBoundaryState& rhs = static_cast<const PhraseBoundaryState&>(other);
|
||||
bool ret = *m_targetWord == *rhs.m_targetWord && *m_sourceWord == *rhs.m_sourceWord;
|
||||
return ret;
|
||||
}
|
||||
@ -86,7 +78,7 @@ FFState* PhraseBoundaryFeature::EvaluateWhenApplied
|
||||
(const Hypothesis& cur_hypo, const FFState* prev_state,
|
||||
ScoreComponentCollection* scores) const
|
||||
{
|
||||
const PhraseBoundaryState* pbState = dynamic_cast<const PhraseBoundaryState*>(prev_state);
|
||||
const PhraseBoundaryState* pbState = static_cast<const PhraseBoundaryState*>(prev_state);
|
||||
const Phrase& targetPhrase = cur_hypo.GetCurrTargetPhrase();
|
||||
if (targetPhrase.GetSize() == 0) {
|
||||
return new PhraseBoundaryState(*pbState);
|
||||
|
@ -23,7 +23,6 @@ public:
|
||||
const Word* GetTargetWord() const {
|
||||
return m_targetWord;
|
||||
}
|
||||
virtual int Compare(const FFState& other) const;
|
||||
virtual size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
|
@ -99,47 +99,6 @@ public:
|
||||
return m_rightBoundaryNonTerminalR2LScores[2];
|
||||
}
|
||||
|
||||
|
||||
int Compare(const FFState& other) const {
|
||||
if (!m_distinguishStates) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const PhraseOrientationFeatureState &otherState = static_cast<const PhraseOrientationFeatureState&>(other);
|
||||
|
||||
if (!m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet &&
|
||||
!m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) {
|
||||
return 0;
|
||||
}
|
||||
if (m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet) {
|
||||
return 1;
|
||||
}
|
||||
if (!m_leftBoundaryIsSet && otherState.m_leftBoundaryIsSet) {
|
||||
return -1;
|
||||
}
|
||||
if (m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) {
|
||||
return 1;
|
||||
}
|
||||
if (!m_rightBoundaryIsSet && otherState.m_rightBoundaryIsSet) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (m_leftBoundaryIsSet) {
|
||||
int compareLeft = CompareLeftBoundaryRecursive(*this, otherState, m_useSparseNT);
|
||||
if (compareLeft != 0) {
|
||||
return compareLeft;
|
||||
}
|
||||
}
|
||||
if (m_rightBoundaryIsSet) {
|
||||
int compareRight = CompareRightBoundaryRecursive(*this, otherState, m_useSparseNT);
|
||||
if (compareRight != 0) {
|
||||
return compareRight;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
};
|
||||
|
||||
virtual size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
|
@ -7,14 +7,6 @@ using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
int SkeletonState::Compare(const FFState& other) const
|
||||
{
|
||||
const SkeletonState &otherState = static_cast<const SkeletonState&>(other);
|
||||
|
||||
if (m_targetLen == otherState.m_targetLen)
|
||||
return 0;
|
||||
return (m_targetLen < otherState.m_targetLen) ? -1 : +1;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
SkeletonStatefulFF::SkeletonStatefulFF(const std::string &line)
|
||||
|
@ -15,8 +15,6 @@ public:
|
||||
:m_targetLen(targetLen) {
|
||||
}
|
||||
|
||||
int Compare(const FFState& other) const;
|
||||
|
||||
virtual size_t hash() const {
|
||||
return (size_t) m_targetLen;
|
||||
}
|
||||
|
@ -178,7 +178,7 @@ void SparseHieroReorderingFeature::EvaluateWhenApplied(
|
||||
|
||||
//Iterate through block pairs
|
||||
const Sentence& sentence =
|
||||
dynamic_cast<const Sentence&>(cur_hypo.GetManager().GetSource());
|
||||
static_cast<const Sentence&>(cur_hypo.GetManager().GetSource());
|
||||
//const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase();
|
||||
for (size_t i = 0; i < sourceBlocks.size()-1; ++i) {
|
||||
Block& leftSourceBlock = sourceBlocks[i];
|
||||
|
@ -11,12 +11,6 @@ using namespace std;
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
int TargetBigramState::Compare(const FFState& other) const
|
||||
{
|
||||
const TargetBigramState& rhs = dynamic_cast<const TargetBigramState&>(other);
|
||||
return Word::Compare(m_word,rhs.m_word);
|
||||
}
|
||||
|
||||
size_t TargetBigramState::hash() const
|
||||
{
|
||||
std::size_t ret = hash_value(m_word);
|
||||
@ -25,7 +19,7 @@ size_t TargetBigramState::hash() const
|
||||
|
||||
bool TargetBigramState::operator==(const FFState& other) const
|
||||
{
|
||||
const TargetBigramState& rhs = dynamic_cast<const TargetBigramState&>(other);
|
||||
const TargetBigramState& rhs = static_cast<const TargetBigramState&>(other);
|
||||
return m_word == rhs.m_word;
|
||||
}
|
||||
|
||||
@ -81,7 +75,7 @@ FFState* TargetBigramFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo,
|
||||
const FFState* prev_state,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{
|
||||
const TargetBigramState* tbState = dynamic_cast<const TargetBigramState*>(prev_state);
|
||||
const TargetBigramState* tbState = static_cast<const TargetBigramState*>(prev_state);
|
||||
assert(tbState);
|
||||
|
||||
// current hypothesis target phrase
|
||||
|
@ -20,7 +20,6 @@ public:
|
||||
const Word& GetWord() const {
|
||||
return m_word;
|
||||
}
|
||||
virtual int Compare(const FFState& other) const;
|
||||
size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
|
@ -12,31 +12,6 @@ namespace Moses
|
||||
|
||||
using namespace std;
|
||||
|
||||
int TargetNgramState::Compare(const FFState& other) const
|
||||
{
|
||||
const TargetNgramState& rhs = dynamic_cast<const TargetNgramState&>(other);
|
||||
int result;
|
||||
if (m_words.size() == rhs.m_words.size()) {
|
||||
for (size_t i = 0; i < m_words.size(); ++i) {
|
||||
result = Word::Compare(m_words[i],rhs.m_words[i]);
|
||||
if (result != 0) return result;
|
||||
}
|
||||
return 0;
|
||||
} else if (m_words.size() < rhs.m_words.size()) {
|
||||
for (size_t i = 0; i < m_words.size(); ++i) {
|
||||
result = Word::Compare(m_words[i],rhs.m_words[i]);
|
||||
if (result != 0) return result;
|
||||
}
|
||||
return -1;
|
||||
} else {
|
||||
for (size_t i = 0; i < rhs.m_words.size(); ++i) {
|
||||
result = Word::Compare(m_words[i],rhs.m_words[i]);
|
||||
if (result != 0) return result;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
size_t TargetNgramState::hash() const
|
||||
{
|
||||
std::size_t ret = boost::hash_range(m_words.begin(), m_words.end());
|
||||
@ -45,24 +20,24 @@ size_t TargetNgramState::hash() const
|
||||
|
||||
bool TargetNgramState::operator==(const FFState& other) const
|
||||
{
|
||||
const TargetNgramState& rhs = dynamic_cast<const TargetNgramState&>(other);
|
||||
int result;
|
||||
const TargetNgramState& rhs = static_cast<const TargetNgramState&>(other);
|
||||
bool result;
|
||||
if (m_words.size() == rhs.m_words.size()) {
|
||||
for (size_t i = 0; i < m_words.size(); ++i) {
|
||||
result = Word::Compare(m_words[i],rhs.m_words[i]);
|
||||
if (result != 0) return false;
|
||||
result = m_words[i] == rhs.m_words[i];
|
||||
if (!result) return false;
|
||||
}
|
||||
return true;
|
||||
} else if (m_words.size() < rhs.m_words.size()) {
|
||||
for (size_t i = 0; i < m_words.size(); ++i) {
|
||||
result = Word::Compare(m_words[i],rhs.m_words[i]);
|
||||
if (result != 0) return false;
|
||||
result = m_words[i] == rhs.m_words[i];
|
||||
if (!result) return false;
|
||||
}
|
||||
return true;
|
||||
} else {
|
||||
for (size_t i = 0; i < rhs.m_words.size(); ++i) {
|
||||
result = Word::Compare(m_words[i],rhs.m_words[i]);
|
||||
if (result != 0) return false;
|
||||
result = m_words[i] == rhs.m_words[i];
|
||||
if (!result) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
@ -26,7 +26,6 @@ public:
|
||||
const std::vector<Word> GetWords() const {
|
||||
return m_words;
|
||||
}
|
||||
virtual int Compare(const FFState& other) const;
|
||||
|
||||
size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
@ -159,25 +158,6 @@ public:
|
||||
return m_contextSuffix;
|
||||
}
|
||||
|
||||
int Compare(const FFState& o) const {
|
||||
const TargetNgramChartState &other =
|
||||
static_cast<const TargetNgramChartState &>( o );
|
||||
|
||||
// prefix
|
||||
if (m_startPos > 0) { // not for "<s> ..."
|
||||
int ret = GetPrefix().Compare(other.GetPrefix());
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (m_endPos < m_inputSize - 1) { // not for "... </s>"
|
||||
int ret = GetSuffix().Compare(other.GetSuffix());
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t hash() const {
|
||||
// not sure if this is correct
|
||||
size_t ret;
|
||||
@ -203,14 +183,12 @@ public:
|
||||
|
||||
// prefix
|
||||
if (m_startPos > 0) { // not for "<s> ..."
|
||||
int ret = GetPrefix().Compare(other.GetPrefix());
|
||||
if (ret != 0)
|
||||
if (GetPrefix() != other.GetPrefix())
|
||||
return false;
|
||||
}
|
||||
|
||||
if (m_endPos < m_inputSize - 1) { // not for "... </s>"
|
||||
int ret = GetSuffix().Compare(other.GetSuffix());
|
||||
if (ret != 0)
|
||||
if (GetSuffix() != other.GetSuffix())
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
@ -34,7 +34,7 @@ FFState* TreeStructureFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hy
|
||||
if (word.IsNonTerminal()) {
|
||||
size_t nonTermInd = cur_hypo.GetCurrTargetPhrase().GetAlignNonTerm().GetNonTermIndexMap()[pos];
|
||||
const ChartHypothesis *prevHypo = cur_hypo.GetPrevHypo(nonTermInd);
|
||||
const TreeState* prev = dynamic_cast<const TreeState*>(prevHypo->GetFFState(featureID));
|
||||
const TreeState* prev = static_cast<const TreeState*>(prevHypo->GetFFState(featureID));
|
||||
const TreePointer prev_tree = prev->GetTree();
|
||||
previous_trees.push_back(prev_tree);
|
||||
}
|
||||
|
@ -23,9 +23,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
#define moses_GenerationDictionary_h
|
||||
|
||||
#include <list>
|
||||
#include <map>
|
||||
#include <stdexcept>
|
||||
#include <vector>
|
||||
#include <boost/unordered_map.hpp>
|
||||
#include "ScoreComponentCollection.h"
|
||||
#include "Phrase.h"
|
||||
#include "TypeDef.h"
|
||||
@ -36,7 +36,7 @@ namespace Moses
|
||||
|
||||
class FactorCollection;
|
||||
|
||||
typedef std::map < Word , ScoreComponentCollection > OutputWordCollection;
|
||||
typedef boost::unordered_map < Word , ScoreComponentCollection > OutputWordCollection;
|
||||
// 1st = output phrase
|
||||
// 2nd = log probability (score)
|
||||
|
||||
@ -44,7 +44,7 @@ typedef std::map < Word , ScoreComponentCollection > OutputWordCollection;
|
||||
*/
|
||||
class GenerationDictionary : public DecodeFeature
|
||||
{
|
||||
typedef std::map<const Word* , OutputWordCollection, WordComparer> Collection;
|
||||
typedef boost::unordered_map<const Word* , OutputWordCollection, UnorderedComparer<Word>, UnorderedComparer<Word> > Collection;
|
||||
protected:
|
||||
static std::vector<GenerationDictionary*> s_staticColl;
|
||||
|
||||
|
@ -314,23 +314,10 @@ struct CompareHypothesisTotalScore {
|
||||
ObjectPool<Hypothesis> &pool = Hypothesis::GetObjectPool(); \
|
||||
pool.freeObject(hypo); \
|
||||
} \
|
||||
|
||||
|
||||
#else
|
||||
#define FREEHYPO(hypo) delete hypo
|
||||
#endif
|
||||
|
||||
class HypothesisRecombinationUnordered
|
||||
{
|
||||
public:
|
||||
size_t operator()(const Hypothesis* hypo) const {
|
||||
return hypo->hash();
|
||||
}
|
||||
|
||||
bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const {
|
||||
return (*hypoA) == (*hypoB);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
#endif
|
||||
|
@ -19,7 +19,7 @@ class HypothesisStack
|
||||
{
|
||||
|
||||
protected:
|
||||
typedef boost::unordered_set< Hypothesis*, HypothesisRecombinationUnordered, HypothesisRecombinationUnordered > _HCType;
|
||||
typedef boost::unordered_set< Hypothesis*, UnorderedComparer<Hypothesis>, UnorderedComparer<Hypothesis> > _HCType;
|
||||
_HCType m_hypos; /**< contains hypotheses */
|
||||
Manager& m_manager;
|
||||
|
||||
|
@ -450,7 +450,7 @@ void Manager::OutputDetailedTreeFragmentsTranslationReport(OutputCollector *coll
|
||||
}
|
||||
|
||||
const search::Applied *applied = &Completed()[0];
|
||||
const Sentence &sentence = dynamic_cast<const Sentence &>(m_source);
|
||||
const Sentence &sentence = static_cast<const Sentence &>(m_source);
|
||||
const size_t translationId = m_source.GetTranslationId();
|
||||
|
||||
std::ostringstream out;
|
||||
|
@ -25,12 +25,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
int BackwardLMState::Compare(const FFState &o) const
|
||||
{
|
||||
const BackwardLMState &other = static_cast<const BackwardLMState &>(o);
|
||||
return state.left.Compare(other.state.left);
|
||||
}
|
||||
|
||||
size_t BackwardLMState::hash() const
|
||||
{
|
||||
size_t ret = hash_value(state.left);
|
||||
|
@ -47,8 +47,6 @@ class BackwardLMState : public FFState
|
||||
|
||||
public:
|
||||
|
||||
int Compare(const FFState &o) const;
|
||||
|
||||
size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
|
@ -7,15 +7,6 @@ using namespace std;
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
int BilingualLMState::Compare(const FFState& other) const
|
||||
{
|
||||
const BilingualLMState &otherState = static_cast<const BilingualLMState&>(other);
|
||||
|
||||
if (m_hash == otherState.m_hash)
|
||||
return 0;
|
||||
return (m_hash < otherState.m_hash) ? -1 : +1;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
BilingualLM::BilingualLM(const std::string &line)
|
||||
: StatefulFeatureFunction(1, line),
|
||||
|
@ -37,8 +37,6 @@ public:
|
||||
return neuralLM_ids;
|
||||
}
|
||||
|
||||
int Compare(const FFState& other) const;
|
||||
|
||||
virtual size_t hash() const {
|
||||
return m_hash;
|
||||
}
|
||||
|
@ -145,29 +145,6 @@ public:
|
||||
return m_contextSuffix;
|
||||
}
|
||||
|
||||
int Compare(const FFState& o) const {
|
||||
/*
|
||||
const LanguageModelChartState &other =
|
||||
dynamic_cast<const LanguageModelChartState &>( o );
|
||||
|
||||
// prefix
|
||||
if (m_hypo.GetCurrSourceRange().GetStartPos() > 0) { // not for "<s> ..."
|
||||
int ret = GetPrefix().Compare(other.GetPrefix());
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
}
|
||||
|
||||
// suffix
|
||||
size_t inputSize = m_hypo.GetManager().GetSource().GetSize();
|
||||
if (m_hypo.GetCurrSourceRange().GetEndPos() < inputSize - 1) { // not for "... </s>"
|
||||
int ret = other.GetRightContext()->Compare(*m_lmRightContext);
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
}
|
||||
return 0;
|
||||
*/
|
||||
}
|
||||
|
||||
size_t hash() const {
|
||||
size_t ret;
|
||||
|
||||
@ -190,7 +167,7 @@ public:
|
||||
}
|
||||
virtual bool operator==(const FFState& o) const {
|
||||
const LanguageModelChartState &other =
|
||||
dynamic_cast<const LanguageModelChartState &>( o );
|
||||
static_cast<const LanguageModelChartState &>( o );
|
||||
|
||||
// prefix
|
||||
if (m_hypo.GetCurrSourceRange().GetStartPos() > 0) { // not for "<s> ..."
|
||||
|
@ -56,13 +56,6 @@ namespace
|
||||
|
||||
struct KenLMState : public FFState {
|
||||
lm::ngram::State state;
|
||||
int Compare(const FFState &o) const {
|
||||
const KenLMState &other = static_cast<const KenLMState &>(o);
|
||||
if (state.length < other.state.length) return -1;
|
||||
if (state.length > other.state.length) return 1;
|
||||
return std::memcmp(state.words, other.state.words, sizeof(lm::WordIndex) * state.length);
|
||||
}
|
||||
|
||||
virtual size_t hash() const {
|
||||
size_t ret = hash_value(state);
|
||||
return ret;
|
||||
@ -313,12 +306,6 @@ public:
|
||||
return m_state;
|
||||
}
|
||||
|
||||
int Compare(const FFState& o) const {
|
||||
const LanguageModelChartStateKenLM &other = static_cast<const LanguageModelChartStateKenLM&>(o);
|
||||
int ret = m_state.Compare(other.m_state);
|
||||
return ret;
|
||||
}
|
||||
|
||||
size_t hash() const {
|
||||
size_t ret = hash_value(m_state);
|
||||
return ret;
|
||||
|
@ -10,13 +10,6 @@ struct PointerState : public FFState {
|
||||
PointerState(const void* lms) {
|
||||
lmstate = lms;
|
||||
}
|
||||
int Compare(const FFState& o) const {
|
||||
const PointerState& other = static_cast<const PointerState&>(o);
|
||||
if (other.lmstate > lmstate) return 1;
|
||||
else if (other.lmstate < lmstate) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual size_t hash() const {
|
||||
return (size_t) lmstate;
|
||||
}
|
||||
|
@ -521,7 +521,7 @@ void Manager::OutputWordGraph(std::ostream &outputWordGraphStream, const Hypothe
|
||||
const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions();
|
||||
for (size_t i = 0; i < statefulFFs.size(); ++i) {
|
||||
const StatefulFeatureFunction *ff = statefulFFs[i];
|
||||
const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff);
|
||||
const LanguageModel *lm = static_cast<const LanguageModel*>(ff);
|
||||
|
||||
vector<float> scores = hypo->GetScoreBreakdown().GetScoresForProducer(lm);
|
||||
|
||||
|
@ -254,6 +254,35 @@ int Phrase::Compare(const Phrase &other) const
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t Phrase::hash() const
|
||||
{
|
||||
size_t seed = 0;
|
||||
for (size_t i = 0; i < GetSize(); ++i) {
|
||||
boost::hash_combine(seed, GetWord(i));
|
||||
}
|
||||
return seed;
|
||||
}
|
||||
|
||||
bool Phrase::operator== (const Phrase &other) const
|
||||
{
|
||||
size_t thisSize = GetSize()
|
||||
,compareSize = other.GetSize();
|
||||
if (thisSize != compareSize) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (size_t pos = 0 ; pos < thisSize ; pos++) {
|
||||
const Word &thisWord = GetWord(pos)
|
||||
,&otherWord = other.GetWord(pos);
|
||||
bool ret = thisWord == otherWord;
|
||||
if (!ret) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
bool Phrase::Contains(const vector< vector<string> > &subPhraseVector
|
||||
, const vector<FactorType> &inputFactor) const
|
||||
|
@ -192,8 +192,11 @@ public:
|
||||
return Compare(compare) < 0;
|
||||
}
|
||||
|
||||
bool operator== (const Phrase &compare) const {
|
||||
return Compare(compare) == 0;
|
||||
size_t hash() const;
|
||||
|
||||
bool operator==(const Phrase &compare) const;
|
||||
bool operator!=(const Phrase &compare) const {
|
||||
return ! (*this == compare);
|
||||
}
|
||||
|
||||
void OnlyTheseFactors(const FactorMask &factors);
|
||||
@ -202,11 +205,7 @@ public:
|
||||
|
||||
inline size_t hash_value(const Phrase& phrase)
|
||||
{
|
||||
size_t seed = 0;
|
||||
for (size_t i = 0; i < phrase.GetSize(); ++i) {
|
||||
boost::hash_combine(seed, phrase.GetWord(i));
|
||||
}
|
||||
return seed;
|
||||
return phrase.hash();
|
||||
}
|
||||
|
||||
struct PhrasePtrComparator {
|
||||
|
@ -1,10 +1,10 @@
|
||||
#pragma once
|
||||
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/shared_ptr.hpp>
|
||||
#include <boost/unordered_map.hpp>
|
||||
#include <boost/unordered_set.hpp>
|
||||
|
||||
#include "moses/InputType.h"
|
||||
#include "moses/Syntax/KBestExtractor.h"
|
||||
|
@ -65,7 +65,7 @@ void Manager::OutputUnknowns(OutputCollector *collector) const
|
||||
long translationId = m_source.GetTranslationId();
|
||||
|
||||
std::ostringstream out;
|
||||
for (std::set<Moses::Word>::const_iterator p = m_oovs.begin();
|
||||
for (boost::unordered_set<Moses::Word>::const_iterator p = m_oovs.begin();
|
||||
p != m_oovs.end(); ++p) {
|
||||
out << *p;
|
||||
}
|
||||
|
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <boost/unordered_set.hpp>
|
||||
#include "moses/InputType.h"
|
||||
#include "moses/BaseManager.h"
|
||||
|
||||
@ -50,7 +51,7 @@ public:
|
||||
virtual const SHyperedge *GetBestSHyperedge() const = 0;
|
||||
|
||||
protected:
|
||||
std::set<Word> m_oovs;
|
||||
boost::unordered_set<Word> m_oovs;
|
||||
|
||||
private:
|
||||
// Syntax-specific helper functions used to implement OutputNBest.
|
||||
|
@ -108,7 +108,7 @@ void Manager<Parser>::InitializeParsers(PChart &pchart,
|
||||
// Find the set of OOVs for this input. This function assumes that the
|
||||
// PChart argument has already been initialized from the input.
|
||||
template<typename Parser>
|
||||
void Manager<Parser>::FindOovs(const PChart &pchart, std::set<Word> &oovs,
|
||||
void Manager<Parser>::FindOovs(const PChart &pchart, boost::unordered_set<Word> &oovs,
|
||||
std::size_t maxOovWidth)
|
||||
{
|
||||
// Get the set of RuleTries.
|
||||
|
@ -45,7 +45,7 @@ public:
|
||||
void OutputDetailedTranslationReport(OutputCollector *collector) const;
|
||||
|
||||
private:
|
||||
void FindOovs(const PChart &, std::set<Word> &, std::size_t);
|
||||
void FindOovs(const PChart &, boost::unordered_set<Word> &, std::size_t);
|
||||
|
||||
void InitializeCharts();
|
||||
|
||||
|
@ -230,29 +230,6 @@ void swap(TargetPhrase &first, TargetPhrase &second);
|
||||
|
||||
std::ostream& operator<<(std::ostream&, const TargetPhrase&);
|
||||
|
||||
/**
|
||||
* Hasher that looks at source and target phrase.
|
||||
**/
|
||||
struct TargetPhraseHasher {
|
||||
inline size_t operator()(const TargetPhrase& targetPhrase) const {
|
||||
size_t seed = 0;
|
||||
boost::hash_combine(seed, targetPhrase);
|
||||
boost::hash_combine(seed, targetPhrase.GetAlignTerm());
|
||||
boost::hash_combine(seed, targetPhrase.GetAlignNonTerm());
|
||||
|
||||
return seed;
|
||||
}
|
||||
};
|
||||
|
||||
struct TargetPhraseComparator {
|
||||
inline bool operator()(const TargetPhrase& lhs, const TargetPhrase& rhs) const {
|
||||
return lhs.Compare(rhs) == 0 &&
|
||||
lhs.GetAlignTerm() == rhs.GetAlignTerm() &&
|
||||
lhs.GetAlignNonTerm() == rhs.GetAlignNonTerm();
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
@ -115,8 +115,7 @@ CreateTargetPhraseCollection(const ttasksptr& ttask, const Phrase& src) const
|
||||
{
|
||||
// Aggregation of phrases and the scores that will be applied to them
|
||||
vector<TargetPhrase*> allPhrases;
|
||||
unordered_map<const TargetPhrase*, vector<float>, PhrasePtrHasher,
|
||||
PhrasePtrComparator> allScores;
|
||||
unordered_map<const TargetPhrase*, vector<float>, UnorderedComparer<Phrase>, UnorderedComparer<Phrase> > allScores;
|
||||
|
||||
// For each model
|
||||
size_t offset = 0;
|
||||
|
@ -143,7 +143,7 @@ InputPathList &TranslationOptionCollectionConfusionNet::GetInputPathList(size_t
|
||||
*/
|
||||
void TranslationOptionCollectionConfusionNet::ProcessUnknownWord(size_t sourcePos)
|
||||
{
|
||||
ConfusionNet const& source=dynamic_cast<ConfusionNet const&>(m_source);
|
||||
ConfusionNet const& source=static_cast<ConfusionNet const&>(m_source);
|
||||
|
||||
ConfusionNet::Column const& coll=source.GetColumn(sourcePos);
|
||||
const InputPathList &inputPathList = GetInputPathList(sourcePos, sourcePos);
|
||||
|
@ -75,7 +75,7 @@ void TranslationOptionCollectionText::ProcessUnknownWord(size_t sourcePos)
|
||||
*/
|
||||
bool TranslationOptionCollectionText::HasXmlOptionsOverlappingRange(size_t startPosition, size_t endPosition) const
|
||||
{
|
||||
Sentence const& source=dynamic_cast<Sentence const&>(m_source);
|
||||
Sentence const& source=static_cast<Sentence const&>(m_source);
|
||||
return source.XmlOverlap(startPosition,endPosition);
|
||||
}
|
||||
|
||||
@ -85,7 +85,7 @@ bool TranslationOptionCollectionText::HasXmlOptionsOverlappingRange(size_t start
|
||||
bool TranslationOptionCollectionText::ViolatesXmlOptionsConstraint(size_t startPosition, size_t endPosition, TranslationOption *transOpt) const
|
||||
{
|
||||
// skip if there is no overlap
|
||||
Sentence const& source=dynamic_cast<Sentence const&>(m_source);
|
||||
Sentence const& source=static_cast<Sentence const&>(m_source);
|
||||
if (!source.XmlOverlap(startPosition,endPosition)) {
|
||||
return false;
|
||||
}
|
||||
@ -135,7 +135,7 @@ bool TranslationOptionCollectionText::ViolatesXmlOptionsConstraint(size_t startP
|
||||
*/
|
||||
void TranslationOptionCollectionText::CreateXmlOptionsForRange(size_t startPos, size_t endPos)
|
||||
{
|
||||
Sentence const& source=dynamic_cast<Sentence const&>(m_source);
|
||||
Sentence const& source=static_cast<Sentence const&>(m_source);
|
||||
InputPath &inputPath = GetInputPath(startPos,endPos);
|
||||
|
||||
vector <TranslationOption*> xmlOptions;
|
||||
|
15
moses/Util.h
15
moses/Util.h
@ -428,7 +428,7 @@ inline float CalcTranslationScore(const std::vector<float> &probVector,
|
||||
out << *this; \
|
||||
return out.str(); \
|
||||
} \
|
||||
|
||||
|
||||
//! delete and remove every element of a collection object such as set, list etc
|
||||
template<class COLL>
|
||||
void RemoveAllInColl(COLL &coll)
|
||||
@ -537,6 +537,19 @@ class FeatureFunction;
|
||||
void PrintFeatureWeight(const FeatureFunction* ff);
|
||||
void ShowWeights();
|
||||
|
||||
template<typename T>
|
||||
class UnorderedComparer
|
||||
{
|
||||
public:
|
||||
size_t operator()(const T* obj) const {
|
||||
return obj->hash();
|
||||
}
|
||||
|
||||
bool operator()(const T* a, const T* b) const {
|
||||
return (*a) == (*b);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
|
@ -64,7 +64,23 @@ int Word::Compare(const Word &targetWord, const Word &sourceWord)
|
||||
return (targetFactor<sourceFactor) ? -1 : +1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool Word::operator==(const Word &compare) const
|
||||
{
|
||||
if (IsNonTerminal() != compare.IsNonTerminal()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (size_t factorType = 0 ; factorType < MAX_NUM_FACTORS ; factorType++) {
|
||||
const Factor *thisFactor = GetFactor(factorType);
|
||||
const Factor *otherFactor = compare.GetFactor(factorType);
|
||||
|
||||
if (thisFactor != otherFactor) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void Word::Merge(const Word &sourceWord)
|
||||
|
30
moses/Word.h
30
moses/Word.h
@ -116,30 +116,12 @@ public:
|
||||
StringPiece GetString(FactorType factorType) const;
|
||||
TO_STRING();
|
||||
|
||||
//! transitive comparison of Word objects
|
||||
inline bool operator< (const Word &compare) const {
|
||||
// needed to store word in GenerationDictionary map
|
||||
// uses comparison of FactorKey
|
||||
// 'proper' comparison, not address/id comparison
|
||||
return Compare(*this, compare) < 0;
|
||||
}
|
||||
|
||||
inline bool operator== (const Word &compare) const {
|
||||
// needed to store word in GenerationDictionary map
|
||||
// uses comparison of FactorKey
|
||||
// 'proper' comparison, not address/id comparison
|
||||
return Compare(*this, compare) == 0;
|
||||
}
|
||||
bool operator== (const Word &compare) const;
|
||||
|
||||
inline bool operator!= (const Word &compare) const {
|
||||
return Compare(*this, compare) != 0;
|
||||
return !(*this == compare);
|
||||
}
|
||||
|
||||
int Compare(const Word &other) const {
|
||||
return Compare(*this, other);
|
||||
}
|
||||
|
||||
|
||||
/* static functions */
|
||||
|
||||
/** transitive comparison of 2 word objects. Used by operator<.
|
||||
@ -163,14 +145,6 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
struct WordComparer {
|
||||
//! returns true if hypoA can be recombined with hypoB
|
||||
bool operator()(const Word *a, const Word *b) const {
|
||||
return *a < *b;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
inline size_t hash_value(const Word& word)
|
||||
{
|
||||
return word.hash();
|
||||
|
Loading…
Reference in New Issue
Block a user