This commit is contained in:
Nicola Bertoldi 2014-01-15 16:42:02 +01:00
parent a098550f33
commit bd83999264
91 changed files with 820 additions and 842 deletions

View File

@ -161,13 +161,13 @@ BackwardsEdge::BackwardsEdge(const BitmapContainer &prevBitmapContainer
} }
if (m_translations.size() > 1) { if (m_translations.size() > 1) {
UTIL_THROW_IF2(m_translations.Get(0)->GetFutureScore() < m_translations.Get(1)->GetFutureScore(), UTIL_THROW_IF2(m_translations.Get(0)->GetFutureScore() < m_translations.Get(1)->GetFutureScore(),
"Non-monotonic future score"); "Non-monotonic future score");
} }
if (m_hypotheses.size() > 1) { if (m_hypotheses.size() > 1) {
UTIL_THROW_IF2(m_hypotheses[0]->GetTotalScore() < m_hypotheses[1]->GetTotalScore(), UTIL_THROW_IF2(m_hypotheses[0]->GetTotalScore() < m_hypotheses[1]->GetTotalScore(),
"Non-monotonic total score"); "Non-monotonic total score");
} }
HypothesisScoreOrdererWithDistortion orderer (&transOptRange); HypothesisScoreOrdererWithDistortion orderer (&transOptRange);
@ -442,7 +442,7 @@ BitmapContainer::ProcessBestHypothesis()
if (!Empty()) { if (!Empty()) {
HypothesisQueueItem *check = Dequeue(true); HypothesisQueueItem *check = Dequeue(true);
UTIL_THROW_IF2(item->GetHypothesis()->GetTotalScore() < check->GetHypothesis()->GetTotalScore(), UTIL_THROW_IF2(item->GetHypothesis()->GetTotalScore() < check->GetHypothesis()->GetTotalScore(),
"Non-monotonic total score"); "Non-monotonic total score");
} }
// Logging for the criminally insane // Logging for the criminally insane

View File

@ -108,12 +108,12 @@ void ChartHypothesis::GetOutputPhrase(Phrase &outPhrase) const
if (sourcePosSet.size() == 1) { if (sourcePosSet.size() == 1) {
const std::vector<const Word*> *ruleSourceFromInputPath = GetTranslationOption().GetSourceRuleFromInputPath(); const std::vector<const Word*> *ruleSourceFromInputPath = GetTranslationOption().GetSourceRuleFromInputPath();
UTIL_THROW_IF2(ruleSourceFromInputPath == NULL, UTIL_THROW_IF2(ruleSourceFromInputPath == NULL,
"No source rule"); "No source rule");
size_t sourcePos = *sourcePosSet.begin(); size_t sourcePos = *sourcePosSet.begin();
const Word *sourceWord = ruleSourceFromInputPath->at(sourcePos); const Word *sourceWord = ruleSourceFromInputPath->at(sourcePos);
UTIL_THROW_IF2(sourceWord == NULL, UTIL_THROW_IF2(sourceWord == NULL,
"No source word"); "No source word");
const Factor *factor = sourceWord->GetFactor(placeholderFactor); const Factor *factor = sourceWord->GetFactor(placeholderFactor);
if (factor) { if (factor) {
outPhrase.Back()[0] = factor; outPhrase.Back()[0] = factor;
@ -247,10 +247,10 @@ void ChartHypothesis::CleanupArcList()
if (!distinctNBest && m_arcList->size() > nBestSize) { if (!distinctNBest && m_arcList->size() > nBestSize) {
// prune arc list only if there too many arcs // prune arc list only if there too many arcs
NTH_ELEMENT4(m_arcList->begin() NTH_ELEMENT4(m_arcList->begin()
, m_arcList->begin() + nBestSize - 1 , m_arcList->begin() + nBestSize - 1
, m_arcList->end() , m_arcList->end()
, CompareChartChartHypothesisTotalScore()); , CompareChartChartHypothesisTotalScore());
// delete bad ones // delete bad ones
ChartArcList::iterator iter; ChartArcList::iterator iter;

View File

@ -89,7 +89,7 @@ bool ChartHypothesisCollection::AddHypothesis(ChartHypothesis *hypo, ChartManage
HCType::iterator &iterExisting = addRet.first; HCType::iterator &iterExisting = addRet.first;
ChartHypothesis *hypoExisting = *iterExisting; ChartHypothesis *hypoExisting = *iterExisting;
UTIL_THROW_IF2(iterExisting == m_hypos.end(), UTIL_THROW_IF2(iterExisting == m_hypos.end(),
"Adding a hypothesis should have returned a valid iterator"); "Adding a hypothesis should have returned a valid iterator");
//StaticData::Instance().GetSentenceStats().AddRecombination(*hypo, **iterExisting); //StaticData::Instance().GetSentenceStats().AddRecombination(*hypo, **iterExisting);
@ -253,7 +253,7 @@ void ChartHypothesisCollection::PruneToSize(ChartManager &manager)
ChartHypothesis *hypo = *iter; ChartHypothesis *hypo = *iter;
HCType::iterator iterFindHypo = m_hypos.find(hypo); HCType::iterator iterFindHypo = m_hypos.find(hypo);
UTIL_THROW_IF2(iterFindHypo == m_hypos.end(), UTIL_THROW_IF2(iterFindHypo == m_hypos.end(),
"Adding a hypothesis should have returned a valid iterator"); "Adding a hypothesis should have returned a valid iterator");
Remove(iterFindHypo); Remove(iterFindHypo);
} }

View File

@ -204,7 +204,7 @@ void ChartParser::CreateInputPaths(const InputType &input)
m_inputPathMatrix.resize(size); m_inputPathMatrix.resize(size);
UTIL_THROW_IF2(input.GetType() != SentenceInput && input.GetType() != TreeInputType, UTIL_THROW_IF2(input.GetType() != SentenceInput && input.GetType() != TreeInputType,
"Input must be a sentence or a tree, not lattice or confusion networks"); "Input must be a sentence or a tree, not lattice or confusion networks");
for (size_t phaseSize = 1; phaseSize <= size; ++phaseSize) { for (size_t phaseSize = 1; phaseSize <= size; ++phaseSize) {
for (size_t startPos = 0; startPos < size - phaseSize + 1; ++startPos) { for (size_t startPos = 0; startPos < size - phaseSize + 1; ++startPos) {
size_t endPos = startPos + phaseSize -1; size_t endPos = startPos + phaseSize -1;
@ -238,7 +238,7 @@ const InputPath &ChartParser::GetInputPath(size_t startPos, size_t endPos) const
{ {
size_t offset = endPos - startPos; size_t offset = endPos - startPos;
UTIL_THROW_IF2(offset >= m_inputPathMatrix[startPos].size(), UTIL_THROW_IF2(offset >= m_inputPathMatrix[startPos].size(),
"Out of bound: " << offset); "Out of bound: " << offset);
return *m_inputPathMatrix[startPos][offset]; return *m_inputPathMatrix[startPos][offset];
} }
@ -246,7 +246,7 @@ InputPath &ChartParser::GetInputPath(size_t startPos, size_t endPos)
{ {
size_t offset = endPos - startPos; size_t offset = endPos - startPos;
UTIL_THROW_IF2(offset >= m_inputPathMatrix[startPos].size(), UTIL_THROW_IF2(offset >= m_inputPathMatrix[startPos].size(),
"Out of bound: " << offset); "Out of bound: " << offset);
return *m_inputPathMatrix[startPos][offset]; return *m_inputPathMatrix[startPos][offset];
} }
/* /*

View File

@ -102,10 +102,10 @@ void ChartTranslationOptionList::Add(const TargetPhraseCollection &tpc,
// Prune if bursting // Prune if bursting
if (m_size == m_ruleLimit * 2) { if (m_size == m_ruleLimit * 2) {
NTH_ELEMENT4(m_collection.begin(), NTH_ELEMENT4(m_collection.begin(),
m_collection.begin() + m_ruleLimit - 1, m_collection.begin() + m_ruleLimit - 1,
m_collection.begin() + m_size, m_collection.begin() + m_size,
ChartTranslationOptionOrderer()); ChartTranslationOptionOrderer());
m_scoreThreshold = m_collection[m_ruleLimit-1]->GetEstimateOfBestScore(); m_scoreThreshold = m_collection[m_ruleLimit-1]->GetEstimateOfBestScore();
m_size = m_ruleLimit; m_size = m_ruleLimit;
} }
@ -129,9 +129,9 @@ void ChartTranslationOptionList::ApplyThreshold()
// Reduce the list to the best m_ruleLimit options. The remaining // Reduce the list to the best m_ruleLimit options. The remaining
// options can be overwritten on subsequent calls to Add(). // options can be overwritten on subsequent calls to Add().
NTH_ELEMENT4(m_collection.begin(), NTH_ELEMENT4(m_collection.begin(),
m_collection.begin()+m_ruleLimit, m_collection.begin()+m_ruleLimit,
m_collection.begin()+m_size, m_collection.begin()+m_size,
ChartTranslationOptionOrderer()); ChartTranslationOptionOrderer());
m_size = m_ruleLimit; m_size = m_ruleLimit;
} }
@ -167,11 +167,11 @@ void ChartTranslationOptionList::Evaluate(const InputType &input, const InputPat
std::ostream& operator<<(std::ostream &out, const ChartTranslationOptionList &obj) std::ostream& operator<<(std::ostream &out, const ChartTranslationOptionList &obj)
{ {
for (size_t i = 0; i < obj.m_collection.size(); ++i) { for (size_t i = 0; i < obj.m_collection.size(); ++i) {
const ChartTranslationOptions &transOpts = *obj.m_collection[i]; const ChartTranslationOptions &transOpts = *obj.m_collection[i];
out << transOpts << endl; out << transOpts << endl;
} }
return out; return out;
} }
} }

View File

@ -137,12 +137,12 @@ void ChartTranslationOptions::CreateSourceRuleFromInputPath()
std::ostream& operator<<(std::ostream &out, const ChartTranslationOptions &obj) std::ostream& operator<<(std::ostream &out, const ChartTranslationOptions &obj)
{ {
for (size_t i = 0; i < obj.m_collection.size(); ++i) { for (size_t i = 0; i < obj.m_collection.size(); ++i) {
const ChartTranslationOption &transOpt = *obj.m_collection[i]; const ChartTranslationOption &transOpt = *obj.m_collection[i];
out << transOpt << endl; out << transOpt << endl;
} }
return out; return out;
} }
} }

View File

@ -59,8 +59,9 @@ public:
static float CalcEstimateOfBestScore(const TargetPhraseCollection &, static float CalcEstimateOfBestScore(const TargetPhraseCollection &,
const StackVec &); const StackVec &);
size_t GetSize() const size_t GetSize() const {
{ return m_collection.size(); } return m_collection.size();
}
//! @todo dunno //! @todo dunno
const StackVec &GetStackVec() const { const StackVec &GetStackVec() const {

View File

@ -45,7 +45,7 @@ public:
const Column& GetColumn(size_t i) const { const Column& GetColumn(size_t i) const {
UTIL_THROW_IF2(i >= data.size(), UTIL_THROW_IF2(i >= data.size(),
"Out of bounds. Trying to access " << i << " when vector only contains " << data.size()); "Out of bounds. Trying to access " << i << " when vector only contains " << data.size());
return data[i]; return data[i];
} }
const Column& operator[](size_t i) const { const Column& operator[](size_t i) const {

View File

@ -33,7 +33,8 @@ DecodeGraph::~DecodeGraph()
} }
//! Add another decode step to the graph //! Add another decode step to the graph
void DecodeGraph::Add(DecodeStep *decodeStep) { void DecodeGraph::Add(DecodeStep *decodeStep)
{
m_steps.push_back(decodeStep); m_steps.push_back(decodeStep);
decodeStep->SetContainer(this); decodeStep->SetContainer(this);
} }

View File

@ -49,8 +49,8 @@ public:
DecodeGraph(size_t position) DecodeGraph(size_t position)
: m_position(position) : m_position(position)
, m_maxChartSpan(NOT_FOUND) , m_maxChartSpan(NOT_FOUND)
, m_backoff(0) , m_backoff(0)
{} {}
// for chart decoding // for chart decoding
DecodeGraph(size_t position, size_t maxChartSpan) DecodeGraph(size_t position, size_t maxChartSpan)
@ -78,7 +78,7 @@ public:
} }
size_t GetMaxChartSpan() const { size_t GetMaxChartSpan() const {
UTIL_THROW_IF2(m_maxChartSpan == NOT_FOUND, "Max chart span not specified"); UTIL_THROW_IF2(m_maxChartSpan == NOT_FOUND, "Max chart span not specified");
return m_maxChartSpan; return m_maxChartSpan;
} }
@ -86,7 +86,7 @@ public:
return m_backoff; return m_backoff;
} }
void SetBackoff(size_t backoff){ void SetBackoff(size_t backoff) {
m_backoff = backoff; m_backoff = backoff;
} }

View File

@ -105,10 +105,12 @@ public:
void RemoveFeature(const FeatureFunction *ff); void RemoveFeature(const FeatureFunction *ff);
void SetContainer(const DecodeGraph *container) void SetContainer(const DecodeGraph *container) {
{ m_container = container; } m_container = container;
const DecodeGraph *GetContainer() const }
{ return m_container; } const DecodeGraph *GetContainer() const {
return m_container;
}
}; };

View File

@ -85,7 +85,7 @@ void DecodeStepTranslation::Process(const TranslationOption &inputPartialTranslO
outPhrase.Merge(targetPhrase, m_newOutputFactors); outPhrase.Merge(targetPhrase, m_newOutputFactors);
outPhrase.Evaluate(inputPath.GetPhrase(), m_featuresToApply); // need to do this as all non-transcores would be screwed up outPhrase.Evaluate(inputPath.GetPhrase(), m_featuresToApply); // need to do this as all non-transcores would be screwed up
cerr << "DecodeStepTranslation::Process is calling outPhrase.Evaluate(inputPath.GetPhrase(), m_featuresToApply)" << endl; cerr << "DecodeStepTranslation::Process is calling outPhrase.Evaluate(inputPath.GetPhrase(), m_featuresToApply)" << endl;
TranslationOption *newTransOpt = new TranslationOption(sourceWordsRange, outPhrase); TranslationOption *newTransOpt = new TranslationOption(sourceWordsRange, outPhrase);
assert(newTransOpt != NULL); assert(newTransOpt != NULL);
@ -199,11 +199,11 @@ const InputPath &DecodeStepTranslation::GetInputPathLEGACY(
const Word *wordIP = NULL; const Word *wordIP = NULL;
for (size_t i = 0; i < phraseFromIP.GetSize(); ++i) { for (size_t i = 0; i < phraseFromIP.GetSize(); ++i) {
const Word &tempWord = phraseFromIP.GetWord(i); const Word &tempWord = phraseFromIP.GetWord(i);
if (!tempWord.IsEpsilon()) { if (!tempWord.IsEpsilon()) {
wordIP = &tempWord; wordIP = &tempWord;
break; break;
} }
} }
// const WordsRange &range = inputPath.GetWordsRange(); // const WordsRange &range = inputPath.GetWordsRange();
@ -238,7 +238,7 @@ void DecodeStepTranslation::ProcessLEGACY(const TranslationOption &inputPartialT
const size_t tableLimit = phraseDictionary->GetTableLimit(); const size_t tableLimit = phraseDictionary->GetTableLimit();
const TargetPhraseCollectionWithSourcePhrase *phraseColl const TargetPhraseCollectionWithSourcePhrase *phraseColl
= phraseDictionary->GetTargetPhraseCollectionLEGACY(toc->GetSource(),sourceWordsRange); = phraseDictionary->GetTargetPhraseCollectionLEGACY(toc->GetSource(),sourceWordsRange);
if (phraseColl != NULL) { if (phraseColl != NULL) {

View File

@ -62,7 +62,7 @@ class BleuScoreFeature : public StatefulFeatureFunction
{ {
public: public:
static const std::vector<BleuScoreFeature*>& GetColl() { static const std::vector<BleuScoreFeature*>& GetColl() {
return s_staticColl; return s_staticColl;
} }
typedef boost::unordered_map<size_t, RefValue > RefCounts; typedef boost::unordered_map<size_t, RefValue > RefCounts;

View File

@ -72,7 +72,7 @@ void ConstrainedDecoding::Load()
std::vector<float> ConstrainedDecoding::DefaultWeights() const std::vector<float> ConstrainedDecoding::DefaultWeights() const
{ {
UTIL_THROW_IF2(m_numScoreComponents != 1, UTIL_THROW_IF2(m_numScoreComponents != 1,
"ConstrainedDecoding must only have 1 score"); "ConstrainedDecoding must only have 1 score");
vector<float> ret(1, 1); vector<float> ret(1, 1);
return ret; return ret;
} }
@ -112,18 +112,16 @@ FFState* ConstrainedDecoding::Evaluate(
float score; float score;
if (hypo.IsSourceCompleted()) { if (hypo.IsSourceCompleted()) {
// translated entire sentence. // translated entire sentence.
bool match = (searchPos == 0) && (ref->GetSize() == outputPhrase.GetSize()); bool match = (searchPos == 0) && (ref->GetSize() == outputPhrase.GetSize());
if (!m_negate) { if (!m_negate) {
score = match ? 0 : - std::numeric_limits<float>::infinity(); score = match ? 0 : - std::numeric_limits<float>::infinity();
} } else {
else { score = !match ? 0 : - std::numeric_limits<float>::infinity();
score = !match ? 0 : - std::numeric_limits<float>::infinity(); }
}
} else if (m_negate) { } else if (m_negate) {
// keep all derivations // keep all derivations
score = 0; score = 0;
} } else {
else {
score = (searchPos != NOT_FOUND) ? 0 : - std::numeric_limits<float>::infinity(); score = (searchPos != NOT_FOUND) ? 0 : - std::numeric_limits<float>::infinity();
} }
@ -151,17 +149,16 @@ FFState* ConstrainedDecoding::EvaluateChart(
if (hypo.GetCurrSourceRange().GetStartPos() == 0 && if (hypo.GetCurrSourceRange().GetStartPos() == 0 &&
hypo.GetCurrSourceRange().GetEndPos() == source.GetSize() - 1) { hypo.GetCurrSourceRange().GetEndPos() == source.GetSize() - 1) {
// translated entire sentence. // translated entire sentence.
bool match = (searchPos == 0) && (ref->GetSize() == outputPhrase.GetSize()); bool match = (searchPos == 0) && (ref->GetSize() == outputPhrase.GetSize());
if (!m_negate) { if (!m_negate) {
score = match ? 0 : - std::numeric_limits<float>::infinity(); score = match ? 0 : - std::numeric_limits<float>::infinity();
} } else {
else { score = !match ? 0 : - std::numeric_limits<float>::infinity();
score = !match ? 0 : - std::numeric_limits<float>::infinity(); }
}
} else if (m_negate) { } else if (m_negate) {
// keep all derivations // keep all derivations
score = 0; score = 0;
} else { } else {
score = (searchPos != NOT_FOUND) ? 0 : - std::numeric_limits<float>::infinity(); score = (searchPos != NOT_FOUND) ? 0 : - std::numeric_limits<float>::infinity();
} }
@ -178,7 +175,7 @@ void ConstrainedDecoding::SetParameter(const std::string& key, const std::string
} else if (key == "max-unknowns") { } else if (key == "max-unknowns") {
m_maxUnknowns = Scan<int>(value); m_maxUnknowns = Scan<int>(value);
} else if (key == "negate") { } else if (key == "negate") {
m_negate = Scan<bool>(value); m_negate = Scan<bool>(value);
} else { } else {
StatefulFeatureFunction::SetParameter(key, value); StatefulFeatureFunction::SetParameter(key, value);
} }

View File

@ -50,7 +50,7 @@ int ControlRecombinationState::Compare(const FFState& other) const
std::vector<float> ControlRecombination::DefaultWeights() const std::vector<float> ControlRecombination::DefaultWeights() const
{ {
UTIL_THROW_IF2(m_numScoreComponents, UTIL_THROW_IF2(m_numScoreComponents,
"ControlRecombination should not have any scores"); "ControlRecombination should not have any scores");
vector<float> ret(0); vector<float> ret(0);
return ret; return ret;
} }

View File

@ -22,43 +22,43 @@ int CoveredReferenceState::Compare(const FFState& other) const
const CoveredReferenceState &otherState = static_cast<const CoveredReferenceState&>(other); const CoveredReferenceState &otherState = static_cast<const CoveredReferenceState&>(other);
if (m_coveredRef.size() != otherState.m_coveredRef.size()) { if (m_coveredRef.size() != otherState.m_coveredRef.size()) {
return (m_coveredRef.size() < otherState.m_coveredRef.size()) ? -1 : +1; return (m_coveredRef.size() < otherState.m_coveredRef.size()) ? -1 : +1;
} else { } else {
multiset<string>::const_iterator thisIt, otherIt; multiset<string>::const_iterator thisIt, otherIt;
for (thisIt = m_coveredRef.begin(), otherIt = otherState.m_coveredRef.begin(); for (thisIt = m_coveredRef.begin(), otherIt = otherState.m_coveredRef.begin();
thisIt != m_coveredRef.end(); thisIt != m_coveredRef.end();
thisIt++, otherIt++) { thisIt++, otherIt++) {
if (*thisIt != *otherIt) return thisIt->compare(*otherIt); if (*thisIt != *otherIt) return thisIt->compare(*otherIt);
} }
} }
return 0; return 0;
// return m_coveredRef == otherState.m_coveredRef; // return m_coveredRef == otherState.m_coveredRef;
// if (m_coveredRef == otherState.m_coveredRef) // if (m_coveredRef == otherState.m_coveredRef)
// return 0; // return 0;
// return (m_coveredRef.size() < otherState.m_coveredRef.size()) ? -1 : +1; // return (m_coveredRef.size() < otherState.m_coveredRef.size()) ? -1 : +1;
} }
void CoveredReferenceFeature::Evaluate(const Phrase &source void CoveredReferenceFeature::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase , const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown , ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const , ScoreComponentCollection &estimatedFutureScore) const
{} {}
void CoveredReferenceFeature::Evaluate(const InputType &input void CoveredReferenceFeature::Evaluate(const InputType &input
, const InputPath &inputPath , const InputPath &inputPath
, const TargetPhrase &targetPhrase , const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown , ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore) const , ScoreComponentCollection *estimatedFutureScore) const
{ {
long id = input.GetTranslationId(); long id = input.GetTranslationId();
boost::unordered_map<long, std::multiset<string> >::const_iterator refIt = m_refs.find(id); boost::unordered_map<long, std::multiset<string> >::const_iterator refIt = m_refs.find(id);
multiset<string> wordsInPhrase = GetWordsInPhrase(targetPhrase); multiset<string> wordsInPhrase = GetWordsInPhrase(targetPhrase);
multiset<string> covered; multiset<string> covered;
set_intersection(wordsInPhrase.begin(), wordsInPhrase.end(), set_intersection(wordsInPhrase.begin(), wordsInPhrase.end(),
refIt->second.begin(), refIt->second.end(), refIt->second.begin(), refIt->second.end(),
inserter(covered, covered.begin())); inserter(covered, covered.begin()));
vector<float> scores; vector<float> scores;
scores.push_back(covered.size()); scores.push_back(covered.size());
@ -66,7 +66,8 @@ void CoveredReferenceFeature::Evaluate(const InputType &input
estimatedFutureScore->Assign(this, scores); estimatedFutureScore->Assign(this, scores);
} }
void CoveredReferenceFeature::Load() { void CoveredReferenceFeature::Load()
{
InputFileStream refFile(m_path); InputFileStream refFile(m_path);
std::string line; std::string line;
const StaticData &staticData = StaticData::Instance(); const StaticData &staticData = StaticData::Instance();
@ -75,7 +76,7 @@ void CoveredReferenceFeature::Load() {
vector<string> words = Tokenize(line, " "); vector<string> words = Tokenize(line, " ");
multiset<string> wordSet; multiset<string> wordSet;
// TODO make Tokenize work with other containers than vector // TODO make Tokenize work with other containers than vector
copy(words.begin(), words.end(), inserter(wordSet, wordSet.begin())); copy(words.begin(), words.end(), inserter(wordSet, wordSet.begin()));
m_refs.insert(make_pair(sentenceID++, wordSet)); m_refs.insert(make_pair(sentenceID++, wordSet));
} }
} }
@ -106,15 +107,15 @@ FFState* CoveredReferenceFeature::Evaluate(
boost::unordered_map<long, std::multiset<string> >::const_iterator refIt = m_refs.find(id); boost::unordered_map<long, std::multiset<string> >::const_iterator refIt = m_refs.find(id);
if (refIt == m_refs.end()) UTIL_THROW(util::Exception, "Sentence id out of range: " + SPrint<long>(id)); if (refIt == m_refs.end()) UTIL_THROW(util::Exception, "Sentence id out of range: " + SPrint<long>(id));
set_difference(refIt->second.begin(), refIt->second.end(), set_difference(refIt->second.begin(), refIt->second.end(),
ret->m_coveredRef.begin(), ret->m_coveredRef.end(), ret->m_coveredRef.begin(), ret->m_coveredRef.end(),
inserter(remaining, remaining.begin())); inserter(remaining, remaining.begin()));
// which of the remaining words are present in the current phrase // which of the remaining words are present in the current phrase
multiset<string> wordsInPhrase = GetWordsInPhrase(cur_hypo.GetCurrTargetPhrase()); multiset<string> wordsInPhrase = GetWordsInPhrase(cur_hypo.GetCurrTargetPhrase());
multiset<string> newCovered; multiset<string> newCovered;
set_intersection(wordsInPhrase.begin(), wordsInPhrase.end(), set_intersection(wordsInPhrase.begin(), wordsInPhrase.end(),
remaining.begin(), remaining.end(), remaining.begin(), remaining.end(),
inserter(newCovered, newCovered.begin())); inserter(newCovered, newCovered.begin()));
vector<float> estimateScore = vector<float> estimateScore =
cur_hypo.GetCurrTargetPhrase().GetScoreBreakdown().GetScoresForProducer(this); cur_hypo.GetCurrTargetPhrase().GetScoreBreakdown().GetScoresForProducer(this);

View File

@ -37,8 +37,7 @@ class CoveredReferenceFeature : public StatefulFeatureFunction
public: public:
CoveredReferenceFeature(const std::string &line) CoveredReferenceFeature(const std::string &line)
:StatefulFeatureFunction(1, line) :StatefulFeatureFunction(1, line) {
{
m_tuneable = true; m_tuneable = true;
ReadParameters(); ReadParameters();
} }

View File

@ -36,7 +36,7 @@ DecodeFeature::DecodeFeature(const std::string &line)
} }
DecodeFeature::DecodeFeature(size_t numScoreComponents DecodeFeature::DecodeFeature(size_t numScoreComponents
, const std::string &line) , const std::string &line)
: StatelessFeatureFunction(numScoreComponents, line) : StatelessFeatureFunction(numScoreComponents, line)
{ {
VERBOSE(2,"DecodeFeature: no factors yet" << std::endl); VERBOSE(2,"DecodeFeature: no factors yet" << std::endl);

View File

@ -42,12 +42,12 @@ public:
DecodeFeature(const std::string &line); DecodeFeature(const std::string &line);
DecodeFeature(size_t numScoreComponents DecodeFeature(size_t numScoreComponents
, const std::string &line); , const std::string &line);
DecodeFeature(size_t numScoreComponents DecodeFeature(size_t numScoreComponents
, const std::vector<FactorType> &input , const std::vector<FactorType> &input
, const std::vector<FactorType> &output , const std::vector<FactorType> &output
, const std::string &line); , const std::string &line);
//! returns output factor types as specified by the ini file //! returns output factor types as specified by the ini file
const FactorMask& GetOutputFactorMask() const; const FactorMask& GetOutputFactorMask() const;
@ -79,8 +79,9 @@ public:
, ScoreComponentCollection &estimatedFutureScore) const , ScoreComponentCollection &estimatedFutureScore) const
{} {}
void SetContainer(const DecodeStep *container) void SetContainer(const DecodeStep *container) {
{ m_container = container; } m_container = container;
}
protected: protected:
std::vector<FactorType> m_input; std::vector<FactorType> m_input;

View File

@ -49,12 +49,9 @@ void DynamicCacheBasedLanguageModel::SetPreComputedScores()
float DynamicCacheBasedLanguageModel::GetPreComputedScores(const unsigned int age) float DynamicCacheBasedLanguageModel::GetPreComputedScores(const unsigned int age)
{ {
if (age < precomputedScores.size()) if (age < precomputedScores.size()) {
{
return precomputedScores.at(age); return precomputedScores.at(age);
} } else {
else
{
return precomputedScores.at(m_maxAge); return precomputedScores.at(m_maxAge);
} }
} }
@ -117,8 +114,7 @@ float DynamicCacheBasedLanguageModel::Evaluate_Whole_String(const TargetPhrase&
if (it != m_cache.end()) { //found! if (it != m_cache.end()) { //found!
score = ((*it).second).second; score = ((*it).second).second;
VERBOSE(3,"cblm::Evaluate_Whole_String: found w:|" << w << "| actual score:|" << ((*it).second).second << "| score:|" << score << "|" << std::endl); VERBOSE(3,"cblm::Evaluate_Whole_String: found w:|" << w << "| actual score:|" << ((*it).second).second << "| score:|" << score << "|" << std::endl);
} } else {
else{
score = m_lower_score; score = m_lower_score;
} }
@ -134,7 +130,7 @@ float DynamicCacheBasedLanguageModel::Evaluate_All_Substrings(const TargetPhrase
decaying_cache_t::const_iterator it; decaying_cache_t::const_iterator it;
float score = 0.0; float score = 0.0;
for (size_t startpos = 0 ; startpos < tp.GetSize() ; ++startpos) { for (size_t startpos = 0 ; startpos < tp.GetSize() ; ++startpos) {
std::string w = ""; std::string w = "";
for (size_t endpos = startpos; endpos < tp.GetSize() ; ++endpos) { for (size_t endpos = startpos; endpos < tp.GetSize() ; ++endpos) {
@ -144,9 +140,8 @@ float DynamicCacheBasedLanguageModel::Evaluate_All_Substrings(const TargetPhrase
if (it != m_cache.end()) { //found! if (it != m_cache.end()) { //found!
score += ((*it).second).second; score += ((*it).second).second;
VERBOSE(3,"cblm::Evaluate_All_Substrings: found w:|" << w << "| actual score:|" << ((*it).second).second << "| score:|" << score << "|" << std::endl); VERBOSE(3,"cblm::Evaluate_All_Substrings: found w:|" << w << "| actual score:|" << ((*it).second).second << "| score:|" << score << "|" << std::endl);
} } else {
else{ score += m_lower_score;
score += m_lower_score;
} }
if (endpos == startpos) { if (endpos == startpos) {

View File

@ -113,16 +113,16 @@ public:
{} {}
void Evaluate(const Hypothesis& hypo, void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const ScoreComponentCollection* accumulator) const
{} {}
void EvaluateChart(const ChartHypothesis &hypo, void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const ScoreComponentCollection* accumulator) const
{} {}
void Evaluate(const InputType &input void Evaluate(const InputType &input
, const InputPath &inputPath , const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const , ScoreComponentCollection &scoreBreakdown) const
{} {}
void SetQueryType(size_t type); void SetQueryType(size_t type);

View File

@ -236,12 +236,12 @@ void FeatureRegistry::Construct(const std::string &name, const std::string &line
void FeatureRegistry::PrintFF() const void FeatureRegistry::PrintFF() const
{ {
std::cerr << "Available feature functions:" << std::endl; std::cerr << "Available feature functions:" << std::endl;
Map::const_iterator iter; Map::const_iterator iter;
for (iter = registry_.begin(); iter != registry_.end(); ++iter) { for (iter = registry_.begin(); iter != registry_.end(); ++iter) {
const string &ffName = iter->first; const string &ffName = iter->first;
std::cerr << ffName << std::endl; std::cerr << ffName << std::endl;
} }
} }

View File

@ -75,7 +75,7 @@ void FeatureFunction::ParseLine(const std::string &line)
for (size_t i = 1; i < toks.size(); ++i) { for (size_t i = 1; i < toks.size(); ++i) {
vector<string> args = TokenizeFirstOnly(toks[i], "="); vector<string> args = TokenizeFirstOnly(toks[i], "=");
UTIL_THROW_IF2(args.size() != 2, UTIL_THROW_IF2(args.size() != 2,
"Incorrect format for feature function arg: " << toks[i]); "Incorrect format for feature function arg: " << toks[i]);
pair<set<string>::iterator,bool> ret = keys.insert(args[0]); pair<set<string>::iterator,bool> ret = keys.insert(args[0]);
UTIL_THROW_IF2(!ret.second, "Duplicate key in line " << line); UTIL_THROW_IF2(!ret.second, "Duplicate key in line " << line);

View File

@ -14,7 +14,7 @@ GlobalLexicalModelUnlimited::GlobalLexicalModelUnlimited(const std::string &line
:StatelessFeatureFunction(0, line) :StatelessFeatureFunction(0, line)
{ {
UTIL_THROW(util::Exception, UTIL_THROW(util::Exception,
"GlobalLexicalModelUnlimited hasn't been refactored for new feature function framework yet"); // TODO need to update arguments to key=value "GlobalLexicalModelUnlimited hasn't been refactored for new feature function framework yet"); // TODO need to update arguments to key=value
const vector<string> modelSpec = Tokenize(line); const vector<string> modelSpec = Tokenize(line);

View File

@ -6,36 +6,36 @@ using namespace std;
namespace Moses namespace Moses
{ {
void InternalStructStatelessFF::Evaluate(const Phrase &source void InternalStructStatelessFF::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase , const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown , ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const , ScoreComponentCollection &estimatedFutureScore) const
{ {
// cerr << "MARIA!!!" << endl; // cerr << "MARIA!!!" << endl;
scoreBreakdown.PlusEquals(this, 0); scoreBreakdown.PlusEquals(this, 0);
} }
void InternalStructStatelessFF::Evaluate(const InputType &input void InternalStructStatelessFF::Evaluate(const InputType &input
, const InputPath &inputPath , const InputPath &inputPath
, const TargetPhrase &targetPhrase , const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown , ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore) const , ScoreComponentCollection *estimatedFutureScore) const
{ {
cerr << "HHHHH" << scoreBreakdown << endl; cerr << "HHHHH" << scoreBreakdown << endl;
scoreBreakdown.PlusEquals(this, 66); scoreBreakdown.PlusEquals(this, 66);
/* FactorList f_mask; /* FactorList f_mask;
f_mask.push_back(0); f_mask.push_back(0);
//if(inputPath.GetPhrase().GetStringRep(f_mask).) //if(inputPath.GetPhrase().GetStringRep(f_mask).)
int score =50; int score =50;
for(size_t i=0;i<inputPath.GetPhrase().GetSize();i++){ for(size_t i=0;i<inputPath.GetPhrase().GetSize();i++){
if(inputPath.GetPhrase(). GetFactor(i,0)->GetString().as_string()=="ist"){ if(inputPath.GetPhrase(). GetFactor(i,0)->GetString().as_string()=="ist"){
//cout<<inputPath.GetPhrase().GetStringRep(f_mask); //cout<<inputPath.GetPhrase().GetStringRep(f_mask);
score+=1; score+=1;
} }
} }
scoreBreakdown.PlusEquals(this, score); scoreBreakdown.PlusEquals(this, score);
*/ */
} }
} }

View File

@ -9,29 +9,30 @@ namespace Moses
class InternalStructStatelessFF : public StatelessFeatureFunction class InternalStructStatelessFF : public StatelessFeatureFunction
{ {
public: public:
InternalStructStatelessFF(const std::string &line) InternalStructStatelessFF(const std::string &line)
:StatelessFeatureFunction(line) :StatelessFeatureFunction(line)
{} {}
bool IsUseable(const FactorMask &mask) const bool IsUseable(const FactorMask &mask) const {
{ return true; } return true;
}
void Evaluate(const Phrase &source void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase , const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown , ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const; , ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input void Evaluate(const InputType &input
, const InputPath &inputPath , const InputPath &inputPath
, const TargetPhrase &targetPhrase , const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown , ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const; , ScoreComponentCollection *estimatedFutureScore = NULL) const;
virtual void Evaluate(const Hypothesis& hypo, virtual void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const ScoreComponentCollection* accumulator) const
{} {}
void EvaluateChart(const ChartHypothesis &hypo, void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const ScoreComponentCollection* accumulator) const
{} {}
}; };

View File

@ -13,9 +13,9 @@ PhrasePenalty::PhrasePenalty(const std::string &line)
} }
void PhrasePenalty::Evaluate(const Phrase &source void PhrasePenalty::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase , const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown , ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const , ScoreComponentCollection &estimatedFutureScore) const
{ {
scoreBreakdown.Assign(this, - 1.0f); scoreBreakdown.Assign(this, - 1.0f);
} }

View File

@ -61,7 +61,7 @@ void TargetNgramFeature::SetParameter(const std::string& key, const std::string&
} else if (key == "lower-ngrams") { } else if (key == "lower-ngrams") {
m_lower_ngrams = Scan<bool>(value); m_lower_ngrams = Scan<bool>(value);
} else if (key == "file") { } else if (key == "file") {
m_file = value; m_file = value;
} else { } else {
StatefulFeatureFunction::SetParameter(key, value); StatefulFeatureFunction::SetParameter(key, value);
} }
@ -74,7 +74,7 @@ void TargetNgramFeature::Load()
if (m_file == "*") return; //allow all if (m_file == "*") return; //allow all
ifstream inFile(m_file.c_str()); ifstream inFile(m_file.c_str());
if (!inFile) { if (!inFile) {
UTIL_THROW(util::Exception, "Couldn't open file" << m_file); UTIL_THROW(util::Exception, "Couldn't open file" << m_file);
} }
std::string line; std::string line;

View File

@ -45,10 +45,10 @@ public:
{} {}
/* /*
virtual void Evaluate(const InputType &source virtual void Evaluate(const InputType &source
, ScoreComponentCollection &scoreBreakdown) const; , ScoreComponentCollection &scoreBreakdown) const;
*/ */
}; };
} }

View File

@ -38,7 +38,7 @@ static const OFF_T InvalidOffT=-1;
template<typename T> inline size_t fWrite(FILE* f,const T& t) template<typename T> inline size_t fWrite(FILE* f,const T& t)
{ {
if(fwrite(&t,sizeof(t),1,f)!=1) { if(fwrite(&t,sizeof(t),1,f)!=1) {
UTIL_THROW2("ERROR:: fwrite!"); UTIL_THROW2("ERROR:: fwrite!");
} }
return sizeof(t); return sizeof(t);
} }
@ -46,7 +46,7 @@ template<typename T> inline size_t fWrite(FILE* f,const T& t)
template<typename T> inline void fRead(FILE* f,T& t) template<typename T> inline void fRead(FILE* f,T& t)
{ {
if(fread(&t,sizeof(t),1,f)!=1) { if(fread(&t,sizeof(t),1,f)!=1) {
UTIL_THROW2("ERROR: fread!"); UTIL_THROW2("ERROR: fread!");
} }
} }
@ -55,7 +55,7 @@ template<typename T> inline size_t fWrite(FILE* f,const T* b,const T* e)
UINT32 s=std::distance(b,e); UINT32 s=std::distance(b,e);
size_t rv=fWrite(f,s); size_t rv=fWrite(f,s);
if(fwrite(b,sizeof(T),s,f)!=s) { if(fwrite(b,sizeof(T),s,f)!=s) {
UTIL_THROW2("ERROR: fwrite!"); UTIL_THROW2("ERROR: fwrite!");
} }
return rv+sizeof(T)*s; return rv+sizeof(T)*s;
} }
@ -65,7 +65,7 @@ template<typename T> inline size_t fWrite(FILE* f,const T b,const T e)
UINT32 s=std::distance(b,e); UINT32 s=std::distance(b,e);
size_t rv=fWrite(f,s); size_t rv=fWrite(f,s);
if(fwrite(&(*b),sizeof(T),s,f)!=s) { if(fwrite(&(*b),sizeof(T),s,f)!=s) {
UTIL_THROW2("ERROR: fwrite!"); UTIL_THROW2("ERROR: fwrite!");
} }
return rv+sizeof(T)*s; return rv+sizeof(T)*s;
} }
@ -75,7 +75,7 @@ template<typename C> inline size_t fWriteVector(FILE* f,const C& v)
UINT32 s=v.size(); UINT32 s=v.size();
size_t rv=fWrite(f,s); size_t rv=fWrite(f,s);
if(fwrite(&v[0],sizeof(typename C::value_type),s,f)!=s) { if(fwrite(&v[0],sizeof(typename C::value_type),s,f)!=s) {
UTIL_THROW2("ERROR: fwrite!"); UTIL_THROW2("ERROR: fwrite!");
} }
return rv+sizeof(typename C::value_type)*s; return rv+sizeof(typename C::value_type)*s;
} }
@ -87,7 +87,7 @@ template<typename C> inline void fReadVector(FILE* f, C& v)
v.resize(s); v.resize(s);
size_t r=fread(&(*v.begin()),sizeof(typename C::value_type),s,f); size_t r=fread(&(*v.begin()),sizeof(typename C::value_type),s,f);
if(r!=s) { if(r!=s) {
UTIL_THROW2("ERROR: freadVec! "<<r<<" "<<s); UTIL_THROW2("ERROR: freadVec! "<<r<<" "<<s);
} }
} }
@ -106,7 +106,7 @@ inline void fReadString(FILE* f,std::string& e)
fRead(f,s); fRead(f,s);
char* a=new char[s+1]; char* a=new char[s+1];
if(fread(a,sizeof(char),s,f)!=s) { if(fread(a,sizeof(char),s,f)!=s) {
UTIL_THROW2("ERROR: fread!"); UTIL_THROW2("ERROR: fread!");
} }
a[s]='\0'; a[s]='\0';
e.assign(a); e.assign(a);
@ -142,8 +142,8 @@ inline OFF_T fTell(FILE* f)
inline void fSeek(FILE* f,OFF_T o) inline void fSeek(FILE* f,OFF_T o)
{ {
if(FSEEKO(f,o,SEEK_SET)<0) { if(FSEEKO(f,o,SEEK_SET)<0) {
std::stringstream strme; std::stringstream strme;
strme << "ERROR: could not fseeko position " << o <<"\n"; strme << "ERROR: could not fseeko position " << o <<"\n";
if(o==InvalidOffT) strme << "You tried to seek for 'InvalidOffT'!\n"; if(o==InvalidOffT) strme << "You tried to seek for 'InvalidOffT'!\n";
UTIL_THROW2(strme.str()); UTIL_THROW2(strme.str());
} }

View File

@ -55,7 +55,7 @@ protected:
public: public:
static const std::vector<GenerationDictionary*>& GetColl() { static const std::vector<GenerationDictionary*>& GetColl() {
return s_staticColl; return s_staticColl;
} }
GenerationDictionary(const std::string &line); GenerationDictionary(const std::string &line);

View File

@ -340,10 +340,10 @@ void Hypothesis::CleanupArcList()
if (!distinctNBest && m_arcList->size() > nBestSize * 5) { if (!distinctNBest && m_arcList->size() > nBestSize * 5) {
// prune arc list only if there too many arcs // prune arc list only if there too many arcs
NTH_ELEMENT4(m_arcList->begin() NTH_ELEMENT4(m_arcList->begin()
, m_arcList->begin() + nBestSize - 1 , m_arcList->begin() + nBestSize - 1
, m_arcList->end() , m_arcList->end()
, CompareHypothesisTotalScore()); , CompareHypothesisTotalScore());
// delete bad ones // delete bad ones
ArcList::iterator iter; ArcList::iterator iter;

View File

@ -145,7 +145,7 @@ void HypothesisStackCubePruning::AddInitial(Hypothesis *hypo)
{ {
std::pair<iterator, bool> addRet = Add(hypo); std::pair<iterator, bool> addRet = Add(hypo);
UTIL_THROW_IF2(!addRet.second, UTIL_THROW_IF2(!addRet.second,
"Should have added hypothesis " << *hypo); "Should have added hypothesis " << *hypo);
const WordsBitmap &bitmap = hypo->GetWordsBitmap(); const WordsBitmap &bitmap = hypo->GetWordsBitmap();
m_bitmapAccessor[bitmap] = new BitmapContainer(bitmap, *this); m_bitmapAccessor[bitmap] = new BitmapContainer(bitmap, *this);

View File

@ -157,7 +157,7 @@ template <class Model> void Fill<Model>::AddPhraseOOV(TargetPhrase &phrase, std:
{ {
std::vector<lm::WordIndex> words; std::vector<lm::WordIndex> words;
UTIL_THROW_IF2(phrase.GetSize() > 1, UTIL_THROW_IF2(phrase.GetSize() > 1,
"OOV target phrase should be 0 or 1 word in length"); "OOV target phrase should be 0 or 1 word in length");
if (phrase.GetSize()) if (phrase.GetSize())
words.push_back(Convert(phrase.GetWord(0))); words.push_back(Convert(phrase.GetWord(0)));

View File

@ -87,8 +87,9 @@ public:
, const TargetPhraseCollection *targetPhrases , const TargetPhraseCollection *targetPhrases
, const void *ptNode); , const void *ptNode);
const TargetPhraseCollection *GetTargetPhrases(const PhraseDictionary &phraseDictionary) const; const TargetPhraseCollection *GetTargetPhrases(const PhraseDictionary &phraseDictionary) const;
const TargetPhrases &GetTargetPhrases() const const TargetPhrases &GetTargetPhrases() const {
{ return m_targetPhrases; } return m_targetPhrases;
}
// pointer to internal node in phrase-table. Since this is implementation dependent, this is a void* // pointer to internal node in phrase-table. Since this is implementation dependent, this is a void*
const void *GetPtNode(const PhraseDictionary &phraseDictionary) const; const void *GetPtNode(const PhraseDictionary &phraseDictionary) const;

View File

@ -131,8 +131,8 @@ template <class Model> void BackwardLanguageModel<Model>::CalcScore(const Phrase
lm::ngram::RuleScore<Model> scorer(*m_ngram, discarded_sadly); lm::ngram::RuleScore<Model> scorer(*m_ngram, discarded_sadly);
UTIL_THROW_IF2(m_beginSentenceFactor == phrase.GetWord(0).GetFactor(m_factorType), UTIL_THROW_IF2(m_beginSentenceFactor == phrase.GetWord(0).GetFactor(m_factorType),
"BackwardLanguageModel does not currently support rules that include <s>" "BackwardLanguageModel does not currently support rules that include <s>"
); );
float before_boundary = 0.0f; float before_boundary = 0.0f;
@ -144,8 +144,8 @@ template <class Model> void BackwardLanguageModel<Model>::CalcScore(const Phrase
for (position = lastWord; position >= 0; position-=1) { for (position = lastWord; position >= 0; position-=1) {
const Word &word = phrase.GetWord(position); const Word &word = phrase.GetWord(position);
UTIL_THROW_IF2(word.IsNonTerminal(), UTIL_THROW_IF2(word.IsNonTerminal(),
"BackwardLanguageModel does not currently support rules that include non-terminals " "BackwardLanguageModel does not currently support rules that include non-terminals "
); );
lm::WordIndex index = TranslateID(word); lm::WordIndex index = TranslateID(word);
scorer.Terminal(index); scorer.Terminal(index);
@ -259,8 +259,8 @@ template <class Model> FFState *BackwardLanguageModel<Model>::Evaluate(const Phr
for (int position=std::min( lastWord, ngramBoundary - 1); position >= 0; position-=1) { for (int position=std::min( lastWord, ngramBoundary - 1); position >= 0; position-=1) {
const Word &word = phrase.GetWord(position); const Word &word = phrase.GetWord(position);
UTIL_THROW_IF2(word.IsNonTerminal(), UTIL_THROW_IF2(word.IsNonTerminal(),
"BackwardLanguageModel does not currently support rules that include non-terminals " "BackwardLanguageModel does not currently support rules that include non-terminals "
); );
lm::WordIndex index = TranslateID(word); lm::WordIndex index = TranslateID(word);
scorer.Terminal(index); scorer.Terminal(index);
@ -285,27 +285,27 @@ template <class Model> FFState *BackwardLanguageModel<Model>::Evaluate(const Phr
LanguageModel *ConstructBackwardLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy) LanguageModel *ConstructBackwardLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy)
{ {
lm::ngram::ModelType model_type; lm::ngram::ModelType model_type;
if (lm::ngram::RecognizeBinary(file.c_str(), model_type)) { if (lm::ngram::RecognizeBinary(file.c_str(), model_type)) {
switch(model_type) { switch(model_type) {
case lm::ngram::PROBING: case lm::ngram::PROBING:
return new BackwardLanguageModel<lm::ngram::ProbingModel>(line, file, factorType, lazy); return new BackwardLanguageModel<lm::ngram::ProbingModel>(line, file, factorType, lazy);
case lm::ngram::REST_PROBING: case lm::ngram::REST_PROBING:
return new BackwardLanguageModel<lm::ngram::RestProbingModel>(line, file, factorType, lazy); return new BackwardLanguageModel<lm::ngram::RestProbingModel>(line, file, factorType, lazy);
case lm::ngram::TRIE: case lm::ngram::TRIE:
return new BackwardLanguageModel<lm::ngram::TrieModel>(line, file, factorType, lazy); return new BackwardLanguageModel<lm::ngram::TrieModel>(line, file, factorType, lazy);
case lm::ngram::QUANT_TRIE: case lm::ngram::QUANT_TRIE:
return new BackwardLanguageModel<lm::ngram::QuantTrieModel>(line, file, factorType, lazy); return new BackwardLanguageModel<lm::ngram::QuantTrieModel>(line, file, factorType, lazy);
case lm::ngram::ARRAY_TRIE: case lm::ngram::ARRAY_TRIE:
return new BackwardLanguageModel<lm::ngram::ArrayTrieModel>(line, file, factorType, lazy); return new BackwardLanguageModel<lm::ngram::ArrayTrieModel>(line, file, factorType, lazy);
case lm::ngram::QUANT_ARRAY_TRIE: case lm::ngram::QUANT_ARRAY_TRIE:
return new BackwardLanguageModel<lm::ngram::QuantArrayTrieModel>(line, file, factorType, lazy); return new BackwardLanguageModel<lm::ngram::QuantArrayTrieModel>(line, file, factorType, lazy);
default: default:
UTIL_THROW2("Unrecognized kenlm model type " << model_type); UTIL_THROW2("Unrecognized kenlm model type " << model_type);
}
} else {
return new BackwardLanguageModel<lm::ngram::ProbingModel>(line, file, factorType, lazy);
} }
} else {
return new BackwardLanguageModel<lm::ngram::ProbingModel>(line, file, factorType, lazy);
}
} }
} // namespace Moses } // namespace Moses

View File

@ -15,24 +15,25 @@
using namespace std; using namespace std;
///////////////////////// /////////////////////////
void read_ini(const char *inifile, string &model, string &words, string &wordstxt){ void read_ini(const char *inifile, string &model, string &words, string &wordstxt)
ifstream ifs(inifile); {
string line; ifstream ifs(inifile);
string line;
getline(ifs, line); getline(ifs, line);
while(ifs){ while(ifs) {
unsigned int pos = line.find("="); unsigned int pos = line.find("=");
string key = line.substr(0, pos); string key = line.substr(0, pos);
string value = line.substr(pos+1, line.size()-pos); string value = line.substr(pos+1, line.size()-pos);
if(key=="MODEL"){ if(key=="MODEL") {
model = value; model = value;
}else if(key=="WORDS"){ } else if(key=="WORDS") {
words = value; words = value;
}else if(key=="WORDSTXT"){ } else if(key=="WORDSTXT") {
wordstxt = value; wordstxt = value;
} }
getline(ifs, line); getline(ifs, line);
} }
} }
///////////////////////// /////////////////////////
@ -43,35 +44,35 @@ namespace Moses
class DALMState : public FFState class DALMState : public FFState
{ {
private: private:
DALM::State *state; DALM::State *state;
public: public:
DALMState(unsigned short order){ DALMState(unsigned short order) {
state = new DALM::State(order); state = new DALM::State(order);
} }
DALMState(const DALMState &from){ DALMState(const DALMState &from) {
state = new DALM::State(*from.state); state = new DALM::State(*from.state);
} }
virtual ~DALMState(){ virtual ~DALMState() {
delete state; delete state;
} }
virtual int Compare(const FFState& other) const{ virtual int Compare(const FFState& other) const {
const DALMState &o = static_cast<const DALMState &>(other); const DALMState &o = static_cast<const DALMState &>(other);
if(state->get_count() < o.state->get_count()) return -1; if(state->get_count() < o.state->get_count()) return -1;
else if(state->get_count() > o.state->get_count()) return 1; else if(state->get_count() > o.state->get_count()) return 1;
else return state->compare(o.state); else return state->compare(o.state);
} }
DALM::State *get_state() const{ DALM::State *get_state() const {
return state; return state;
} }
void refresh(){ void refresh() {
state->refresh(); state->refresh();
} }
}; };
LanguageModelDALM::LanguageModelDALM(const std::string &line) LanguageModelDALM::LanguageModelDALM(const std::string &line)
@ -86,62 +87,64 @@ LanguageModelDALM::LanguageModelDALM(const std::string &line)
LanguageModelDALM::~LanguageModelDALM() LanguageModelDALM::~LanguageModelDALM()
{ {
delete m_logger; delete m_logger;
delete m_vocab; delete m_vocab;
delete m_lm; delete m_lm;
} }
void LanguageModelDALM::Load() void LanguageModelDALM::Load()
{ {
///////////////////// /////////////////////
// READING INIFILE // // READING INIFILE //
///////////////////// /////////////////////
string inifile= m_filePath + "/dalm.ini"; string inifile= m_filePath + "/dalm.ini";
string model; // Path to the double-array file. string model; // Path to the double-array file.
string words; // Path to the vocabulary file. string words; // Path to the vocabulary file.
string wordstxt; //Path to the vocabulary file in text format. string wordstxt; //Path to the vocabulary file in text format.
read_ini(inifile.c_str(), model, words, wordstxt); read_ini(inifile.c_str(), model, words, wordstxt);
model = m_filePath + "/" + model; model = m_filePath + "/" + model;
words = m_filePath + "/" + words; words = m_filePath + "/" + words;
wordstxt = m_filePath + "/" + wordstxt; wordstxt = m_filePath + "/" + wordstxt;
UTIL_THROW_IF(model.empty() || words.empty() || wordstxt.empty(), UTIL_THROW_IF(model.empty() || words.empty() || wordstxt.empty(),
util::FileOpenException, util::FileOpenException,
"Failed to read DALM ini file " << m_filePath << ". Probably doesn't exist"); "Failed to read DALM ini file " << m_filePath << ". Probably doesn't exist");
//////////////// ////////////////
// LOADING LM // // LOADING LM //
//////////////// ////////////////
// Preparing a logger object. // Preparing a logger object.
m_logger = new DALM::Logger(stderr); m_logger = new DALM::Logger(stderr);
m_logger->setLevel(DALM::LOGGER_INFO); m_logger->setLevel(DALM::LOGGER_INFO);
// Load the vocabulary file. // Load the vocabulary file.
m_vocab = new DALM::Vocabulary(words, *m_logger); m_vocab = new DALM::Vocabulary(words, *m_logger);
// Load the language model. // Load the language model.
m_lm = new DALM::LM(model, *m_vocab, *m_logger); m_lm = new DALM::LM(model, *m_vocab, *m_logger);
wid_start = m_vocab->lookup(BOS_);
wid_end = m_vocab->lookup(EOS_);
// vocab mapping wid_start = m_vocab->lookup(BOS_);
CreateVocabMapping(wordstxt); wid_end = m_vocab->lookup(EOS_);
FactorCollection &collection = FactorCollection::Instance(); // vocab mapping
m_beginSentenceFactor = collection.AddFactor(BOS_); CreateVocabMapping(wordstxt);
FactorCollection &collection = FactorCollection::Instance();
m_beginSentenceFactor = collection.AddFactor(BOS_);
} }
const FFState *LanguageModelDALM::EmptyHypothesisState(const InputType &/*input*/) const{ const FFState *LanguageModelDALM::EmptyHypothesisState(const InputType &/*input*/) const
DALMState *s = new DALMState(m_nGramOrder); {
m_lm->init_state(*s->get_state()); DALMState *s = new DALMState(m_nGramOrder);
return s; m_lm->init_state(*s->get_state());
return s;
} }
void LanguageModelDALM::CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const{ void LanguageModelDALM::CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const
{
fullScore = 0; fullScore = 0;
ngramScore = 0; ngramScore = 0;
@ -149,12 +152,12 @@ void LanguageModelDALM::CalcScore(const Phrase &phrase, float &fullScore, float
size_t phraseSize = phrase.GetSize(); size_t phraseSize = phrase.GetSize();
if (!phraseSize) return; if (!phraseSize) return;
DALMState *dalm_state = new DALMState(m_nGramOrder); DALMState *dalm_state = new DALMState(m_nGramOrder);
size_t currPos = 0; size_t currPos = 0;
size_t hist_count = 0; size_t hist_count = 0;
while (currPos < phraseSize) { while (currPos < phraseSize) {
const Word &word = phrase.GetWord(currPos); const Word &word = phrase.GetWord(currPos);
hist_count++; hist_count++;
@ -169,7 +172,7 @@ void LanguageModelDALM::CalcScore(const Phrase &phrase, float &fullScore, float
if (currPos != 0) { if (currPos != 0) {
UTIL_THROW2("Either your data contains <s> in a position other than the first word or your language model is missing <s>. Did you build your ARPA using IRSTLM and forget to run add-start-end.sh?"); UTIL_THROW2("Either your data contains <s> in a position other than the first word or your language model is missing <s>. Did you build your ARPA using IRSTLM and forget to run add-start-end.sh?");
} }
m_lm->init_state(*dalm_state->get_state()); m_lm->init_state(*dalm_state->get_state());
} else { } else {
LMResult result = GetValue(word, dalm_state->get_state()); LMResult result = GetValue(word, dalm_state->get_state());
fullScore += result.score; fullScore += result.score;
@ -180,10 +183,11 @@ void LanguageModelDALM::CalcScore(const Phrase &phrase, float &fullScore, float
currPos++; currPos++;
} }
delete dalm_state; delete dalm_state;
} }
LMResult LanguageModelDALM::GetValue(DALM::VocabId wid, DALM::State* finalState) const{ LMResult LanguageModelDALM::GetValue(DALM::VocabId wid, DALM::State* finalState) const
{
LMResult ret; LMResult ret;
// last word is unk? // last word is unk?
@ -200,41 +204,42 @@ LMResult LanguageModelDALM::GetValue(DALM::VocabId wid, DALM::State* finalState)
LMResult LanguageModelDALM::GetValue(const Word &word, DALM::State* finalState) const LMResult LanguageModelDALM::GetValue(const Word &word, DALM::State* finalState) const
{ {
DALM::VocabId wid = GetVocabId(word.GetFactor(m_factorType)); DALM::VocabId wid = GetVocabId(word.GetFactor(m_factorType));
return GetValue(wid, finalState); return GetValue(wid, finalState);
} }
FFState *LanguageModelDALM::Evaluate(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const{ FFState *LanguageModelDALM::Evaluate(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const
{
// In this function, we only compute the LM scores of n-grams that overlap a // In this function, we only compute the LM scores of n-grams that overlap a
// phrase boundary. Phrase-internal scores are taken directly from the // phrase boundary. Phrase-internal scores are taken directly from the
// translation option. // translation option.
const DALMState *dalm_ps = static_cast<const DALMState *>(ps); const DALMState *dalm_ps = static_cast<const DALMState *>(ps);
// Empty phrase added? nothing to be done // Empty phrase added? nothing to be done
if (hypo.GetCurrTargetLength() == 0){ if (hypo.GetCurrTargetLength() == 0) {
return dalm_ps ? new DALMState(*dalm_ps) : NULL; return dalm_ps ? new DALMState(*dalm_ps) : NULL;
} }
const std::size_t begin = hypo.GetCurrTargetWordsRange().GetStartPos(); const std::size_t begin = hypo.GetCurrTargetWordsRange().GetStartPos();
//[begin, end) in STL-like fashion. //[begin, end) in STL-like fashion.
const std::size_t end = hypo.GetCurrTargetWordsRange().GetEndPos() + 1; const std::size_t end = hypo.GetCurrTargetWordsRange().GetEndPos() + 1;
const std::size_t adjust_end = std::min(end, begin + m_nGramOrder - 1); const std::size_t adjust_end = std::min(end, begin + m_nGramOrder - 1);
DALMState *dalm_state = new DALMState(*dalm_ps); DALMState *dalm_state = new DALMState(*dalm_ps);
std::size_t position = begin; std::size_t position = begin;
float score = 0.0; float score = 0.0;
for(; position < adjust_end; position++){ for(; position < adjust_end; position++) {
score += GetValue(hypo.GetWord(position), dalm_state->get_state()).score; score += GetValue(hypo.GetWord(position), dalm_state->get_state()).score;
} }
if (hypo.IsSourceCompleted()) { if (hypo.IsSourceCompleted()) {
// Score end of sentence. // Score end of sentence.
std::vector<DALM::VocabId> indices(m_nGramOrder-1); std::vector<DALM::VocabId> indices(m_nGramOrder-1);
const DALM::VocabId *last = LastIDs(hypo, &indices.front()); const DALM::VocabId *last = LastIDs(hypo, &indices.front());
m_lm->set_state(&indices.front(), (last-&indices.front()), *dalm_state->get_state()); m_lm->set_state(&indices.front(), (last-&indices.front()), *dalm_state->get_state());
float s = GetValue(wid_end, dalm_state->get_state()).score; float s = GetValue(wid_end, dalm_state->get_state()).score;
score += s; score += s;
} else if (adjust_end < end) { } else if (adjust_end < end) {
@ -252,14 +257,15 @@ FFState *LanguageModelDALM::Evaluate(const Hypothesis &hypo, const FFState *ps,
} else { } else {
out->PlusEquals(this, score); out->PlusEquals(this, score);
} }
return dalm_state; return dalm_state;
} }
FFState *LanguageModelDALM::EvaluateChart(const ChartHypothesis& hypo, int featureID, ScoreComponentCollection *out) const{ FFState *LanguageModelDALM::EvaluateChart(const ChartHypothesis& hypo, int featureID, ScoreComponentCollection *out) const
{
LanguageModelChartState *ret = new LanguageModelChartState(hypo, featureID, m_nGramOrder); LanguageModelChartState *ret = new LanguageModelChartState(hypo, featureID, m_nGramOrder);
// initialize language model context state // initialize language model context state
DALMState *dalm_state = new DALMState(m_nGramOrder); DALMState *dalm_state = new DALMState(m_nGramOrder);
// initial language model scores // initial language model scores
float prefixScore = 0.0; // not yet final for initial words (lack context) float prefixScore = 0.0; // not yet final for initial words (lack context)
@ -280,9 +286,9 @@ FFState *LanguageModelDALM::EvaluateChart(const ChartHypothesis& hypo, int featu
if (!word.IsNonTerminal()) { if (!word.IsNonTerminal()) {
// beginning of sentence symbol <s>? -> just update state // beginning of sentence symbol <s>? -> just update state
if (word.GetFactor(m_factorType) == m_beginSentenceFactor) { if (word.GetFactor(m_factorType) == m_beginSentenceFactor) {
UTIL_THROW_IF2(phrasePos != 0, UTIL_THROW_IF2(phrasePos != 0,
"Sentence start symbol must be at the beginning of sentence"); "Sentence start symbol must be at the beginning of sentence");
m_lm->init_state(*dalm_state->get_state()); m_lm->init_state(*dalm_state->get_state());
} }
// score a regular word added by the rule // score a regular word added by the rule
else { else {
@ -298,7 +304,7 @@ FFState *LanguageModelDALM::EvaluateChart(const ChartHypothesis& hypo, int featu
const LanguageModelChartState* prevState = const LanguageModelChartState* prevState =
static_cast<const LanguageModelChartState*>(prevHypo->GetFFState(featureID)); static_cast<const LanguageModelChartState*>(prevHypo->GetFFState(featureID));
size_t subPhraseLength = prevState->GetNumTargetTerminals(); size_t subPhraseLength = prevState->GetNumTargetTerminals();
// special case: rule starts with non-terminal -> copy everything // special case: rule starts with non-terminal -> copy everything
if (phrasePos == 0) { if (phrasePos == 0) {
@ -310,13 +316,13 @@ FFState *LanguageModelDALM::EvaluateChart(const ChartHypothesis& hypo, int featu
// get language model state // get language model state
delete dalm_state; delete dalm_state;
dalm_state = new DALMState( *static_cast<DALMState*>(prevState->GetRightContext()) ); dalm_state = new DALMState( *static_cast<DALMState*>(prevState->GetRightContext()) );
wordPos += subPhraseLength; wordPos += subPhraseLength;
} }
// internal non-terminal // internal non-terminal
else { else {
// score its prefix // score its prefix
size_t wpos = wordPos; size_t wpos = wordPos;
for(size_t prefixPos = 0; for(size_t prefixPos = 0;
prefixPos < m_nGramOrder-1 // up to LM order window prefixPos < m_nGramOrder-1 // up to LM order window
&& prefixPos < subPhraseLength; // up to length && prefixPos < subPhraseLength; // up to length
@ -324,7 +330,7 @@ FFState *LanguageModelDALM::EvaluateChart(const ChartHypothesis& hypo, int featu
const Word &word = prevState->GetPrefix().GetWord(prefixPos); const Word &word = prevState->GetPrefix().GetWord(prefixPos);
updateChartScore( &prefixScore, &finalizedScore, GetValue(word, dalm_state->get_state()).score, ++wpos ); updateChartScore( &prefixScore, &finalizedScore, GetValue(word, dalm_state->get_state()).score, ++wpos );
} }
wordPos += subPhraseLength; wordPos += subPhraseLength;
// check if we are dealing with a large sub-phrase // check if we are dealing with a large sub-phrase
if (subPhraseLength > m_nGramOrder - 1) { if (subPhraseLength > m_nGramOrder - 1) {
@ -360,36 +366,35 @@ void LanguageModelDALM::CreateVocabMapping(const std::string &wordstxt)
string line; string line;
while(getline(vocabStrm, line)) { while(getline(vocabStrm, line)) {
const Factor *factor = FactorCollection::Instance().AddFactor(line); const Factor *factor = FactorCollection::Instance().AddFactor(line);
DALM::VocabId wid = m_vocab->lookup(line.c_str()); DALM::VocabId wid = m_vocab->lookup(line.c_str());
VocabMap::value_type entry(factor, wid); VocabMap::value_type entry(factor, wid);
m_vocabMap.insert(entry); m_vocabMap.insert(entry);
} }
} }
DALM::VocabId LanguageModelDALM::GetVocabId(const Factor *factor) const DALM::VocabId LanguageModelDALM::GetVocabId(const Factor *factor) const
{ {
VocabMap::left_map::const_iterator iter; VocabMap::left_map::const_iterator iter;
iter = m_vocabMap.left.find(factor); iter = m_vocabMap.left.find(factor);
if (iter != m_vocabMap.left.end()) { if (iter != m_vocabMap.left.end()) {
return iter->second; return iter->second;
} } else {
else { // not in mapping. Must be UNK
// not in mapping. Must be UNK return m_vocab->unk();
return m_vocab->unk(); }
}
} }
void LanguageModelDALM::SetParameter(const std::string& key, const std::string& value) void LanguageModelDALM::SetParameter(const std::string& key, const std::string& value)
{ {
if (key == "factor") { if (key == "factor") {
m_factorType = Scan<FactorType>(value); m_factorType = Scan<FactorType>(value);
} else if (key == "order") { } else if (key == "order") {
m_nGramOrder = Scan<size_t>(value); m_nGramOrder = Scan<size_t>(value);
} else if (key == "path") { } else if (key == "path") {
m_filePath = value; m_filePath = value;
} else { } else {
LanguageModel::SetParameter(key, value); LanguageModel::SetParameter(key, value);
} }

View File

@ -25,7 +25,7 @@ class LanguageModelDALM : public LanguageModel
public: public:
LanguageModelDALM(const std::string &line); LanguageModelDALM(const std::string &line);
virtual ~LanguageModelDALM(); virtual ~LanguageModelDALM();
void Load(); void Load();
virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const; virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const;
@ -38,8 +38,8 @@ public:
virtual bool IsUseable(const FactorMask &mask) const; virtual bool IsUseable(const FactorMask &mask) const;
virtual void SetParameter(const std::string& key, const std::string& value); virtual void SetParameter(const std::string& key, const std::string& value);
protected: protected:
const Factor *m_beginSentenceFactor; const Factor *m_beginSentenceFactor;
@ -48,22 +48,22 @@ protected:
std::string m_filePath; std::string m_filePath;
size_t m_nGramOrder; //! max n-gram length contained in this LM size_t m_nGramOrder; //! max n-gram length contained in this LM
DALM::Logger *m_logger; DALM::Logger *m_logger;
DALM::Vocabulary *m_vocab; DALM::Vocabulary *m_vocab;
DALM::LM *m_lm; DALM::LM *m_lm;
DALM::VocabId wid_start, wid_end; DALM::VocabId wid_start, wid_end;
typedef boost::bimap<const Factor *, DALM::VocabId> VocabMap; typedef boost::bimap<const Factor *, DALM::VocabId> VocabMap;
mutable VocabMap m_vocabMap; mutable VocabMap m_vocabMap;
void CreateVocabMapping(const std::string &wordstxt); void CreateVocabMapping(const std::string &wordstxt);
DALM::VocabId GetVocabId(const Factor *factor) const; DALM::VocabId GetVocabId(const Factor *factor) const;
private: private:
LMResult GetValue(DALM::VocabId wid, DALM::State* finalState) const; LMResult GetValue(DALM::VocabId wid, DALM::State* finalState) const;
LMResult GetValue(const Word &word, DALM::State* finalState) const; LMResult GetValue(const Word &word, DALM::State* finalState) const;
void updateChartScore(float *prefixScore, float *finalizedScore, float score, size_t wordPos) const; void updateChartScore(float *prefixScore, float *finalizedScore, float score, size_t wordPos) const;
// Convert last words of hypothesis into vocab ids, returning an end pointer. // Convert last words of hypothesis into vocab ids, returning an end pointer.
DALM::VocabId *LastIDs(const Hypothesis &hypo, DALM::VocabId *indices) const { DALM::VocabId *LastIDs(const Hypothesis &hypo, DALM::VocabId *indices) const {
DALM::VocabId *index = indices; DALM::VocabId *index = indices;

View File

@ -114,7 +114,7 @@ void LanguageModelImplementation::CalcScore(const Phrase &phrase, float &fullSco
} else { } else {
ShiftOrPush(contextFactor, word); ShiftOrPush(contextFactor, word);
UTIL_THROW_IF2(contextFactor.size() > GetNGramOrder(), UTIL_THROW_IF2(contextFactor.size() > GetNGramOrder(),
"Can only calculate LM score of phrases up to the n-gram order"); "Can only calculate LM score of phrases up to the n-gram order");
if (word == GetSentenceStartWord()) { if (word == GetSentenceStartWord()) {
// do nothing, don't include prob for <s> unigram // do nothing, don't include prob for <s> unigram
@ -253,8 +253,8 @@ FFState* LanguageModelImplementation::EvaluateChart(const ChartHypothesis& hypo,
// beginning of sentence symbol <s>? -> just update state // beginning of sentence symbol <s>? -> just update state
if (word == GetSentenceStartWord()) { if (word == GetSentenceStartWord()) {
UTIL_THROW_IF2(phrasePos != 0, UTIL_THROW_IF2(phrasePos != 0,
"Sentence start symbol must be at the beginning of sentence"); "Sentence start symbol must be at the beginning of sentence");
delete lmState; delete lmState;
lmState = NewState( GetBeginSentenceState() ); lmState = NewState( GetBeginSentenceState() );
} }

View File

@ -396,7 +396,7 @@ LanguageModel *ConstructKenLM(const std::string &line)
for (size_t i = 1; i < toks.size(); ++i) { for (size_t i = 1; i < toks.size(); ++i) {
vector<string> args = Tokenize(toks[i], "="); vector<string> args = Tokenize(toks[i], "=");
UTIL_THROW_IF2(args.size() != 2, UTIL_THROW_IF2(args.size() != 2,
"Incorrect format of KenLM property: " << toks[i]); "Incorrect format of KenLM property: " << toks[i]);
if (args[0] == "factor") { if (args[0] == "factor") {
factorType = Scan<FactorType>(args[1]); factorType = Scan<FactorType>(args[1]);
@ -416,28 +416,28 @@ LanguageModel *ConstructKenLM(const std::string &line)
LanguageModel *ConstructKenLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy) LanguageModel *ConstructKenLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy)
{ {
lm::ngram::ModelType model_type; lm::ngram::ModelType model_type;
if (lm::ngram::RecognizeBinary(file.c_str(), model_type)) { if (lm::ngram::RecognizeBinary(file.c_str(), model_type)) {
switch(model_type) { switch(model_type) {
case lm::ngram::PROBING: case lm::ngram::PROBING:
return new LanguageModelKen<lm::ngram::ProbingModel>(line, file, factorType, lazy);
case lm::ngram::REST_PROBING:
return new LanguageModelKen<lm::ngram::RestProbingModel>(line, file, factorType, lazy);
case lm::ngram::TRIE:
return new LanguageModelKen<lm::ngram::TrieModel>(line, file, factorType, lazy);
case lm::ngram::QUANT_TRIE:
return new LanguageModelKen<lm::ngram::QuantTrieModel>(line, file, factorType, lazy);
case lm::ngram::ARRAY_TRIE:
return new LanguageModelKen<lm::ngram::ArrayTrieModel>(line, file, factorType, lazy);
case lm::ngram::QUANT_ARRAY_TRIE:
return new LanguageModelKen<lm::ngram::QuantArrayTrieModel>(line, file, factorType, lazy);
default:
UTIL_THROW2("Unrecognized kenlm model type " << model_type);
}
} else {
return new LanguageModelKen<lm::ngram::ProbingModel>(line, file, factorType, lazy); return new LanguageModelKen<lm::ngram::ProbingModel>(line, file, factorType, lazy);
case lm::ngram::REST_PROBING:
return new LanguageModelKen<lm::ngram::RestProbingModel>(line, file, factorType, lazy);
case lm::ngram::TRIE:
return new LanguageModelKen<lm::ngram::TrieModel>(line, file, factorType, lazy);
case lm::ngram::QUANT_TRIE:
return new LanguageModelKen<lm::ngram::QuantTrieModel>(line, file, factorType, lazy);
case lm::ngram::ARRAY_TRIE:
return new LanguageModelKen<lm::ngram::ArrayTrieModel>(line, file, factorType, lazy);
case lm::ngram::QUANT_ARRAY_TRIE:
return new LanguageModelKen<lm::ngram::QuantArrayTrieModel>(line, file, factorType, lazy);
default:
UTIL_THROW2("Unrecognized kenlm model type " << model_type);
} }
} else {
return new LanguageModelKen<lm::ngram::ProbingModel>(line, file, factorType, lazy);
}
} }
} }

View File

@ -162,7 +162,7 @@ LMResult LanguageModelMaxEntSRI::GetValue(const vector<const Word*> &contextFact
ngram[count] = Vocab_None; ngram[count] = Vocab_None;
UTIL_THROW_IF2((*contextFactor[count-1])[factorType] == NULL, UTIL_THROW_IF2((*contextFactor[count-1])[factorType] == NULL,
"No factor " << factorType << " at position " << (count-1)); "No factor " << factorType << " at position " << (count-1));
// call sri lm fn // call sri lm fn
VocabIndex lmId = GetLmID((*contextFactor[count-1])[factorType]); VocabIndex lmId = GetLmID((*contextFactor[count-1])[factorType]);
ret = GetValue(lmId, ngram+1); ret = GetValue(lmId, ngram+1);

View File

@ -10,7 +10,7 @@ using namespace std;
namespace Moses namespace Moses
{ {
NeuralLMWrapper::NeuralLMWrapper(const std::string &line) NeuralLMWrapper::NeuralLMWrapper(const std::string &line)
:LanguageModelSingleFactor(line) :LanguageModelSingleFactor(line)
{ {
// This space intentionally left blank // This space intentionally left blank
} }

View File

@ -2,8 +2,9 @@
#include "SingleFactor.h" #include "SingleFactor.h"
namespace nplm { namespace nplm
class neuralLM; {
class neuralLM;
} }
namespace Moses namespace Moses

View File

@ -162,7 +162,7 @@ LMResult LanguageModelSRI::GetValue(const vector<const Word*> &contextFactor, St
ngram[count] = Vocab_None; ngram[count] = Vocab_None;
UTIL_THROW_IF2((*contextFactor[count-1])[factorType] == NULL, UTIL_THROW_IF2((*contextFactor[count-1])[factorType] == NULL,
"No factor " << factorType << " at position " << (count-1)); "No factor " << factorType << " at position " << (count-1));
// call sri lm fn // call sri lm fn
VocabIndex lmId = GetLmID((*contextFactor[count-1])[factorType]); VocabIndex lmId = GetLmID((*contextFactor[count-1])[factorType]);
ret = GetValue(lmId, ngram+1); ret = GetValue(lmId, ngram+1);

View File

@ -48,8 +48,8 @@ LanguageModelSingleFactor::LanguageModelSingleFactor(const std::string &line)
LanguageModelSingleFactor::~LanguageModelSingleFactor() LanguageModelSingleFactor::~LanguageModelSingleFactor()
{ {
delete m_nullContextState; delete m_nullContextState;
delete m_beginSentenceState; delete m_beginSentenceState;
} }
const FFState *LanguageModelSingleFactor::GetNullContextState() const const FFState *LanguageModelSingleFactor::GetNullContextState() const

View File

@ -44,7 +44,7 @@ public:
return p.first->second; return p.first->second;
} }
Key const& symbol(LabelId i) const { Key const& symbol(LabelId i) const {
assert(static_cast<size_t>(i)<data.size()); assert(static_cast<size_t>(i)<data.size());
return data[i]; return data[i];
} }

View File

@ -113,7 +113,7 @@ void Manager::ProcessSentence()
searchTime.start(); searchTime.start();
m_search->ProcessSentence(); m_search->ProcessSentence();
VERBOSE(1, "Line " << m_lineNumber << ": Search took " << searchTime << " seconds" << endl); VERBOSE(1, "Line " << m_lineNumber << ": Search took " << searchTime << " seconds" << endl);
IFVERBOSE(2) { IFVERBOSE(2) {
GetSentenceStats().StopTimeTotal(); GetSentenceStats().StopTimeTotal();
TRACE_ERR(GetSentenceStats()); TRACE_ERR(GetSentenceStats());
} }
@ -321,12 +321,12 @@ void Manager::CalcLatticeSamples(size_t count, TrellisPathList &ret) const
if (i->forward >= 0) { if (i->forward >= 0) {
map<int,const Hypothesis*>::const_iterator idToHypIter = idToHyp.find(i->forward); map<int,const Hypothesis*>::const_iterator idToHypIter = idToHyp.find(i->forward);
UTIL_THROW_IF2(idToHypIter == idToHyp.end(), UTIL_THROW_IF2(idToHypIter == idToHyp.end(),
"Couldn't find hypothesis " << i->forward); "Couldn't find hypothesis " << i->forward);
const Hypothesis* nextHypo = idToHypIter->second; const Hypothesis* nextHypo = idToHypIter->second;
outgoingHyps[hypo].insert(nextHypo); outgoingHyps[hypo].insert(nextHypo);
map<int,float>::const_iterator fscoreIter = fscores.find(nextHypo->GetId()); map<int,float>::const_iterator fscoreIter = fscores.find(nextHypo->GetId());
UTIL_THROW_IF2(fscoreIter == fscores.end(), UTIL_THROW_IF2(fscoreIter == fscores.end(),
"Couldn't find scores for hypothsis " << nextHypo->GetId()); "Couldn't find scores for hypothsis " << nextHypo->GetId());
edgeScores[Edge(hypo->GetId(),nextHypo->GetId())] = edgeScores[Edge(hypo->GetId(),nextHypo->GetId())] =
i->fscore - fscoreIter->second; i->fscore - fscoreIter->second;
} }
@ -344,17 +344,17 @@ void Manager::CalcLatticeSamples(size_t count, TrellisPathList &ret) const
outgoingHyps.find(i->hypo); outgoingHyps.find(i->hypo);
UTIL_THROW_IF2(outIter == outgoingHyps.end(), UTIL_THROW_IF2(outIter == outgoingHyps.end(),
"Couldn't find hypothesis " << i->hypo->GetId()); "Couldn't find hypothesis " << i->hypo->GetId());
float sigma = 0; float sigma = 0;
for (set<const Hypothesis*>::const_iterator j = outIter->second.begin(); for (set<const Hypothesis*>::const_iterator j = outIter->second.begin();
j != outIter->second.end(); ++j) { j != outIter->second.end(); ++j) {
map<const Hypothesis*, float>::const_iterator succIter = sigmas.find(*j); map<const Hypothesis*, float>::const_iterator succIter = sigmas.find(*j);
UTIL_THROW_IF2(succIter == sigmas.end(), UTIL_THROW_IF2(succIter == sigmas.end(),
"Couldn't find hypothesis " << (*j)->GetId()); "Couldn't find hypothesis " << (*j)->GetId());
map<Edge,float>::const_iterator edgeScoreIter = map<Edge,float>::const_iterator edgeScoreIter =
edgeScores.find(Edge(i->hypo->GetId(),(*j)->GetId())); edgeScores.find(Edge(i->hypo->GetId(),(*j)->GetId()));
UTIL_THROW_IF2(edgeScoreIter == edgeScores.end(), UTIL_THROW_IF2(edgeScoreIter == edgeScores.end(),
"Couldn't find edge for hypothesis " << (*j)->GetId()); "Couldn't find edge for hypothesis " << (*j)->GetId());
float term = edgeScoreIter->second + succIter->second; // Add sigma(*j) float term = edgeScoreIter->second + succIter->second; // Add sigma(*j)
if (sigma == 0) { if (sigma == 0) {
sigma = term; sigma = term;
@ -387,10 +387,10 @@ void Manager::CalcLatticeSamples(size_t count, TrellisPathList &ret) const
j != outIter->second.end(); ++j) { j != outIter->second.end(); ++j) {
candidates.push_back(*j); candidates.push_back(*j);
UTIL_THROW_IF2(sigmas.find(*j) == sigmas.end(), UTIL_THROW_IF2(sigmas.find(*j) == sigmas.end(),
"Hypothesis " << (*j)->GetId() << " not found"); "Hypothesis " << (*j)->GetId() << " not found");
Edge edge(path.back()->GetId(),(*j)->GetId()); Edge edge(path.back()->GetId(),(*j)->GetId());
UTIL_THROW_IF2(edgeScores.find(edge) == edgeScores.end(), UTIL_THROW_IF2(edgeScores.find(edge) == edgeScores.end(),
"Edge not found"); "Edge not found");
candidateScores.push_back(sigmas[*j] + edgeScores[edge]); candidateScores.push_back(sigmas[*j] + edgeScores[edge]);
if (scoreTotal == 0) { if (scoreTotal == 0) {
scoreTotal = candidateScores.back(); scoreTotal = candidateScores.back();
@ -545,13 +545,14 @@ void OutputWordGraph(std::ostream &outputWordGraphStream, const Hypothesis *hypo
outputWordGraphStream << endl; outputWordGraphStream << endl;
} }
void Manager::GetOutputLanguageModelOrder( std::ostream &out, const Hypothesis *hypo ) { void Manager::GetOutputLanguageModelOrder( std::ostream &out, const Hypothesis *hypo )
{
Phrase translation; Phrase translation;
hypo->GetOutputPhrase(translation); hypo->GetOutputPhrase(translation);
const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions(); const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions();
for (size_t i = 0; i < statefulFFs.size(); ++i) { for (size_t i = 0; i < statefulFFs.size(); ++i) {
const StatefulFeatureFunction *ff = statefulFFs[i]; const StatefulFeatureFunction *ff = statefulFFs[i];
if (const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff)) { if (const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff)) {
lm->ReportHistoryOrder(out, translation); lm->ReportHistoryOrder(out, translation);
} }
} }
@ -1345,7 +1346,7 @@ void Manager::SerializeSearchGraphPB(
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) { for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
const Hypothesis *loserHypo = *iterArcList; const Hypothesis *loserHypo = *iterArcList;
UTIL_THROW_IF2(!connected[loserHypo->GetId()], UTIL_THROW_IF2(!connected[loserHypo->GetId()],
"Hypothesis " << loserHypo->GetId() << " is not connected"); "Hypothesis " << loserHypo->GetId() << " is not connected");
Hypergraph_Edge* edge = hg.add_edges(); Hypergraph_Edge* edge = hg.add_edges();
SerializeEdgeInfo(loserHypo, edge); SerializeEdgeInfo(loserHypo, edge);
edge->set_head_node(headNodeIdx); edge->set_head_node(headNodeIdx);

View File

@ -131,7 +131,7 @@ public:
TargetPhraseCollectionWithSourcePhrase const* TargetPhraseCollectionWithSourcePhrase const*
GetTargetPhraseCollection(Phrase const &src) const { GetTargetPhraseCollection(Phrase const &src) const {
assert(m_dict); assert(m_dict);
if(src.GetSize()==0) return 0; if(src.GetSize()==0) return 0;
std::pair<MapSrc2Tgt::iterator,bool> piter; std::pair<MapSrc2Tgt::iterator,bool> piter;
@ -314,7 +314,7 @@ public:
const std::vector<Phrase> &sourcePhrases) const { const std::vector<Phrase> &sourcePhrases) const {
// convert into TargetPhraseCollection // convert into TargetPhraseCollection
UTIL_THROW_IF2(tCands.size() != sourcePhrases.size(), UTIL_THROW_IF2(tCands.size() != sourcePhrases.size(),
"Number of target phrases must equal number of source phrases"); "Number of target phrases must equal number of source phrases");
TargetPhraseCollectionWithSourcePhrase *rv=new TargetPhraseCollectionWithSourcePhrase; TargetPhraseCollectionWithSourcePhrase *rv=new TargetPhraseCollectionWithSourcePhrase;
@ -351,7 +351,7 @@ public:
}; };
void CacheSource(ConfusionNet const& src) { void CacheSource(ConfusionNet const& src) {
assert(m_dict); assert(m_dict);
const size_t srcSize=src.GetSize(); const size_t srcSize=src.GetSize();
std::vector<size_t> exploredPaths(srcSize+1,0); std::vector<size_t> exploredPaths(srcSize+1,0);
@ -414,7 +414,7 @@ public:
//assert that we have the right number of link params in this CN option //assert that we have the right number of link params in this CN option
UTIL_THROW_IF2(currCol[colidx].second.denseScores.size() < m_numInputScores, UTIL_THROW_IF2(currCol[colidx].second.denseScores.size() < m_numInputScores,
"Incorrect number of input scores"); "Incorrect number of input scores");
// do not start with epsilon (except at first position) // do not start with epsilon (except at first position)
if(isEpsilon && curr.begin()==curr.end() && curr.begin()>0) continue; if(isEpsilon && curr.begin()==curr.end() && curr.begin()>0) continue;
@ -473,7 +473,7 @@ public:
//put input scores in first - already logged, just drop in directly //put input scores in first - already logged, just drop in directly
std::vector<float> transcores(m_obj->GetNumScoreComponents()); std::vector<float> transcores(m_obj->GetNumScoreComponents());
UTIL_THROW_IF2(transcores.size() != weightTrans.size(), UTIL_THROW_IF2(transcores.size() != weightTrans.size(),
"Incorrect number of translation scores"); "Incorrect number of translation scores");
//put in phrase table scores, logging as we insert //put in phrase table scores, logging as we insert
std::transform(tcands[i].scores.begin() std::transform(tcands[i].scores.begin()

View File

@ -560,8 +560,8 @@ void Parameter::ConvertWeightArgsPhraseModel(const string &oldWeightName)
vector<float> weights(numFF); vector<float> weights(numFF);
for (size_t currFF = 0; currFF < numFF; ++currFF) { for (size_t currFF = 0; currFF < numFF; ++currFF) {
UTIL_THROW_IF2(currOldInd >= oldWeights.size(), UTIL_THROW_IF2(currOldInd >= oldWeights.size(),
"Errors converting old phrase-table weights to new weights"); "Errors converting old phrase-table weights to new weights");
float weight = Scan<float>(oldWeights[currOldInd]); float weight = Scan<float>(oldWeights[currOldInd]);
weights[currFF] = weight; weights[currFF] = weight;
@ -651,8 +651,8 @@ void Parameter::ConvertWeightArgsDistortion()
vector<float> weights(numFF); vector<float> weights(numFF);
for (size_t currFF = 0; currFF < numFF; ++currFF) { for (size_t currFF = 0; currFF < numFF; ++currFF) {
UTIL_THROW_IF2(currOldInd >= oldWeights.size(), UTIL_THROW_IF2(currOldInd >= oldWeights.size(),
"Errors converting old distortion weights to new weights"); "Errors converting old distortion weights to new weights");
float weight = Scan<float>(oldWeights[currOldInd]); float weight = Scan<float>(oldWeights[currOldInd]);
weights[currFF] = weight; weights[currFF] = weight;
@ -666,8 +666,8 @@ void Parameter::ConvertWeightArgsDistortion()
vector<FactorType> factors = Tokenize<FactorType>(toks[0], "-"); vector<FactorType> factors = Tokenize<FactorType>(toks[0], "-");
UTIL_THROW_IF2(factors.size() != 2, UTIL_THROW_IF2(factors.size() != 2,
"Error in old factor specification for lexicalized reordering model: " "Error in old factor specification for lexicalized reordering model: "
<< toks[0]); << toks[0]);
strme << "input-factor=" << factors[0] strme << "input-factor=" << factors[0]
<< " output-factor=" << factors[1] << " "; << " output-factor=" << factors[1] << " ";
@ -731,7 +731,7 @@ void Parameter::ConvertWeightArgsLM()
newFeatureName = "KENLM"; newFeatureName = "KENLM";
break; break;
default: default:
UTIL_THROW2("Unkown language model type id:" << lmType); UTIL_THROW2("Unkown language model type id:" << lmType);
} }
size_t numFF = 1; size_t numFF = 1;
@ -740,8 +740,8 @@ void Parameter::ConvertWeightArgsLM()
vector<float> weightsLM(numFF); vector<float> weightsLM(numFF);
for (size_t currFF = 0; currFF < numFF; ++currFF) { for (size_t currFF = 0; currFF < numFF; ++currFF) {
UTIL_THROW_IF2(currOldInd >= weights.size(), UTIL_THROW_IF2(currOldInd >= weights.size(),
"Errors converting old LM weights to new weights"); "Errors converting old LM weights to new weights");
weightsLM[currFF] = Scan<float>(weights[currOldInd]); weightsLM[currFF] = Scan<float>(weights[currOldInd]);
if (isChartDecoding) { if (isChartDecoding) {
weightsLM[currFF] = UntransformLMScore(weightsLM[currFF]); weightsLM[currFF] = UntransformLMScore(weightsLM[currFF]);
@ -792,8 +792,8 @@ void Parameter::ConvertWeightArgsGeneration(const std::string &oldWeightName, co
vector<float> weights(numFF); vector<float> weights(numFF);
for (size_t currFF = 0; currFF < numFF; ++currFF) { for (size_t currFF = 0; currFF < numFF; ++currFF) {
UTIL_THROW_IF2(currOldInd >= oldWeights.size(), UTIL_THROW_IF2(currOldInd >= oldWeights.size(),
"Errors converting old generation weights to new weights"); "Errors converting old generation weights to new weights");
float weight = Scan<float>(oldWeights[currOldInd]); float weight = Scan<float>(oldWeights[currOldInd]);
weights[currFF] = weight; weights[currFF] = weight;
@ -853,8 +853,8 @@ void Parameter::ConvertPhrasePenalty()
{ {
string oldWeightName = "weight-p"; string oldWeightName = "weight-p";
if (isParamSpecified(oldWeightName)) { if (isParamSpecified(oldWeightName)) {
UTIL_THROW_IF2(m_setting[oldWeightName].size() != 1, UTIL_THROW_IF2(m_setting[oldWeightName].size() != 1,
"There should be only 1 phrase-penalty weight"); "There should be only 1 phrase-penalty weight");
float weight = Scan<float>(m_setting[oldWeightName][0]); float weight = Scan<float>(m_setting[oldWeightName][0]);
AddFeature("PhrasePenalty"); AddFeature("PhrasePenalty");
SetWeight("PhrasePenalty", 0, weight); SetWeight("PhrasePenalty", 0, weight);
@ -867,7 +867,7 @@ void Parameter::ConvertWeightArgs()
{ {
// can't handle discr LM. must do it manually 'cos of bigram/n-gram split // can't handle discr LM. must do it manually 'cos of bigram/n-gram split
UTIL_THROW_IF2( m_setting.count("weight-dlm") != 0, UTIL_THROW_IF2( m_setting.count("weight-dlm") != 0,
"Can't handle discr LM. must do it manually 'cos of bigram/n-gram split"); "Can't handle discr LM. must do it manually 'cos of bigram/n-gram split");
// check that old & new format aren't mixed // check that old & new format aren't mixed
if (m_setting.count("weight") && if (m_setting.count("weight") &&
@ -912,7 +912,7 @@ void Parameter::CreateWeightsMap()
const string &line = vec[i]; const string &line = vec[i];
vector<string> toks = Tokenize(line); vector<string> toks = Tokenize(line);
UTIL_THROW_IF2(toks.size() < 2, UTIL_THROW_IF2(toks.size() < 2,
"Error in format of weights: " << line); "Error in format of weights: " << line);
string name = toks[0]; string name = toks[0];
name = name.substr(0, name.size() - 1); name = name.substr(0, name.size() - 1);
@ -936,7 +936,7 @@ void Parameter::WeightOverwrite()
// should only be on 1 line // should only be on 1 line
UTIL_THROW_IF2(vec.size() != 1, UTIL_THROW_IF2(vec.size() != 1,
"Weight override should only be on 1 line"); "Weight override should only be on 1 line");
string name(""); string name("");
vector<float> weights; vector<float> weights;
@ -1305,8 +1305,8 @@ void Parameter::OverwriteParam(const string &paramName, PARAM_VEC values)
if (m_setting[paramName].size() > 1) { if (m_setting[paramName].size() > 1) {
VERBOSE(2," (the parameter had " << m_setting[paramName].size() << " previous values)"); VERBOSE(2," (the parameter had " << m_setting[paramName].size() << " previous values)");
UTIL_THROW_IF2(m_setting[paramName].size() != values.size(), UTIL_THROW_IF2(m_setting[paramName].size() != values.size(),
"Number of weight override for " << paramName "Number of weight override for " << paramName
<< " is not the same as the original number of weights"); << " is not the same as the original number of weights");
} else { } else {
VERBOSE(2," (the parameter does not have previous values)"); VERBOSE(2," (the parameter does not have previous values)");
m_setting[paramName].resize(values.size()); m_setting[paramName].resize(values.size());

View File

@ -83,9 +83,9 @@ void PartialTranslOptColl::Prune()
// find nth element // find nth element
NTH_ELEMENT4(m_list.begin(), NTH_ELEMENT4(m_list.begin(),
m_list.begin() + m_maxSize, m_list.begin() + m_maxSize,
m_list.end(), m_list.end(),
ComparePartialTranslationOption); ComparePartialTranslationOption);
m_worstScore = m_list[ m_maxSize-1 ]->GetFutureScore(); m_worstScore = m_list[ m_maxSize-1 ]->GetFutureScore();
// delete the rest // delete the rest

View File

@ -210,8 +210,8 @@ void Phrase::CreateFromString(FactorDirection direction
size_t nextPos = annotatedWord.find('[', 1); size_t nextPos = annotatedWord.find('[', 1);
UTIL_THROW_IF2(nextPos == string::npos, UTIL_THROW_IF2(nextPos == string::npos,
"Incorrect formatting of non-terminal. Should have 2 non-terms, eg. [X][X]. " "Incorrect formatting of non-terminal. Should have 2 non-terms, eg. [X][X]. "
<< "Current string: " << annotatedWord); << "Current string: " << annotatedWord);
if (direction == Input) if (direction == Input)
annotatedWord = annotatedWord.substr(1, nextPos - 2); annotatedWord = annotatedWord.substr(1, nextPos - 2);

View File

@ -155,8 +155,8 @@ public:
} }
void RemoveWord(size_t pos) { void RemoveWord(size_t pos) {
UTIL_THROW_IF2(pos >= m_words.size(), UTIL_THROW_IF2(pos >= m_words.size(),
"Referencing position " << pos << " out of bound"); "Referencing position " << pos << " out of bound");
m_words.erase(m_words.begin() + pos); m_words.erase(m_words.begin() + pos);
} }

View File

@ -148,7 +148,7 @@ void PrefixTreeMap::GetCandidates(const IPhrase& key, Candidates* cands)
return; return;
} }
UTIL_THROW_IF2(m_Data[key[0]]->findKey(key[0]) >= m_Data[key[0]]->size(), UTIL_THROW_IF2(m_Data[key[0]]->findKey(key[0]) >= m_Data[key[0]]->size(),
"Key not found: " << key[0]); "Key not found: " << key[0]);
OFF_T candOffset = m_Data[key[0]]->find(key); OFF_T candOffset = m_Data[key[0]]->find(key);
if(candOffset == InvalidOffT) { if(candOffset == InvalidOffT) {
@ -175,7 +175,7 @@ void PrefixTreeMap::GetCandidates(const PPimp& p, Candidates* cands)
std::vector< std::string const * > PrefixTreeMap::ConvertPhrase(const IPhrase& p, unsigned int voc) const std::vector< std::string const * > PrefixTreeMap::ConvertPhrase(const IPhrase& p, unsigned int voc) const
{ {
UTIL_THROW_IF2(voc >= m_Voc.size() || m_Voc[voc] == 0, UTIL_THROW_IF2(voc >= m_Voc.size() || m_Voc[voc] == 0,
"Invalid vocab id: " << voc); "Invalid vocab id: " << voc);
std::vector< std::string const * > result; std::vector< std::string const * > result;
result.reserve(p.size()); result.reserve(p.size());
for(IPhrase::const_iterator i = p.begin(); i != p.end(); ++i) { for(IPhrase::const_iterator i = p.begin(); i != p.end(); ++i) {
@ -187,7 +187,7 @@ std::vector< std::string const * > PrefixTreeMap::ConvertPhrase(const IPhrase& p
IPhrase PrefixTreeMap::ConvertPhrase(const std::vector< std::string >& p, unsigned int voc) const IPhrase PrefixTreeMap::ConvertPhrase(const std::vector< std::string >& p, unsigned int voc) const
{ {
UTIL_THROW_IF2(voc >= m_Voc.size() || m_Voc[voc] == 0, UTIL_THROW_IF2(voc >= m_Voc.size() || m_Voc[voc] == 0,
"Invalid vocab id: " << voc); "Invalid vocab id: " << voc);
IPhrase result; IPhrase result;
result.reserve(p.size()); result.reserve(p.size());
for(size_t i = 0; i < p.size(); ++i) { for(size_t i = 0; i < p.size(); ++i) {
@ -199,14 +199,14 @@ IPhrase PrefixTreeMap::ConvertPhrase(const std::vector< std::string >& p, unsign
LabelId PrefixTreeMap::ConvertWord(const std::string& w, unsigned int voc) const LabelId PrefixTreeMap::ConvertWord(const std::string& w, unsigned int voc) const
{ {
UTIL_THROW_IF2(voc >= m_Voc.size() || m_Voc[voc] == 0, UTIL_THROW_IF2(voc >= m_Voc.size() || m_Voc[voc] == 0,
"Invalid vocab id: " << voc); "Invalid vocab id: " << voc);
return m_Voc[voc]->index(w); return m_Voc[voc]->index(w);
} }
std::string PrefixTreeMap::ConvertWord(LabelId w, unsigned int voc) const std::string PrefixTreeMap::ConvertWord(LabelId w, unsigned int voc) const
{ {
UTIL_THROW_IF2(voc >= m_Voc.size() || m_Voc[voc] == 0, UTIL_THROW_IF2(voc >= m_Voc.size() || m_Voc[voc] == 0,
"Invalid vocab id: " << voc); "Invalid vocab id: " << voc);
if(w == PrefixTreeMap::MagicWord) { if(w == PrefixTreeMap::MagicWord) {
return "|||"; return "|||";
} else if (w == InvalidLabelId) { } else if (w == InvalidLabelId) {

View File

@ -103,7 +103,7 @@ void RuleCube::CreateNeighbor(const RuleCubeItem &item, int dimensionIndex,
std::ostream& operator<<(std::ostream &out, const RuleCube &obj) std::ostream& operator<<(std::ostream &out, const RuleCube &obj)
{ {
out << obj.GetItemSetSize(); out << obj.GetItemSetSize();
return out; return out;
} }
} }

View File

@ -74,7 +74,7 @@ class RuleCubeItemEqualityPred
public: public:
bool operator()(const RuleCubeItem *p, const RuleCubeItem *q) const { bool operator()(const RuleCubeItem *p, const RuleCubeItem *q) const {
bool ret = p->GetHypothesisDimensions() == q->GetHypothesisDimensions() && bool ret = p->GetHypothesisDimensions() == q->GetHypothesisDimensions() &&
p->GetTranslationDimension() == q->GetTranslationDimension(); p->GetTranslationDimension() == q->GetTranslationDimension();
return ret; return ret;
} }
}; };
@ -92,7 +92,7 @@ public:
~RuleCube(); ~RuleCube();
float GetTopScore() const { float GetTopScore() const {
UTIL_THROW_IF2(m_queue.empty(), "Empty queue, nothing to pop"); UTIL_THROW_IF2(m_queue.empty(), "Empty queue, nothing to pop");
RuleCubeItem *item = m_queue.top(); RuleCubeItem *item = m_queue.top();
return item->GetScore(); return item->GetScore();
} }
@ -107,8 +107,9 @@ public:
return m_transOpt; return m_transOpt;
} }
size_t GetItemSetSize() const size_t GetItemSetSize() const {
{ return m_covered.size(); } return m_covered.size();
}
private: private:
typedef boost::unordered_set<RuleCubeItem*, typedef boost::unordered_set<RuleCubeItem*,

View File

@ -102,9 +102,9 @@ private:
if (indexIter == s_scoreIndexes.end()) { if (indexIter == s_scoreIndexes.end()) {
std::stringstream strme; std::stringstream strme;
strme << "ERROR: FeatureFunction: " << sp->GetScoreProducerDescription() << strme << "ERROR: FeatureFunction: " << sp->GetScoreProducerDescription() <<
" not registered with ScoreIndexMap" << std::endl; " not registered with ScoreIndexMap" << std::endl;
strme << "You must call ScoreComponentCollection.RegisterScoreProducer() " << strme << "You must call ScoreComponentCollection.RegisterScoreProducer() " <<
" for every FeatureFunction" << std::endl; " for every FeatureFunction" << std::endl;
UTIL_THROW2(strme.str()); UTIL_THROW2(strme.str());
} }
return indexIter->second; return indexIter->second;
@ -233,7 +233,7 @@ public:
void PlusEquals(const FeatureFunction* sp, const std::vector<float>& scores) { void PlusEquals(const FeatureFunction* sp, const std::vector<float>& scores) {
IndexPair indexes = GetIndexes(sp); IndexPair indexes = GetIndexes(sp);
UTIL_THROW_IF2(scores.size() != indexes.second - indexes.first, UTIL_THROW_IF2(scores.size() != indexes.second - indexes.first,
"Number of scores is incorrect"); "Number of scores is incorrect");
for (size_t i = 0; i < scores.size(); ++i) { for (size_t i = 0; i < scores.size(); ++i) {
m_scores[i + indexes.first] += scores[i]; m_scores[i + indexes.first] += scores[i];
} }
@ -245,7 +245,7 @@ public:
void PlusEquals(const FeatureFunction* sp, float score) { void PlusEquals(const FeatureFunction* sp, float score) {
IndexPair indexes = GetIndexes(sp); IndexPair indexes = GetIndexes(sp);
UTIL_THROW_IF2(1 != indexes.second - indexes.first, UTIL_THROW_IF2(1 != indexes.second - indexes.first,
"Number of scores is incorrect"); "Number of scores is incorrect");
m_scores[indexes.first] += score; m_scores[indexes.first] += score;
} }
@ -271,7 +271,7 @@ public:
void Assign(const FeatureFunction* sp, float score) { void Assign(const FeatureFunction* sp, float score) {
IndexPair indexes = GetIndexes(sp); IndexPair indexes = GetIndexes(sp);
UTIL_THROW_IF2(1 != indexes.second - indexes.first, UTIL_THROW_IF2(1 != indexes.second - indexes.first,
"Feature function must must only contain 1 score"); "Feature function must must only contain 1 score");
m_scores[indexes.first] = score; m_scores[indexes.first] = score;
} }
@ -302,7 +302,7 @@ public:
float PartialInnerProduct(const FeatureFunction* sp, const std::vector<float>& rhs) const { float PartialInnerProduct(const FeatureFunction* sp, const std::vector<float>& rhs) const {
std::vector<float> lhs = GetScoresForProducer(sp); std::vector<float> lhs = GetScoresForProducer(sp);
UTIL_THROW_IF2(lhs.size() != rhs.size(), UTIL_THROW_IF2(lhs.size() != rhs.size(),
"Number of weights must match number of scores"); "Number of weights must match number of scores");
return std::inner_product(lhs.begin(), lhs.end(), rhs.begin(), 0.0f); return std::inner_product(lhs.begin(), lhs.end(), rhs.begin(), 0.0f);
} }
@ -351,7 +351,7 @@ public:
float GetScoreForProducer(const FeatureFunction* sp) const { float GetScoreForProducer(const FeatureFunction* sp) const {
IndexPair indexes = GetIndexes(sp); IndexPair indexes = GetIndexes(sp);
UTIL_THROW_IF2(indexes.second - indexes.first != 1, UTIL_THROW_IF2(indexes.second - indexes.first != 1,
"Feature function must must only contain 1 score"); "Feature function must must only contain 1 score");
return m_scores[indexes.first]; return m_scores[indexes.first];
} }

View File

@ -30,7 +30,7 @@ Search *Search::CreateSearch(Manager& manager, const InputType &source,
case NormalBatch: case NormalBatch:
return new SearchNormalBatch(manager, source, transOptColl); return new SearchNormalBatch(manager, source, transOptColl);
default: default:
UTIL_THROW2("ERROR: search. Aborting\n"); UTIL_THROW2("ERROR: search. Aborting\n");
return NULL; return NULL;
} }
} }

View File

@ -140,7 +140,7 @@ ExpandHypothesis(const Hypothesis &hypothesis,
} }
m_partial_hypos.push_back(newHypo); m_partial_hypos.push_back(newHypo);
} else { } else {
UTIL_THROW2("can't use early discarding with batch decoding!"); UTIL_THROW2("can't use early discarding with batch decoding!");
} }
} }

View File

@ -135,7 +135,7 @@ int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder)
std::vector< std::map<std::string, std::string> >::iterator dlt_meta_it = dlt_meta.begin(); std::vector< std::map<std::string, std::string> >::iterator dlt_meta_it = dlt_meta.begin();
for (dlt_meta_it = dlt_meta.begin(); dlt_meta_it != dlt_meta.end(); ++dlt_meta_it) { for (dlt_meta_it = dlt_meta.begin(); dlt_meta_it != dlt_meta.end(); ++dlt_meta_it) {
DynamicCacheBasedLanguageModel& cblm = DynamicCacheBasedLanguageModel::InstanceNonConst(); DynamicCacheBasedLanguageModel& cblm = DynamicCacheBasedLanguageModel::InstanceNonConst();
std::cerr << "&cblm:|" << &cblm << "|" << std::endl; std::cerr << "&cblm:|" << &cblm << "|" << std::endl;
PhraseDictionaryDynamicCacheBased& cbtm = PhraseDictionaryDynamicCacheBased::InstanceNonConst(); PhraseDictionaryDynamicCacheBased& cbtm = PhraseDictionaryDynamicCacheBased::InstanceNonConst();
if ((*dlt_meta_it).find("cbtm") != (*dlt_meta_it).end()) { if ((*dlt_meta_it).find("cbtm") != (*dlt_meta_it).end()) {
if (&cbtm) cbtm.Insert((*dlt_meta_it)["cbtm"]); if (&cbtm) cbtm.Insert((*dlt_meta_it)["cbtm"]);
@ -145,7 +145,7 @@ std::cerr << "&cblm:|" << &cblm << "|" << std::endl;
} }
if ((*dlt_meta_it).find("cbtm-file") != (*dlt_meta_it).end()) { if ((*dlt_meta_it).find("cbtm-file") != (*dlt_meta_it).end()) {
if (&cbtm) cbtm.Load((*dlt_meta_it)["cbtm-file"]); if (&cbtm) cbtm.Load((*dlt_meta_it)["cbtm-file"]);
} }
if ((*dlt_meta_it).find("cblm") != (*dlt_meta_it).end()) { if ((*dlt_meta_it).find("cblm") != (*dlt_meta_it).end()) {
if (&cblm) cblm.Insert((*dlt_meta_it)["cblm"]); if (&cblm) cblm.Insert((*dlt_meta_it)["cblm"]);
} }

View File

@ -520,15 +520,14 @@ bool StaticData::LoadData(Parameter *parameter)
string &feature = toks[0]; string &feature = toks[0];
std::map<std::string, std::string>::const_iterator iter = featureNameOverride.find(feature); std::map<std::string, std::string>::const_iterator iter = featureNameOverride.find(feature);
if (iter == featureNameOverride.end()) { if (iter == featureNameOverride.end()) {
// feature name not override // feature name not override
m_registry.Construct(feature, line); m_registry.Construct(feature, line);
} } else {
else { // replace feature name with new name
// replace feature name with new name string newName = iter->second;
string newName = iter->second; feature = newName;
feature = newName; string newLine = Join(" ", toks);
string newLine = Join(" ", toks); m_registry.Construct(newName, newLine);
m_registry.Construct(newName, newLine);
} }
} }
@ -633,7 +632,7 @@ void StaticData::LoadNonTerminals()
while(getline(inStream, line)) { while(getline(inStream, line)) {
vector<string> tokens = Tokenize(line); vector<string> tokens = Tokenize(line);
UTIL_THROW_IF2(tokens.size() != 2, UTIL_THROW_IF2(tokens.size() != 2,
"Incorrect unknown LHS format: " << line); "Incorrect unknown LHS format: " << line);
UnknownLHSEntry entry(tokens[0], Scan<float>(tokens[1])); UnknownLHSEntry entry(tokens[0], Scan<float>(tokens[1]));
m_unknownLHS.push_back(entry); m_unknownLHS.push_back(entry);
} }
@ -682,7 +681,7 @@ bool StaticData::LoadDecodeGraphs()
decodeGraphInd = Scan<size_t>(token[0]); decodeGraphInd = Scan<size_t>(token[0]);
//the vectorList index can only increment by one //the vectorList index can only increment by one
UTIL_THROW_IF2(decodeGraphInd != prevDecodeGraphInd && decodeGraphInd != prevDecodeGraphInd + 1, UTIL_THROW_IF2(decodeGraphInd != prevDecodeGraphInd && decodeGraphInd != prevDecodeGraphInd + 1,
"Malformed mapping"); "Malformed mapping");
if (decodeGraphInd > prevDecodeGraphInd) { if (decodeGraphInd > prevDecodeGraphInd) {
prev = NULL; prev = NULL;
} }
@ -748,11 +747,11 @@ bool StaticData::LoadDecodeGraphs()
// if specified, record maxmimum unseen n-gram size // if specified, record maxmimum unseen n-gram size
const vector<string> &backoffVector = m_parameter->GetParam("decoding-graph-backoff"); const vector<string> &backoffVector = m_parameter->GetParam("decoding-graph-backoff");
for(size_t i=0; i<m_decodeGraphs.size() && i<backoffVector.size(); i++) { for(size_t i=0; i<m_decodeGraphs.size() && i<backoffVector.size(); i++) {
DecodeGraph &decodeGraph = *m_decodeGraphs[i]; DecodeGraph &decodeGraph = *m_decodeGraphs[i];
if (i < backoffVector.size()) { if (i < backoffVector.size()) {
decodeGraph.SetBackoff(Scan<size_t>(backoffVector[i])); decodeGraph.SetBackoff(Scan<size_t>(backoffVector[i]));
} }
} }
return true; return true;
@ -995,7 +994,7 @@ bool StaticData::LoadAlternateWeightSettings()
currentId = args[1]; currentId = args[1];
cerr << "alternate weight setting " << currentId << endl; cerr << "alternate weight setting " << currentId << endl;
UTIL_THROW_IF2(m_weightSetting.find(currentId) != m_weightSetting.end(), UTIL_THROW_IF2(m_weightSetting.find(currentId) != m_weightSetting.end(),
"Duplicate alternate weight id: " << currentId); "Duplicate alternate weight id: " << currentId);
m_weightSetting[ currentId ] = new ScoreComponentCollection; m_weightSetting[ currentId ] = new ScoreComponentCollection;
// other specifications // other specifications
@ -1040,7 +1039,7 @@ bool StaticData::LoadAlternateWeightSettings()
UTIL_THROW_IF2(currentId.empty(), "No alternative weights specified"); UTIL_THROW_IF2(currentId.empty(), "No alternative weights specified");
vector<string> tokens = Tokenize(weightSpecification[i]); vector<string> tokens = Tokenize(weightSpecification[i]);
UTIL_THROW_IF2(tokens.size() < 2 UTIL_THROW_IF2(tokens.size() < 2
, "Incorrect format for alternate weights: " << weightSpecification[i]); , "Incorrect format for alternate weights: " << weightSpecification[i]);
// get name and weight values // get name and weight values
string name = tokens[0]; string name = tokens[0];
@ -1069,36 +1068,36 @@ bool StaticData::LoadAlternateWeightSettings()
void StaticData::NoCache() void StaticData::NoCache()
{ {
bool noCache; bool noCache;
SetBooleanParameter( &noCache, "no-cache", false ); SetBooleanParameter( &noCache, "no-cache", false );
if (noCache) { if (noCache) {
const std::vector<PhraseDictionary*> &pts = PhraseDictionary::GetColl(); const std::vector<PhraseDictionary*> &pts = PhraseDictionary::GetColl();
for (size_t i = 0; i < pts.size(); ++i) { for (size_t i = 0; i < pts.size(); ++i) {
PhraseDictionary &pt = *pts[i]; PhraseDictionary &pt = *pts[i];
pt.SetParameter("cache-size", "0"); pt.SetParameter("cache-size", "0");
} }
} }
} }
std::map<std::string, std::string> StaticData::OverrideFeatureNames() std::map<std::string, std::string> StaticData::OverrideFeatureNames()
{ {
std::map<std::string, std::string> ret; std::map<std::string, std::string> ret;
const PARAM_VEC &params = m_parameter->GetParam("feature-name-overwrite"); const PARAM_VEC &params = m_parameter->GetParam("feature-name-overwrite");
if (params.size()) { if (params.size()) {
UTIL_THROW_IF2(params.size() != 1, "Only provide 1 line in the section [feature-name-overwrite]"); UTIL_THROW_IF2(params.size() != 1, "Only provide 1 line in the section [feature-name-overwrite]");
vector<string> toks = Tokenize(params[0]); vector<string> toks = Tokenize(params[0]);
UTIL_THROW_IF2(toks.size() % 2 != 0, "Format of -feature-name-overwrite must be [old-name new-name]*"); UTIL_THROW_IF2(toks.size() % 2 != 0, "Format of -feature-name-overwrite must be [old-name new-name]*");
for (size_t i = 0; i < toks.size(); i += 2) { for (size_t i = 0; i < toks.size(); i += 2) {
const string &oldName = toks[i]; const string &oldName = toks[i];
const string &newName = toks[i+1]; const string &newName = toks[i+1];
ret[oldName] = newName; ret[oldName] = newName;
} }
} }
return ret; return ret;
} }
void StaticData::OverrideFeatures() void StaticData::OverrideFeatures()

View File

@ -645,7 +645,7 @@ public:
return false; return false;
} }
std::map< std::string, std::set< std::string > >::const_iterator lookupIgnoreFF std::map< std::string, std::set< std::string > >::const_iterator lookupIgnoreFF
= m_weightSettingIgnoreFF.find( m_currentWeightSetting ); = m_weightSettingIgnoreFF.find( m_currentWeightSetting );
if (lookupIgnoreFF == m_weightSettingIgnoreFF.end()) { if (lookupIgnoreFF == m_weightSettingIgnoreFF.end()) {
return false; return false;
} }
@ -663,7 +663,7 @@ public:
return false; return false;
} }
std::map< std::string, std::set< size_t > >::const_iterator lookupIgnoreDP std::map< std::string, std::set< size_t > >::const_iterator lookupIgnoreDP
= m_weightSettingIgnoreDP.find( m_currentWeightSetting ); = m_weightSettingIgnoreDP.find( m_currentWeightSetting );
if (lookupIgnoreDP == m_weightSettingIgnoreDP.end()) { if (lookupIgnoreDP == m_weightSettingIgnoreDP.end()) {
return false; return false;
} }
@ -728,8 +728,9 @@ public:
return m_placeHolderFactor; return m_placeHolderFactor;
} }
const FeatureRegistry &GetFeatureRegistry() const const FeatureRegistry &GetFeatureRegistry() const {
{ return m_registry; } return m_registry;
}
/** check whether we should be using the old code to support binary phrase-table. /** check whether we should be using the old code to support binary phrase-table.
** eventually, we'll stop support the binary phrase-table and delete this legacy code ** eventually, we'll stop support the binary phrase-table and delete this legacy code

View File

@ -234,7 +234,7 @@ void TargetPhrase::SetProperties(const StringPiece &str)
vector<string> keyValue = TokenizeFirstOnly(tok, " "); vector<string> keyValue = TokenizeFirstOnly(tok, " ");
UTIL_THROW_IF2(keyValue.size() != 2, UTIL_THROW_IF2(keyValue.size() != 2,
"Incorrect format of property: " << str); "Incorrect format of property: " << str);
SetProperty(keyValue[0], keyValue[1]); SetProperty(keyValue[0], keyValue[1]);
} }
} }

View File

@ -99,15 +99,15 @@ public:
return m_scoreBreakdown; return m_scoreBreakdown;
} }
/* /*
//TODO: Probably shouldn't copy this, but otherwise ownership is unclear //TODO: Probably shouldn't copy this, but otherwise ownership is unclear
void SetSourcePhrase(const Phrase& p) { void SetSourcePhrase(const Phrase& p) {
m_sourcePhrase=p; m_sourcePhrase=p;
} }
const Phrase& GetSourcePhrase() const { const Phrase& GetSourcePhrase() const {
return m_sourcePhrase; return m_sourcePhrase;
} }
*/ */
void SetTargetLHS(const Word *lhs) { void SetTargetLHS(const Word *lhs) {
m_lhsTarget = lhs; m_lhsTarget = lhs;
} }

View File

@ -72,15 +72,14 @@ public:
//! delete an entry from the collection //! delete an entry from the collection
void Remove(const size_t pos) { void Remove(const size_t pos) {
if (pos < m_collection.size()) if (pos < m_collection.size()) {
{
m_collection.erase(begin() + pos); m_collection.erase(begin() + pos);
} }
} }
//! return an entry of the collection //! return an entry of the collection
const TargetPhrase* GetTargetPhrase(const size_t pos) const { const TargetPhrase* GetTargetPhrase(const size_t pos) const {
return m_collection[pos]; return m_collection[pos];
} }
//! divide collection into 2 buckets using std::nth_element, the top & bottom according to table limit //! divide collection into 2 buckets using std::nth_element, the top & bottom according to table limit

View File

@ -10,7 +10,7 @@ namespace Moses
/*** /***
* Return the total wall time that the timer has been in the "running" * Return the total wall time that the timer has been in the "running"
* state since it was first "started". * state since it was first "started".
*/ */
double Timer::get_elapsed_time() const double Timer::get_elapsed_time() const
{ {
@ -39,8 +39,7 @@ void Timer::start(const char* msg)
if (stopped) { if (stopped) {
start_time = util::WallTime() - (stop_time - start_time); start_time = util::WallTime() - (stop_time - start_time);
stopped = false; stopped = false;
} } else {
else {
start_time = util::WallTime(); start_time = util::WallTime();
running = true; running = true;
} }

View File

@ -62,7 +62,7 @@ Load(
LoadCorpus(Output, targetStrme, m_outputFactors,*m_trgCorpus, m_trgSntBreaks, m_trgVocab); LoadCorpus(Output, targetStrme, m_outputFactors,*m_trgCorpus, m_trgSntBreaks, m_trgVocab);
UTIL_THROW_IF2(m_srcSntBreaks.size() != m_trgSntBreaks.size(), UTIL_THROW_IF2(m_srcSntBreaks.size() != m_trgSntBreaks.size(),
"Source and target arrays aren't the same size"); "Source and target arrays aren't the same size");
// build suffix arrays and auxilliary arrays // build suffix arrays and auxilliary arrays
cerr << "Building Source Suffix Array...\n"; cerr << "Building Source Suffix Array...\n";
@ -130,7 +130,7 @@ LoadRawAlignments(string& align)
vector<int> vtmp; vector<int> vtmp;
Utils::splitToInt(align, vtmp, "- "); Utils::splitToInt(align, vtmp, "- ");
UTIL_THROW_IF2(vtmp.size() % 2 != 0, UTIL_THROW_IF2(vtmp.size() % 2 != 0,
"Alignment format is incorrect: " << align); "Alignment format is incorrect: " << align);
vector<short> vAlgn; // store as short ints for memory vector<short> vAlgn; // store as short ints for memory
for (vector<int>::const_iterator itr = vtmp.begin(); for (vector<int>::const_iterator itr = vtmp.begin();
itr != vtmp.end(); ++itr) { itr != vtmp.end(); ++itr) {
@ -382,7 +382,7 @@ GetMosesFactorIDs(const SAPhrase& phrase, const Phrase& sourcePhrase) const
for(size_t i=0; i < phrase.words.size(); ++i) { // look up trg words for(size_t i=0; i < phrase.words.size(); ++i) { // look up trg words
Word& word = m_trgVocab->GetWord( phrase.words[i]); Word& word = m_trgVocab->GetWord( phrase.words[i]);
UTIL_THROW_IF2(word == m_trgVocab->GetkOOVWord(), UTIL_THROW_IF2(word == m_trgVocab->GetkOOVWord(),
"Unknown word at position " << i); "Unknown word at position " << i);
targetPhrase->AddWord(word); targetPhrase->AddWord(word);
} }
// scoring // scoring

View File

@ -35,11 +35,11 @@ std::vector<PhraseDictionary*> PhraseDictionary::s_staticColl;
CacheColl::~CacheColl() CacheColl::~CacheColl()
{ {
for (iterator iter = begin(); iter != end(); ++iter) { for (iterator iter = begin(); iter != end(); ++iter) {
std::pair<const TargetPhraseCollection*, clock_t> &key = iter->second; std::pair<const TargetPhraseCollection*, clock_t> &key = iter->second;
const TargetPhraseCollection *tps = key.first; const TargetPhraseCollection *tps = key.first;
delete tps; delete tps;
} }
} }
PhraseDictionary::PhraseDictionary(const std::string &line) PhraseDictionary::PhraseDictionary(const std::string &line)
@ -47,7 +47,7 @@ PhraseDictionary::PhraseDictionary(const std::string &line)
,m_tableLimit(20) // default ,m_tableLimit(20) // default
,m_maxCacheSize(DEFAULT_MAX_TRANS_OPT_CACHE_SIZE) ,m_maxCacheSize(DEFAULT_MAX_TRANS_OPT_CACHE_SIZE)
{ {
s_staticColl.push_back(this); s_staticColl.push_back(this);
} }
const TargetPhraseCollection *PhraseDictionary::GetTargetPhraseCollectionLEGACY(const Phrase& src) const const TargetPhraseCollection *PhraseDictionary::GetTargetPhraseCollectionLEGACY(const Phrase& src) const

View File

@ -60,7 +60,7 @@ class CacheColl : public std::map<size_t, std::pair<const TargetPhraseCollection
// 3rd = time of last access // 3rd = time of last access
public: public:
~CacheColl(); ~CacheColl();
}; };
/** /**
@ -70,7 +70,7 @@ class PhraseDictionary : public DecodeFeature
{ {
public: public:
static const std::vector<PhraseDictionary*>& GetColl() { static const std::vector<PhraseDictionary*>& GetColl() {
return s_staticColl; return s_staticColl;
} }
PhraseDictionary(const std::string &line); PhraseDictionary(const std::string &line);

View File

@ -35,7 +35,7 @@ PhraseDictionaryDynamicCacheBased *PhraseDictionaryDynamicCacheBased::s_instance
//! contructor //! contructor
PhraseDictionaryDynamicCacheBased::PhraseDictionaryDynamicCacheBased(const std::string &line) PhraseDictionaryDynamicCacheBased::PhraseDictionaryDynamicCacheBased(const std::string &line)
: PhraseDictionary(line) : PhraseDictionary(line)
{ {
std::cerr << "Initializing PhraseDictionaryDynamicCacheBased feature..." << std::endl; std::cerr << "Initializing PhraseDictionaryDynamicCacheBased feature..." << std::endl;
@ -114,15 +114,13 @@ const TargetPhraseCollection *PhraseDictionaryDynamicCacheBased::GetTargetPhrase
TargetPhraseCollection* tpc = NULL; TargetPhraseCollection* tpc = NULL;
VERBOSE(3,"source:|" << source << "|" << std::endl); VERBOSE(3,"source:|" << source << "|" << std::endl);
cacheMap::const_iterator it = m_cacheTM.find(source); cacheMap::const_iterator it = m_cacheTM.find(source);
if(it != m_cacheTM.end()) if(it != m_cacheTM.end()) {
{
VERBOSE(3,"source:|" << source << "| FOUND" << std::endl); VERBOSE(3,"source:|" << source << "| FOUND" << std::endl);
tpc = (it->second).first; tpc = (it->second).first;
std::vector<const TargetPhrase*>::const_iterator it2 = tpc->begin(); std::vector<const TargetPhrase*>::const_iterator it2 = tpc->begin();
while (it2 != tpc->end()) while (it2 != tpc->end()) {
{
((TargetPhrase*) *it2)->Evaluate(source, GetFeaturesToApply()); ((TargetPhrase*) *it2)->Evaluate(source, GetFeaturesToApply());
it2++; it2++;
} }
@ -145,7 +143,8 @@ ChartRuleLookupManager* PhraseDictionaryDynamicCacheBased::CreateRuleLookupManag
UTIL_THROW(util::Exception, "Phrase table used in chart decoder"); UTIL_THROW(util::Exception, "Phrase table used in chart decoder");
} }
void PhraseDictionaryDynamicCacheBased::SetScoreType(size_t type) { void PhraseDictionaryDynamicCacheBased::SetScoreType(size_t type)
{
#ifdef WITH_THREADS #ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock); boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
#endif #endif
@ -157,22 +156,22 @@ void PhraseDictionaryDynamicCacheBased::SetScoreType(size_t type) {
&& m_score_type != CBTM_SCORE_TYPE_COSINE && m_score_type != CBTM_SCORE_TYPE_COSINE
&& m_score_type != CBTM_SCORE_TYPE_HYPERBOLA_REWARD && m_score_type != CBTM_SCORE_TYPE_HYPERBOLA_REWARD
&& m_score_type != CBTM_SCORE_TYPE_POWER_REWARD && m_score_type != CBTM_SCORE_TYPE_POWER_REWARD
&& m_score_type != CBTM_SCORE_TYPE_EXPONENTIAL_REWARD ) && m_score_type != CBTM_SCORE_TYPE_EXPONENTIAL_REWARD ) {
{
VERBOSE(2, "This score type " << m_score_type << " is unknown. Instead used " << CBTM_SCORE_TYPE_HYPERBOLA << "." << std::endl); VERBOSE(2, "This score type " << m_score_type << " is unknown. Instead used " << CBTM_SCORE_TYPE_HYPERBOLA << "." << std::endl);
m_score_type = CBTM_SCORE_TYPE_HYPERBOLA; m_score_type = CBTM_SCORE_TYPE_HYPERBOLA;
} }
VERBOSE(2, "PhraseDictionaryDynamicCacheBased ScoreType: " << m_score_type << std::endl); VERBOSE(2, "PhraseDictionaryDynamicCacheBased ScoreType: " << m_score_type << std::endl);
} }
void PhraseDictionaryDynamicCacheBased::SetMaxAge(unsigned int age) { void PhraseDictionaryDynamicCacheBased::SetMaxAge(unsigned int age)
{
#ifdef WITH_THREADS #ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock); boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
#endif #endif
m_maxAge = age; m_maxAge = age;
VERBOSE(2, "PhraseDictionaryCache MaxAge: " << m_maxAge << std::endl); VERBOSE(2, "PhraseDictionaryCache MaxAge: " << m_maxAge << std::endl);
} }
@ -185,7 +184,7 @@ ostream& operator<<(ostream& out, const PhraseDictionaryDynamicCacheBased& phras
float PhraseDictionaryDynamicCacheBased::decaying_score(const int age) float PhraseDictionaryDynamicCacheBased::decaying_score(const int age)
{ {
float sc; float sc;
switch(m_score_type){ switch(m_score_type) {
case CBTM_SCORE_TYPE_HYPERBOLA: case CBTM_SCORE_TYPE_HYPERBOLA:
sc = (float) 1.0/age - 1.0; sc = (float) 1.0/age - 1.0;
break; break;
@ -218,28 +217,23 @@ void PhraseDictionaryDynamicCacheBased::SetPreComputedScores(const unsigned int
VERBOSE(3,"m_maxAge:|" << m_maxAge << "|" << std::endl); VERBOSE(3,"m_maxAge:|" << m_maxAge << "|" << std::endl);
#ifdef WITH_THREADS #ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock); boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
#endif #endif
float sc; float sc;
for (size_t i=0; i<=m_maxAge; i++) for (size_t i=0; i<=m_maxAge; i++) {
{ if (i==m_maxAge) {
if (i==m_maxAge){
if ( m_score_type == CBTM_SCORE_TYPE_HYPERBOLA if ( m_score_type == CBTM_SCORE_TYPE_HYPERBOLA
|| m_score_type == CBTM_SCORE_TYPE_POWER || m_score_type == CBTM_SCORE_TYPE_POWER
|| m_score_type == CBTM_SCORE_TYPE_EXPONENTIAL || m_score_type == CBTM_SCORE_TYPE_EXPONENTIAL
|| m_score_type == CBTM_SCORE_TYPE_COSINE ) || m_score_type == CBTM_SCORE_TYPE_COSINE ) {
{
sc = decaying_score(m_maxAge)/numScoreComponent; sc = decaying_score(m_maxAge)/numScoreComponent;
} } else { // m_score_type = CBTM_SCORE_TYPE_XXXXXXXXX_REWARD
else{ // m_score_type = CBTM_SCORE_TYPE_XXXXXXXXX_REWARD
sc = 0.0; sc = 0.0;
} }
} } else {
else{
sc = decaying_score(i)/numScoreComponent; sc = decaying_score(i)/numScoreComponent;
} }
Scores sc_vec; Scores sc_vec;
for (size_t j=0; j<numScoreComponent; j++) for (size_t j=0; j<numScoreComponent; j++) {
{
sc_vec.push_back(sc); //CHECK THIS SCORE sc_vec.push_back(sc); //CHECK THIS SCORE
} }
precomputedScores.push_back(sc_vec); precomputedScores.push_back(sc_vec);
@ -250,12 +244,9 @@ void PhraseDictionaryDynamicCacheBased::SetPreComputedScores(const unsigned int
Scores PhraseDictionaryDynamicCacheBased::GetPreComputedScores(const unsigned int age) Scores PhraseDictionaryDynamicCacheBased::GetPreComputedScores(const unsigned int age)
{ {
VERBOSE(3,"age:|" << age << "|" << std::endl); VERBOSE(3,"age:|" << age << "|" << std::endl);
if (age < precomputedScores.size()) if (age < precomputedScores.size()) {
{
return precomputedScores.at(age); return precomputedScores.at(age);
} } else {
else
{
return precomputedScores.at(m_maxAge); return precomputedScores.at(m_maxAge);
} }
} }
@ -285,13 +276,12 @@ void PhraseDictionaryDynamicCacheBased::Update(std::vector<std::string> entries,
std::vector<std::string> pp; std::vector<std::string> pp;
std::vector<std::string>::iterator it; std::vector<std::string>::iterator it;
for(it = entries.begin(); it!=entries.end(); it++) for(it = entries.begin(); it!=entries.end(); it++) {
{
pp.clear(); pp.clear();
pp = TokenizeMultiCharSeparator((*it), "|||"); pp = TokenizeMultiCharSeparator((*it), "|||");
VERBOSE(3,"pp[0]:|" << pp[0] << "|" << std::endl); VERBOSE(3,"pp[0]:|" << pp[0] << "|" << std::endl);
VERBOSE(3,"pp[1]:|" << pp[1] << "|" << std::endl); VERBOSE(3,"pp[1]:|" << pp[1] << "|" << std::endl);
Update(pp[0], pp[1], ageString); Update(pp[0], pp[1], ageString);
} }
} }
@ -303,7 +293,7 @@ void PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, s
const std::string& factorDelimiter = staticData.GetFactorDelimiter(); const std::string& factorDelimiter = staticData.GetFactorDelimiter();
Phrase sourcePhrase(0); Phrase sourcePhrase(0);
Phrase targetPhrase(0); Phrase targetPhrase(0);
char *err_ind_temp; char *err_ind_temp;
int age = strtod(ageString.c_str(), &err_ind_temp); int age = strtod(ageString.c_str(), &err_ind_temp);
//target //target
@ -311,7 +301,7 @@ void PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, s
VERBOSE(3, "targetPhraseString:|" << targetPhraseString << "|" << std::endl); VERBOSE(3, "targetPhraseString:|" << targetPhraseString << "|" << std::endl);
targetPhrase.CreateFromString(Output, staticData.GetOutputFactorOrder(), targetPhraseString, factorDelimiter, NULL); targetPhrase.CreateFromString(Output, staticData.GetOutputFactorOrder(), targetPhraseString, factorDelimiter, NULL);
VERBOSE(2, "targetPhrase:|" << targetPhrase << "|" << std::endl); VERBOSE(2, "targetPhrase:|" << targetPhrase << "|" << std::endl);
//TODO: Would be better to reuse source phrases, but ownership has to be //TODO: Would be better to reuse source phrases, but ownership has to be
//consistent across phrase table implementations //consistent across phrase table implementations
sourcePhrase.Clear(); sourcePhrase.Clear();
@ -331,8 +321,7 @@ void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
cacheMap::const_iterator it = m_cacheTM.find(sp); cacheMap::const_iterator it = m_cacheTM.find(sp);
VERBOSE(3,"sp:|" << sp << "|" << std::endl); VERBOSE(3,"sp:|" << sp << "|" << std::endl);
if(it!=m_cacheTM.end()) if(it!=m_cacheTM.end()) {
{
VERBOSE(3,"sp:|" << sp << "| FOUND" << std::endl); VERBOSE(3,"sp:|" << sp << "| FOUND" << std::endl);
// p is found // p is found
// here we have to remove the target phrase from targetphrasecollection and from the TargetAgeMap // here we have to remove the target phrase from targetphrasecollection and from the TargetAgeMap
@ -344,18 +333,15 @@ void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
const Phrase* tp_ptr = NULL; const Phrase* tp_ptr = NULL;
bool found = false; bool found = false;
size_t tp_pos=0; size_t tp_pos=0;
while (!found && tp_pos < tpc->GetSize()) while (!found && tp_pos < tpc->GetSize()) {
{
tp_ptr = (const Phrase*) tpc->GetTargetPhrase(tp_pos); tp_ptr = (const Phrase*) tpc->GetTargetPhrase(tp_pos);
if (tp == *tp_ptr) if (tp == *tp_ptr) {
{
found = true; found = true;
continue; continue;
} }
tp_pos++; tp_pos++;
} }
if (!found) if (!found) {
{
VERBOSE(3,"tp:|" << tp << "| NOT FOUND" << std::endl); VERBOSE(3,"tp:|" << tp << "| NOT FOUND" << std::endl);
std::auto_ptr<TargetPhrase> targetPhrase(new TargetPhrase(tp)); std::auto_ptr<TargetPhrase> targetPhrase(new TargetPhrase(tp));
@ -368,9 +354,7 @@ void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
VERBOSE(3,"ac size:|" << ac->size() << "|" << std::endl); VERBOSE(3,"ac size:|" << ac->size() << "|" << std::endl);
VERBOSE(3,"tp:|" << tp << "| INSERTED" << std::endl); VERBOSE(3,"tp:|" << tp << "| INSERTED" << std::endl);
} }
} } else {
else
{
VERBOSE(3,"sp:|" << sp << "| NOT FOUND" << std::endl); VERBOSE(3,"sp:|" << sp << "| NOT FOUND" << std::endl);
// p is not found // p is not found
// create target collection // create target collection
@ -397,21 +381,19 @@ void PhraseDictionaryDynamicCacheBased::Decay()
{ {
#ifdef WITH_THREADS #ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock); boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
#endif #endif
cacheMap::iterator it; cacheMap::iterator it;
for(it = m_cacheTM.begin(); it!=m_cacheTM.end(); it++) for(it = m_cacheTM.begin(); it!=m_cacheTM.end(); it++) {
{
Decay((*it).first); Decay((*it).first);
} }
} }
void PhraseDictionaryDynamicCacheBased::Decay(Phrase p) void PhraseDictionaryDynamicCacheBased::Decay(Phrase p)
{ {
VERBOSE(3,"p:|" << p << "|" << std::endl); VERBOSE(3,"p:|" << p << "|" << std::endl);
cacheMap::const_iterator it = m_cacheTM.find(p); cacheMap::const_iterator it = m_cacheTM.find(p);
VERBOSE(3,"searching:|" << p << "|" << std::endl); VERBOSE(3,"searching:|" << p << "|" << std::endl);
if (it != m_cacheTM.end()) if (it != m_cacheTM.end()) {
{
VERBOSE(3,"found:|" << p << "|" << std::endl); VERBOSE(3,"found:|" << p << "|" << std::endl);
//p is found //p is found
@ -420,8 +402,7 @@ void PhraseDictionaryDynamicCacheBased::Decay(Phrase p)
AgeCollection* ac = TgtCollAgePair.second; AgeCollection* ac = TgtCollAgePair.second;
//loop in inverted order to allow a correct deletion of std::vectors tpc and ac //loop in inverted order to allow a correct deletion of std::vectors tpc and ac
for (int tp_pos = tpc->GetSize() - 1 ; tp_pos >= 0; tp_pos--) for (int tp_pos = tpc->GetSize() - 1 ; tp_pos >= 0; tp_pos--) {
{
VERBOSE(3,"p:|" << p << "|" << std::endl); VERBOSE(3,"p:|" << p << "|" << std::endl);
unsigned int tp_age = ac->at(tp_pos); //increase the age by 1 unsigned int tp_age = ac->at(tp_pos); //increase the age by 1
tp_age++; //increase the age by 1 tp_age++; //increase the age by 1
@ -431,29 +412,26 @@ void PhraseDictionaryDynamicCacheBased::Decay(Phrase p)
VERBOSE(3,"p:|" << p << "| " << "tp_age:|" << tp_age << "| " << "*tp_ptr:|" << *tp_ptr << "|" << std::endl); VERBOSE(3,"p:|" << p << "| " << "tp_age:|" << tp_age << "| " << "*tp_ptr:|" << *tp_ptr << "|" << std::endl);
VERBOSE(3,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl); VERBOSE(3,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
if (tp_age > m_maxAge){ if (tp_age > m_maxAge) {
VERBOSE(3,"tp_age:|" << tp_age << "| TOO BIG" << std::endl); VERBOSE(3,"tp_age:|" << tp_age << "| TOO BIG" << std::endl);
tpc->Remove(tp_pos); //delete entry in the Target Phrase Collection tpc->Remove(tp_pos); //delete entry in the Target Phrase Collection
ac->erase(ac->begin() + tp_pos); //delete entry in the Age Collection ac->erase(ac->begin() + tp_pos); //delete entry in the Age Collection
m_entries--; m_entries--;
} } else {
else{
VERBOSE(3,"tp_age:|" << tp_age << "| STILL GOOD" << std::endl); VERBOSE(3,"tp_age:|" << tp_age << "| STILL GOOD" << std::endl);
tp_ptr->GetScoreBreakdown().Assign(this, GetPreComputedScores(tp_age)); tp_ptr->GetScoreBreakdown().Assign(this, GetPreComputedScores(tp_age));
ac->at(tp_pos) = tp_age; ac->at(tp_pos) = tp_age;
VERBOSE(3,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl); VERBOSE(3,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
} }
} }
if (tpc->GetSize() == 0) if (tpc->GetSize() == 0) {
{// delete the entry from m_cacheTM in case it points to an empty TargetPhraseCollection and AgeCollection // delete the entry from m_cacheTM in case it points to an empty TargetPhraseCollection and AgeCollection
(((*it).second).second)->clear(); (((*it).second).second)->clear();
delete ((*it).second).second; delete ((*it).second).second;
delete ((*it).second).first; delete ((*it).second).first;
m_cacheTM.erase(p); m_cacheTM.erase(p);
} }
} } else {
else
{
//do nothing //do nothing
VERBOSE(3,"p:|" << p << "| NOT FOUND" << std::endl); VERBOSE(3,"p:|" << p << "| NOT FOUND" << std::endl);
} }
@ -493,8 +471,7 @@ void PhraseDictionaryDynamicCacheBased::Clear()
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock); boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
#endif #endif
cacheMap::const_iterator it; cacheMap::const_iterator it;
for(it = m_cacheTM.begin(); it!=m_cacheTM.end(); it++) for(it = m_cacheTM.begin(); it!=m_cacheTM.end(); it++) {
{
(((*it).second).second)->clear(); (((*it).second).second)->clear();
delete ((*it).second).second; delete ((*it).second).second;
delete ((*it).second).first; delete ((*it).second).first;
@ -508,20 +485,18 @@ void PhraseDictionaryDynamicCacheBased::Print() const
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::Print()" << std::endl); VERBOSE(2,"PhraseDictionaryDynamicCacheBased::Print()" << std::endl);
#ifdef WITH_THREADS #ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock); boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
#endif #endif
cacheMap::const_iterator it; cacheMap::const_iterator it;
for(it = m_cacheTM.begin(); it!=m_cacheTM.end(); it++) for(it = m_cacheTM.begin(); it!=m_cacheTM.end(); it++) {
{
std::string source = (it->first).ToString(); std::string source = (it->first).ToString();
TargetPhraseCollection* tpc = (it->second).first; TargetPhraseCollection* tpc = (it->second).first;
TargetPhraseCollection::iterator itr; TargetPhraseCollection::iterator itr;
for(itr = tpc->begin(); itr != tpc->end(); itr++) for(itr = tpc->begin(); itr != tpc->end(); itr++) {
{
std::string target = (*itr)->ToString(); std::string target = (*itr)->ToString();
std::cout << source << " ||| " << target << std::endl; std::cout << source << " ||| " << target << std::endl;
} }
source.clear(); source.clear();
} }
} }
}// end namespace }// end namespace

View File

@ -42,9 +42,10 @@
#define PI 3.14159265 #define PI 3.14159265
namespace Moses { namespace Moses
{
/** Implementation of a Cache-based phrase table. /** Implementation of a Cache-based phrase table.
*/ */
class PhraseDictionaryDynamicCacheBased : public PhraseDictionary class PhraseDictionaryDynamicCacheBased : public PhraseDictionary
{ {
@ -106,7 +107,7 @@ public:
void SetScoreType(size_t type); void SetScoreType(size_t type);
void SetMaxAge(unsigned int age); void SetMaxAge(unsigned int age);
protected: protected:
static PhraseDictionaryDynamicCacheBased *s_instance; static PhraseDictionaryDynamicCacheBased *s_instance;

View File

@ -98,9 +98,9 @@ PhraseDictionaryNodeMemory &PhraseDictionaryMemory::GetOrCreateNode(const Phrase
const Word &sourceNonTerm = word; const Word &sourceNonTerm = word;
UTIL_THROW_IF2(iterAlign == alignmentInfo.end(), UTIL_THROW_IF2(iterAlign == alignmentInfo.end(),
"No alignment for non-term at position " << pos); "No alignment for non-term at position " << pos);
UTIL_THROW_IF2(iterAlign->first != pos, UTIL_THROW_IF2(iterAlign->first != pos,
"Alignment info incorrect at position " << pos); "Alignment info incorrect at position " << pos);
size_t targetNonTermInd = iterAlign->second; size_t targetNonTermInd = iterAlign->second;
++iterAlign; ++iterAlign;
@ -112,7 +112,7 @@ PhraseDictionaryNodeMemory &PhraseDictionaryMemory::GetOrCreateNode(const Phrase
} }
UTIL_THROW_IF2(currNode == NULL, UTIL_THROW_IF2(currNode == NULL,
"Node not found at position " << pos); "Node not found at position " << pos);
} }
// finally, the source LHS // finally, the source LHS

View File

@ -38,18 +38,18 @@ PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &line)
size_t numWeights = m_numScoreComponents; size_t numWeights = m_numScoreComponents;
UTIL_THROW_IF2(m_pdStr.size() != m_multimodelweights.size() & UTIL_THROW_IF2(m_pdStr.size() != m_multimodelweights.size() &
m_pdStr.size()*numWeights != m_multimodelweights.size(), m_pdStr.size()*numWeights != m_multimodelweights.size(),
"Number of scores and weights are not equal"); "Number of scores and weights are not equal");
} }
PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(int type, const std::string &line) PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(int type, const std::string &line)
:PhraseDictionary(line) :PhraseDictionary(line)
{ {
if (type == 1) { if (type == 1) {
// PhraseDictionaryMultiModelCounts // PhraseDictionaryMultiModelCounts
UTIL_THROW_IF2(m_pdStr.size() != m_multimodelweights.size() && UTIL_THROW_IF2(m_pdStr.size() != m_multimodelweights.size() &&
m_pdStr.size()*4 != m_multimodelweights.size(), m_pdStr.size()*4 != m_multimodelweights.size(),
"Number of scores and weights are not equal"); "Number of scores and weights are not equal");
} }
} }
@ -80,7 +80,7 @@ void PhraseDictionaryMultiModel::Load()
PhraseDictionary *pt = FindPhraseDictionary(ptName); PhraseDictionary *pt = FindPhraseDictionary(ptName);
UTIL_THROW_IF2(pt == NULL, UTIL_THROW_IF2(pt == NULL,
"Could not find component phrase table " << ptName); "Could not find component phrase table " << ptName);
m_pd.push_back(pt); m_pd.push_back(pt);
} }
} }

View File

@ -69,7 +69,7 @@ PhraseDictionaryMultiModelCounts::PhraseDictionaryMultiModelCounts(const std::st
ReadParameters(); ReadParameters();
UTIL_THROW_IF2(m_targetTable.size() != m_pdStr.size(), UTIL_THROW_IF2(m_targetTable.size() != m_pdStr.size(),
"List of phrase tables and target tables must be equal"); "List of phrase tables and target tables must be equal");
} }
@ -89,11 +89,11 @@ void PhraseDictionaryMultiModelCounts::SetParameter(const std::string& key, cons
} else if (key == "lex-e2f") { } else if (key == "lex-e2f") {
m_lexE2FStr = Tokenize(value, ","); m_lexE2FStr = Tokenize(value, ",");
UTIL_THROW_IF2(m_lexE2FStr.size() != m_pdStr.size(), UTIL_THROW_IF2(m_lexE2FStr.size() != m_pdStr.size(),
"Number of scores for lexical probability p(f|e) incorrectly specified"); "Number of scores for lexical probability p(f|e) incorrectly specified");
} else if (key == "lex-f2e") { } else if (key == "lex-f2e") {
m_lexF2EStr = Tokenize(value, ","); m_lexF2EStr = Tokenize(value, ",");
UTIL_THROW_IF2(m_lexF2EStr.size() != m_pdStr.size(), UTIL_THROW_IF2(m_lexF2EStr.size() != m_pdStr.size(),
"Number of scores for lexical probability p(e|f) incorrectly specified"); "Number of scores for lexical probability p(e|f) incorrectly specified");
} else if (key == "target-table") { } else if (key == "target-table") {
m_targetTable = Tokenize(value, ","); m_targetTable = Tokenize(value, ",");
} else { } else {
@ -119,14 +119,14 @@ void PhraseDictionaryMultiModelCounts::Load()
PhraseDictionary *pt; PhraseDictionary *pt;
pt = FindPhraseDictionary(ptName); pt = FindPhraseDictionary(ptName);
UTIL_THROW_IF2(pt == NULL, UTIL_THROW_IF2(pt == NULL,
"Could not find component phrase table " << ptName); "Could not find component phrase table " << ptName);
m_pd.push_back(pt); m_pd.push_back(pt);
// reverse // reverse
const string &target_table = m_targetTable[i]; const string &target_table = m_targetTable[i];
pt = FindPhraseDictionary(target_table); pt = FindPhraseDictionary(target_table);
UTIL_THROW_IF2(pt == NULL, UTIL_THROW_IF2(pt == NULL,
"Could not find component phrase table " << target_table); "Could not find component phrase table " << target_table);
m_inverse_pd.push_back(pt); m_inverse_pd.push_back(pt);
// lex // lex

View File

@ -64,9 +64,9 @@ PhraseDictionaryNodeMemory *PhraseDictionaryNodeMemory::GetOrCreateChild(const W
PhraseDictionaryNodeMemory *PhraseDictionaryNodeMemory::GetOrCreateChild(const Word &sourceNonTerm, const Word &targetNonTerm) PhraseDictionaryNodeMemory *PhraseDictionaryNodeMemory::GetOrCreateChild(const Word &sourceNonTerm, const Word &targetNonTerm)
{ {
UTIL_THROW_IF2(!sourceNonTerm.IsNonTerminal(), UTIL_THROW_IF2(!sourceNonTerm.IsNonTerminal(),
"Not a non-terminal: " << sourceNonTerm); "Not a non-terminal: " << sourceNonTerm);
UTIL_THROW_IF2(!targetNonTerm.IsNonTerminal(), UTIL_THROW_IF2(!targetNonTerm.IsNonTerminal(),
"Not a non-terminal: " << targetNonTerm); "Not a non-terminal: " << targetNonTerm);
NonTerminalMapKey key(sourceNonTerm, targetNonTerm); NonTerminalMapKey key(sourceNonTerm, targetNonTerm);
return &m_nonTermMap[NonTerminalMapKey(sourceNonTerm, targetNonTerm)]; return &m_nonTermMap[NonTerminalMapKey(sourceNonTerm, targetNonTerm)];
@ -75,7 +75,7 @@ PhraseDictionaryNodeMemory *PhraseDictionaryNodeMemory::GetOrCreateChild(const W
const PhraseDictionaryNodeMemory *PhraseDictionaryNodeMemory::GetChild(const Word &sourceTerm) const const PhraseDictionaryNodeMemory *PhraseDictionaryNodeMemory::GetChild(const Word &sourceTerm) const
{ {
UTIL_THROW_IF2(sourceTerm.IsNonTerminal(), UTIL_THROW_IF2(sourceTerm.IsNonTerminal(),
"Not a terminal: " << sourceTerm); "Not a terminal: " << sourceTerm);
TerminalMap::const_iterator p = m_sourceTermMap.find(sourceTerm); TerminalMap::const_iterator p = m_sourceTermMap.find(sourceTerm);
return (p == m_sourceTermMap.end()) ? NULL : &p->second; return (p == m_sourceTermMap.end()) ? NULL : &p->second;
@ -84,9 +84,9 @@ const PhraseDictionaryNodeMemory *PhraseDictionaryNodeMemory::GetChild(const Wor
const PhraseDictionaryNodeMemory *PhraseDictionaryNodeMemory::GetChild(const Word &sourceNonTerm, const Word &targetNonTerm) const const PhraseDictionaryNodeMemory *PhraseDictionaryNodeMemory::GetChild(const Word &sourceNonTerm, const Word &targetNonTerm) const
{ {
UTIL_THROW_IF2(!sourceNonTerm.IsNonTerminal(), UTIL_THROW_IF2(!sourceNonTerm.IsNonTerminal(),
"Not a non-terminal: " << sourceNonTerm); "Not a non-terminal: " << sourceNonTerm);
UTIL_THROW_IF2(!targetNonTerm.IsNonTerminal(), UTIL_THROW_IF2(!targetNonTerm.IsNonTerminal(),
"Not a non-terminal: " << targetNonTerm); "Not a non-terminal: " << targetNonTerm);
NonTerminalMapKey key(sourceNonTerm, targetNonTerm); NonTerminalMapKey key(sourceNonTerm, targetNonTerm);
NonTerminalMap::const_iterator p = m_nonTermMap.find(key); NonTerminalMap::const_iterator p = m_nonTermMap.find(key);

View File

@ -14,20 +14,20 @@ PhraseDictionaryTransliteration::PhraseDictionaryTransliteration(const std::stri
{ {
ReadParameters(); ReadParameters();
UTIL_THROW_IF2(m_mosesDir.empty() || UTIL_THROW_IF2(m_mosesDir.empty() ||
m_scriptDir.empty() || m_scriptDir.empty() ||
m_externalDir.empty() || m_externalDir.empty() ||
m_inputLang.empty() || m_inputLang.empty() ||
m_outputLang.empty(), "Must specify all arguments"); m_outputLang.empty(), "Must specify all arguments");
} }
void PhraseDictionaryTransliteration::Load() void PhraseDictionaryTransliteration::Load()
{ {
SetFeaturesToApply(); SetFeaturesToApply();
} }
void PhraseDictionaryTransliteration::CleanUpAfterSentenceProcessing(const InputType& source) void PhraseDictionaryTransliteration::CleanUpAfterSentenceProcessing(const InputType& source)
{ {
ReduceCache(); ReduceCache();
} }
void PhraseDictionaryTransliteration::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const void PhraseDictionaryTransliteration::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
@ -38,14 +38,14 @@ void PhraseDictionaryTransliteration::GetTargetPhraseCollectionBatch(const Input
InputPath &inputPath = **iter; InputPath &inputPath = **iter;
if (!SatisfyBackoff(inputPath)) { if (!SatisfyBackoff(inputPath)) {
continue; continue;
} }
const Phrase &sourcePhrase = inputPath.GetPhrase(); const Phrase &sourcePhrase = inputPath.GetPhrase();
if (sourcePhrase.GetSize() != 1) { if (sourcePhrase.GetSize() != 1) {
// only translit single words. A limitation of the translit script // only translit single words. A limitation of the translit script
continue; continue;
} }
GetTargetPhraseCollection(inputPath); GetTargetPhraseCollection(inputPath);
@ -54,90 +54,89 @@ void PhraseDictionaryTransliteration::GetTargetPhraseCollectionBatch(const Input
void PhraseDictionaryTransliteration::GetTargetPhraseCollection(InputPath &inputPath) const void PhraseDictionaryTransliteration::GetTargetPhraseCollection(InputPath &inputPath) const
{ {
const Phrase &sourcePhrase = inputPath.GetPhrase(); const Phrase &sourcePhrase = inputPath.GetPhrase();
size_t hash = hash_value(sourcePhrase); size_t hash = hash_value(sourcePhrase);
CacheColl &cache = GetCache(); CacheColl &cache = GetCache();
std::map<size_t, std::pair<const TargetPhraseCollection*, clock_t> >::iterator iter; std::map<size_t, std::pair<const TargetPhraseCollection*, clock_t> >::iterator iter;
iter = cache.find(hash); iter = cache.find(hash);
if (iter != cache.end()) { if (iter != cache.end()) {
// already in cache // already in cache
const TargetPhraseCollection *tpColl = iter->second.first; const TargetPhraseCollection *tpColl = iter->second.first;
inputPath.SetTargetPhrases(*this, tpColl, NULL); inputPath.SetTargetPhrases(*this, tpColl, NULL);
} else {
// TRANSLITERATE
char *ptr = tmpnam(NULL);
string inFile(ptr);
ptr = tmpnam(NULL);
string outDir(ptr);
ofstream inStream(inFile.c_str());
inStream << sourcePhrase.ToString() << endl;
inStream.close();
string cmd = m_scriptDir + "/Transliteration/prepare-transliteration-phrase-table.pl" +
" --transliteration-model-dir " + m_filePath +
" --moses-src-dir " + m_mosesDir +
" --external-bin-dir " + m_externalDir +
" --input-extension " + m_inputLang +
" --output-extension " + m_outputLang +
" --oov-file " + inFile +
" --out-dir " + outDir;
int ret = system(cmd.c_str());
UTIL_THROW_IF2(ret != 0, "Transliteration script error");
TargetPhraseCollection *tpColl = new TargetPhraseCollection();
vector<TargetPhrase*> targetPhrases = CreateTargetPhrases(sourcePhrase, outDir);
vector<TargetPhrase*>::const_iterator iter;
for (iter = targetPhrases.begin(); iter != targetPhrases.end(); ++iter) {
TargetPhrase *tp = *iter;
tpColl->Add(tp);
} }
else {
// TRANSLITERATE
char *ptr = tmpnam(NULL);
string inFile(ptr);
ptr = tmpnam(NULL);
string outDir(ptr);
ofstream inStream(inFile.c_str()); std::pair<const TargetPhraseCollection*, clock_t> value(tpColl, clock());
inStream << sourcePhrase.ToString() << endl; cache[hash] = value;
inStream.close();
string cmd = m_scriptDir + "/Transliteration/prepare-transliteration-phrase-table.pl" + inputPath.SetTargetPhrases(*this, tpColl, NULL);
" --transliteration-model-dir " + m_filePath +
" --moses-src-dir " + m_mosesDir +
" --external-bin-dir " + m_externalDir +
" --input-extension " + m_inputLang +
" --output-extension " + m_outputLang +
" --oov-file " + inFile +
" --out-dir " + outDir;
int ret = system(cmd.c_str()); // clean up temporary files
UTIL_THROW_IF2(ret != 0, "Transliteration script error"); remove(inFile.c_str());
TargetPhraseCollection *tpColl = new TargetPhraseCollection(); cmd = "rm -rf " + outDir;
vector<TargetPhrase*> targetPhrases = CreateTargetPhrases(sourcePhrase, outDir); system(cmd.c_str());
vector<TargetPhrase*>::const_iterator iter; }
for (iter = targetPhrases.begin(); iter != targetPhrases.end(); ++iter) {
TargetPhrase *tp = *iter;
tpColl->Add(tp);
}
std::pair<const TargetPhraseCollection*, clock_t> value(tpColl, clock());
cache[hash] = value;
inputPath.SetTargetPhrases(*this, tpColl, NULL);
// clean up temporary files
remove(inFile.c_str());
cmd = "rm -rf " + outDir;
system(cmd.c_str());
}
} }
std::vector<TargetPhrase*> PhraseDictionaryTransliteration::CreateTargetPhrases(const Phrase &sourcePhrase, const string &outDir) const std::vector<TargetPhrase*> PhraseDictionaryTransliteration::CreateTargetPhrases(const Phrase &sourcePhrase, const string &outDir) const
{ {
std::vector<TargetPhrase*> ret; std::vector<TargetPhrase*> ret;
string outPath = outDir + "/out.txt"; string outPath = outDir + "/out.txt";
ifstream outStream(outPath.c_str()); ifstream outStream(outPath.c_str());
string line; string line;
while (getline(outStream, line)) { while (getline(outStream, line)) {
vector<string> toks; vector<string> toks;
Tokenize(toks, line, "\t"); Tokenize(toks, line, "\t");
UTIL_THROW_IF2(toks.size() != 2, "Error in transliteration output file. Expecting word\tscore"); UTIL_THROW_IF2(toks.size() != 2, "Error in transliteration output file. Expecting word\tscore");
TargetPhrase *tp = new TargetPhrase(); TargetPhrase *tp = new TargetPhrase();
Word &word = tp->AddWord(); Word &word = tp->AddWord();
word.CreateFromString(Output, m_output, toks[0], false); word.CreateFromString(Output, m_output, toks[0], false);
float score = Scan<float>(toks[1]); float score = Scan<float>(toks[1]);
tp->GetScoreBreakdown().PlusEquals(this, score); tp->GetScoreBreakdown().PlusEquals(this, score);
// score of all other ff when this rule is being loaded // score of all other ff when this rule is being loaded
tp->Evaluate(sourcePhrase, GetFeaturesToApply()); tp->Evaluate(sourcePhrase, GetFeaturesToApply());
ret.push_back(tp); ret.push_back(tp);
} }
outStream.close(); outStream.close();
return ret; return ret;
} }
@ -145,7 +144,7 @@ std::vector<TargetPhrase*> PhraseDictionaryTransliteration::CreateTargetPhrases(
ChartRuleLookupManager* PhraseDictionaryTransliteration::CreateRuleLookupManager(const ChartParser &parser, ChartRuleLookupManager* PhraseDictionaryTransliteration::CreateRuleLookupManager(const ChartParser &parser,
const ChartCellCollectionBase &cellCollection) const ChartCellCollectionBase &cellCollection)
{ {
return NULL; return NULL;
//return new ChartRuleLookupManagerSkeleton(parser, cellCollection, *this); //return new ChartRuleLookupManagerSkeleton(parser, cellCollection, *this);
} }
@ -154,17 +153,17 @@ PhraseDictionaryTransliteration::
SetParameter(const std::string& key, const std::string& value) SetParameter(const std::string& key, const std::string& value)
{ {
if (key == "moses-dir") { if (key == "moses-dir") {
m_mosesDir = value; m_mosesDir = value;
} else if (key == "script-dir") { } else if (key == "script-dir") {
m_scriptDir = value; m_scriptDir = value;
} else if (key == "external-dir") { } else if (key == "external-dir") {
m_externalDir = value; m_externalDir = value;
} else if (key == "input-lang") { } else if (key == "input-lang") {
m_inputLang = value; m_inputLang = value;
} else if (key == "output-lang") { } else if (key == "output-lang") {
m_outputLang = value; m_outputLang = value;
} else { } else {
PhraseDictionary::SetParameter(key, value); PhraseDictionary::SetParameter(key, value);
} }
} }
@ -177,25 +176,25 @@ bool PhraseDictionaryTransliteration::SatisfyBackoff(const InputPath &inputPath)
size_t backoff = decodeGraph->GetBackoff(); size_t backoff = decodeGraph->GetBackoff();
if (backoff == 0) { if (backoff == 0) {
// ie. don't backoff. Collect ALL translations // ie. don't backoff. Collect ALL translations
return true; return true;
} }
if (sourcePhrase.GetSize() > backoff) { if (sourcePhrase.GetSize() > backoff) {
// source phrase too big // source phrase too big
return false; return false;
} }
// lookup translation only if no other translations // lookup translation only if no other translations
InputPath::TargetPhrases::const_iterator iter; InputPath::TargetPhrases::const_iterator iter;
for (iter = inputPath.GetTargetPhrases().begin(); iter != inputPath.GetTargetPhrases().end(); ++iter) { for (iter = inputPath.GetTargetPhrases().begin(); iter != inputPath.GetTargetPhrases().end(); ++iter) {
const std::pair<const TargetPhraseCollection*, const void*> &temp = iter->second; const std::pair<const TargetPhraseCollection*, const void*> &temp = iter->second;
const TargetPhraseCollection *tpCollPrev = temp.first; const TargetPhraseCollection *tpCollPrev = temp.first;
if (tpCollPrev && tpCollPrev->GetSize()) { if (tpCollPrev && tpCollPrev->GetSize()) {
// already have translation from another pt. Don't create translations // already have translation from another pt. Don't create translations
return false; return false;
} }
} }
return true; return true;

View File

@ -278,7 +278,7 @@ public:
} }
PPtr Extend(PPtr p,const std::string& w) { PPtr Extend(PPtr p,const std::string& w) {
UTIL_THROW_IF2(p == NULL, "Error"); UTIL_THROW_IF2(p == NULL, "Error");
if(w.empty() || w==EPSILON) return p; if(w.empty() || w==EPSILON) return p;
@ -380,8 +380,8 @@ PhraseDictionaryTree::PhraseDictionaryTree()
: imp(new PDTimp) : imp(new PDTimp)
{ {
if(sizeof(OFF_T)!=8) { if(sizeof(OFF_T)!=8) {
UTIL_THROW2("ERROR: size of type 'OFF_T' has to be 64 bit!\n" UTIL_THROW2("ERROR: size of type 'OFF_T' has to be 64 bit!\n"
"In gcc, use compiler settings '-D_FILE_OFFSET_BITS=64 -D_LARGE_FILES'\n"); "In gcc, use compiler settings '-D_FILE_OFFSET_BITS=64 -D_LARGE_FILES'\n");
} }
} }
@ -500,7 +500,7 @@ int PhraseDictionaryTree::Create(std::istream& inFile,const std::string& out)
// init numElement // init numElement
numElement = tokens.size(); numElement = tokens.size();
UTIL_THROW_IF2(numElement < (PrintWordAlignment()?4:3), UTIL_THROW_IF2(numElement < (PrintWordAlignment()?4:3),
"Format error"); "Format error");
} }
if (tokens.size() != numElement) { if (tokens.size() != numElement) {
@ -549,8 +549,8 @@ int PhraseDictionaryTree::Create(std::istream& inFile,const std::string& out)
PSA::Data& d=psa->insert(f); PSA::Data& d=psa->insert(f);
if(d==InvalidOffT) d=fTell(ot); if(d==InvalidOffT) d=fTell(ot);
else { else {
UTIL_THROW2("ERROR: source phrase already inserted (A)!\nline(" << lnc << "): '" UTIL_THROW2("ERROR: source phrase already inserted (A)!\nline(" << lnc << "): '"
<<line); <<line);
} }
} }
@ -559,8 +559,8 @@ int PhraseDictionaryTree::Create(std::istream& inFile,const std::string& out)
if (!sparseFeatureString.empty()) { if (!sparseFeatureString.empty()) {
std::vector<std::string> sparseTokens = Tokenize(sparseFeatureString); std::vector<std::string> sparseTokens = Tokenize(sparseFeatureString);
if (sparseTokens.size() % 2 != 0) { if (sparseTokens.size() % 2 != 0) {
UTIL_THROW2("ERROR: incorrectly formatted sparse feature string: " << UTIL_THROW2("ERROR: incorrectly formatted sparse feature string: " <<
sparseFeatureString); sparseFeatureString);
} }
for (size_t i = 0; i < sparseTokens.size(); i+=2) { for (size_t i = 0; i < sparseTokens.size(); i+=2) {
fnames.push_back(imp->tv.add(sparseTokens[i])); fnames.push_back(imp->tv.add(sparseTokens[i]));
@ -601,13 +601,13 @@ int PhraseDictionaryTree::Create(std::istream& inFile,const std::string& out)
PSA::Data& d=psa->insert(f); PSA::Data& d=psa->insert(f);
if(d==InvalidOffT) d=fTell(ot); if(d==InvalidOffT) d=fTell(ot);
else { else {
UTIL_THROW2("ERROR: xsource phrase already inserted (B)!\nline(" << lnc << "): '" UTIL_THROW2("ERROR: xsource phrase already inserted (B)!\nline(" << lnc << "): '"
<<line); <<line);
} }
} }
tgtCands.push_back(TgtCand(e,sc, alignmentString)); tgtCands.push_back(TgtCand(e,sc, alignmentString));
UTIL_THROW_IF2(currFirstWord == InvalidLabelId, UTIL_THROW_IF2(currFirstWord == InvalidLabelId,
"Uninitialize word"); "Uninitialize word");
tgtCands.back().SetFeatures(fnames, fvalues); tgtCands.back().SetFeatures(fnames, fvalues);
} }
if (PrintWordAlignment()) if (PrintWordAlignment())

View File

@ -53,7 +53,7 @@ void PhraseDictionaryTreeAdaptor::InitializeForInput(InputType const& source)
if(m_numScoreComponents!=weight.size()) { if(m_numScoreComponents!=weight.size()) {
std::stringstream strme; std::stringstream strme;
UTIL_THROW2("ERROR: mismatch of number of scaling factors: " << weight.size() UTIL_THROW2("ERROR: mismatch of number of scaling factors: " << weight.size()
<< " " << m_numScoreComponents); << " " << m_numScoreComponents);
} }
obj->Create(m_input, m_output, m_filePath, weight); obj->Create(m_input, m_output, m_filePath, weight);

View File

@ -14,7 +14,7 @@ SkeletonPT::SkeletonPT(const std::string &line)
void SkeletonPT::Load() void SkeletonPT::Load()
{ {
SetFeaturesToApply(); SetFeaturesToApply();
} }
void SkeletonPT::InitializeForInput(InputType const& source) void SkeletonPT::InitializeForInput(InputType const& source)
@ -37,8 +37,8 @@ void SkeletonPT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQu
// add target phrase to phrase-table cache // add target phrase to phrase-table cache
size_t hash = hash_value(sourcePhrase); size_t hash = hash_value(sourcePhrase);
std::pair<const TargetPhraseCollection*, clock_t> value(tpColl, clock()); std::pair<const TargetPhraseCollection*, clock_t> value(tpColl, clock());
cache[hash] = value; cache[hash] = value;
inputPath.SetTargetPhrases(*this, tpColl, NULL); inputPath.SetTargetPhrases(*this, tpColl, NULL);
} }

View File

@ -80,14 +80,14 @@ void TranslationOption::Evaluate(const InputType &input)
const InputPath &TranslationOption::GetInputPath() const const InputPath &TranslationOption::GetInputPath() const
{ {
UTIL_THROW_IF2(m_inputPath == NULL, UTIL_THROW_IF2(m_inputPath == NULL,
"No input path"); "No input path");
return *m_inputPath; return *m_inputPath;
} }
void TranslationOption::SetInputPath(const InputPath &inputPath) void TranslationOption::SetInputPath(const InputPath &inputPath)
{ {
UTIL_THROW_IF2(m_inputPath, UTIL_THROW_IF2(m_inputPath,
"Input path already specified"); "Input path already specified");
m_inputPath = &inputPath; m_inputPath = &inputPath;
} }

View File

@ -108,7 +108,7 @@ void TranslationOptionCollection::Prune()
if (m_maxNoTransOptPerCoverage > 0 && if (m_maxNoTransOptPerCoverage > 0 &&
fullList.size() > m_maxNoTransOptPerCoverage) { fullList.size() > m_maxNoTransOptPerCoverage) {
// sort in vector // sort in vector
NTH_ELEMENT4(fullList.begin(), fullList.begin() + m_maxNoTransOptPerCoverage, fullList.end(), CompareTranslationOption); NTH_ELEMENT4(fullList.begin(), fullList.begin() + m_maxNoTransOptPerCoverage, fullList.end(), CompareTranslationOption);
totalPruned += fullList.size() - m_maxNoTransOptPerCoverage; totalPruned += fullList.size() - m_maxNoTransOptPerCoverage;
// delete the rest // delete the rest
@ -392,7 +392,7 @@ void TranslationOptionCollection::CreateTranslationOptions()
// ... and that end at endPos // ... and that end at endPos
for (size_t endPos = startPos ; endPos < startPos + maxSize ; endPos++) { for (size_t endPos = startPos ; endPos < startPos + maxSize ; endPos++) {
if (graphInd > 0 && // only skip subsequent graphs if (graphInd > 0 && // only skip subsequent graphs
backoff != 0 && // use of backoff specified backoff != 0 && // use of backoff specified
(endPos-startPos+1 >= backoff || // size exceeds backoff limit or ... (endPos-startPos+1 >= backoff || // size exceeds backoff limit or ...
m_collection[startPos][endPos-startPos].size() > 0)) { // no phrases found so far m_collection[startPos][endPos-startPos].size() > 0)) { // no phrases found so far
VERBOSE(3,"No backoff to graph " << graphInd << " for span [" << startPos << ";" << endPos << "]" << endl); VERBOSE(3,"No backoff to graph " << graphInd << " for span [" << startPos << ";" << endPos << "]" << endl);
@ -611,7 +611,7 @@ void TranslationOptionCollection::Add(TranslationOption *translationOption)
{ {
const WordsRange &coverage = translationOption->GetSourceWordsRange(); const WordsRange &coverage = translationOption->GetSourceWordsRange();
UTIL_THROW_IF2(coverage.GetEndPos() - coverage.GetStartPos() >= m_collection[coverage.GetStartPos()].size(), UTIL_THROW_IF2(coverage.GetEndPos() - coverage.GetStartPos() >= m_collection[coverage.GetStartPos()].size(),
"Out of bound access: " << coverage); "Out of bound access: " << coverage);
m_collection[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()].Add(translationOption); m_collection[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()].Add(translationOption);
} }
@ -684,7 +684,7 @@ TranslationOptionList &TranslationOptionCollection::GetTranslationOptionList(siz
maxSize = std::min(maxSize, maxSizePhrase); maxSize = std::min(maxSize, maxSizePhrase);
UTIL_THROW_IF2(maxSize >= m_collection[startPos].size(), UTIL_THROW_IF2(maxSize >= m_collection[startPos].size(),
"Out of bound access: " << maxSize); "Out of bound access: " << maxSize);
return m_collection[startPos][maxSize]; return m_collection[startPos][maxSize];
} }
@ -695,7 +695,7 @@ const TranslationOptionList &TranslationOptionCollection::GetTranslationOptionLi
maxSize = std::min(maxSize, maxSizePhrase); maxSize = std::min(maxSize, maxSizePhrase);
UTIL_THROW_IF2(maxSize >= m_collection[startPos].size(), UTIL_THROW_IF2(maxSize >= m_collection[startPos].size(),
"Out of bound access: " << maxSize); "Out of bound access: " << maxSize);
return m_collection[startPos][maxSize]; return m_collection[startPos][maxSize];
} }

View File

@ -81,7 +81,7 @@ TranslationOptionCollectionConfusionNet::TranslationOptionCollectionConfusionNet
const Phrase &prevPhrase = prevPath.GetPhrase(); const Phrase &prevPhrase = prevPath.GetPhrase();
const ScorePair *prevInputScore = prevPath.GetInputScore(); const ScorePair *prevInputScore = prevPath.GetInputScore();
UTIL_THROW_IF2(prevInputScore == NULL, UTIL_THROW_IF2(prevInputScore == NULL,
"No input score for path: " << prevPath); "No input score for path: " << prevPath);
// loop thru every word at this position // loop thru every word at this position
const ConfusionNet::Column &col = input.GetColumn(endPos); const ConfusionNet::Column &col = input.GetColumn(endPos);

View File

@ -24,7 +24,7 @@ TranslationOptionCollectionLattice::TranslationOptionCollectionLattice(
: TranslationOptionCollection(input, maxNoTransOptPerCoverage, translationOptionThreshold) : TranslationOptionCollection(input, maxNoTransOptPerCoverage, translationOptionThreshold)
{ {
UTIL_THROW_IF2(StaticData::Instance().GetUseLegacyPT(), UTIL_THROW_IF2(StaticData::Instance().GetUseLegacyPT(),
"Not for models using the legqacy binary phrase table"); "Not for models using the legqacy binary phrase table");
const InputFeature &inputFeature = InputFeature::Instance(); const InputFeature &inputFeature = InputFeature::Instance();
UTIL_THROW_IF2(&inputFeature == NULL, "Input feature must be specified"); UTIL_THROW_IF2(&inputFeature == NULL, "Input feature must be specified");
@ -61,57 +61,57 @@ TranslationOptionCollectionLattice::TranslationOptionCollectionLattice(
} }
// iteratively extend all paths // iteratively extend all paths
for (size_t endPos = 1; endPos < size; ++endPos) { for (size_t endPos = 1; endPos < size; ++endPos) {
const std::vector<size_t> &nextNodes = input.GetNextNodes(endPos); const std::vector<size_t> &nextNodes = input.GetNextNodes(endPos);
// loop thru every previous paths // loop thru every previous paths
size_t numPrevPaths = m_inputPathQueue.size(); size_t numPrevPaths = m_inputPathQueue.size();
for (size_t i = 0; i < numPrevPaths; ++i) { for (size_t i = 0; i < numPrevPaths; ++i) {
//for (size_t pathInd = 0; pathInd < prevPaths.size(); ++pathInd) { //for (size_t pathInd = 0; pathInd < prevPaths.size(); ++pathInd) {
const InputPath &prevPath = *m_inputPathQueue[i]; const InputPath &prevPath = *m_inputPathQueue[i];
size_t nextNode = prevPath.GetNextNode(); size_t nextNode = prevPath.GetNextNode();
if (prevPath.GetWordsRange().GetEndPos() + nextNode != endPos) { if (prevPath.GetWordsRange().GetEndPos() + nextNode != endPos) {
continue; continue;
} }
size_t startPos = prevPath.GetWordsRange().GetStartPos(); size_t startPos = prevPath.GetWordsRange().GetStartPos();
if (endPos - startPos + 1 > maxPhraseLength) { if (endPos - startPos + 1 > maxPhraseLength) {
continue; continue;
} }
WordsRange range(startPos, endPos); WordsRange range(startPos, endPos);
const NonTerminalSet &labels = input.GetLabelSet(startPos, endPos); const NonTerminalSet &labels = input.GetLabelSet(startPos, endPos);
const Phrase &prevPhrase = prevPath.GetPhrase(); const Phrase &prevPhrase = prevPath.GetPhrase();
const ScorePair *prevInputScore = prevPath.GetInputScore(); const ScorePair *prevInputScore = prevPath.GetInputScore();
UTIL_THROW_IF2(prevInputScore == NULL, UTIL_THROW_IF2(prevInputScore == NULL,
"Null previous score"); "Null previous score");
// loop thru every word at this position // loop thru every word at this position
const ConfusionNet::Column &col = input.GetColumn(endPos); const ConfusionNet::Column &col = input.GetColumn(endPos);
for (size_t i = 0; i < col.size(); ++i) { for (size_t i = 0; i < col.size(); ++i) {
const Word &word = col[i].first; const Word &word = col[i].first;
Phrase subphrase(prevPhrase); Phrase subphrase(prevPhrase);
subphrase.AddWord(word); subphrase.AddWord(word);
const ScorePair &scores = col[i].second; const ScorePair &scores = col[i].second;
ScorePair *inputScore = new ScorePair(*prevInputScore); ScorePair *inputScore = new ScorePair(*prevInputScore);
inputScore->PlusEquals(scores); inputScore->PlusEquals(scores);
InputPath *path = new InputPath(subphrase, labels, range, &prevPath, inputScore); InputPath *path = new InputPath(subphrase, labels, range, &prevPath, inputScore);
size_t nextNode = nextNodes[i]; size_t nextNode = nextNodes[i];
path->SetNextNode(nextNode); path->SetNextNode(nextNode);
m_inputPathQueue.push_back(path); m_inputPathQueue.push_back(path);
} // for (size_t i = 0; i < col.size(); ++i) { } // for (size_t i = 0; i < col.size(); ++i) {
} // for (size_t i = 0; i < numPrevPaths; ++i) { } // for (size_t i = 0; i < numPrevPaths; ++i) {
} }
} }
@ -134,19 +134,18 @@ void TranslationOptionCollectionLattice::CreateTranslationOptions()
const WordsRange &range = path.GetWordsRange(); const WordsRange &range = path.GetWordsRange();
if (tpColl) { if (tpColl) {
TargetPhraseCollection::const_iterator iter; TargetPhraseCollection::const_iterator iter;
for (iter = tpColl->begin(); iter != tpColl->end(); ++iter) { for (iter = tpColl->begin(); iter != tpColl->end(); ++iter) {
const TargetPhrase &tp = **iter; const TargetPhrase &tp = **iter;
TranslationOption *transOpt = new TranslationOption(range, tp); TranslationOption *transOpt = new TranslationOption(range, tp);
transOpt->SetInputPath(path); transOpt->SetInputPath(path);
transOpt->Evaluate(m_source); transOpt->Evaluate(m_source);
Add(transOpt); Add(transOpt);
} }
} } else if (path.GetPhrase().GetSize() == 1) {
else if (path.GetPhrase().GetSize() == 1) { // unknown word processing
// unknown word processing ProcessOneUnknownWord(path, path.GetWordsRange().GetEndPos(), 1, path.GetInputScore());
ProcessOneUnknownWord(path, path.GetWordsRange().GetEndPos(), 1, path.GetInputScore());
} }
} }
@ -165,16 +164,16 @@ void TranslationOptionCollectionLattice::CreateTranslationOptions()
void TranslationOptionCollectionLattice::ProcessUnknownWord(size_t sourcePos) void TranslationOptionCollectionLattice::ProcessUnknownWord(size_t sourcePos)
{ {
UTIL_THROW(util::Exception, "ProcessUnknownWord() not implemented for lattice"); UTIL_THROW(util::Exception, "ProcessUnknownWord() not implemented for lattice");
} }
void TranslationOptionCollectionLattice::CreateTranslationOptionsForRange(const DecodeGraph &decodeStepList void TranslationOptionCollectionLattice::CreateTranslationOptionsForRange(const DecodeGraph &decodeStepList
, size_t startPosition , size_t startPosition
, size_t endPosition , size_t endPosition
, bool adhereTableLimit , bool adhereTableLimit
, size_t graphInd) , size_t graphInd)
{ {
UTIL_THROW(util::Exception, "CreateTranslationOptionsForRange() not implemented for lattice"); UTIL_THROW(util::Exception, "CreateTranslationOptionsForRange() not implemented for lattice");
} }
} // namespace } // namespace

View File

@ -15,10 +15,10 @@ class WordLattice;
class TranslationOptionCollectionLattice : public TranslationOptionCollection class TranslationOptionCollectionLattice : public TranslationOptionCollection
{ {
protected: protected:
/* forcibly create translation option for a 1 word. /* forcibly create translation option for a 1 word.
* call the base class' ProcessOneUnknownWord() for each possible word in the confusion network * call the base class' ProcessOneUnknownWord() for each possible word in the confusion network
* at a particular source position * at a particular source position
*/ */
void ProcessUnknownWord(size_t sourcePos); // do not implement void ProcessUnknownWord(size_t sourcePos); // do not implement
public: public:
@ -27,10 +27,10 @@ public:
void CreateTranslationOptions(); void CreateTranslationOptions();
void CreateTranslationOptionsForRange(const DecodeGraph &decodeStepList void CreateTranslationOptionsForRange(const DecodeGraph &decodeStepList
, size_t startPosition , size_t startPosition
, size_t endPosition , size_t endPosition
, bool adhereTableLimit , bool adhereTableLimit
, size_t graphInd); // do not implement , size_t graphInd); // do not implement
protected: protected:

View File

@ -54,7 +54,7 @@ public:
} }
void Remove( size_t ind ) { void Remove( size_t ind ) {
UTIL_THROW_IF2(ind >= m_coll.size(), UTIL_THROW_IF2(ind >= m_coll.size(),
"Out of bound index " << ind); "Out of bound index " << ind);
m_coll.erase( m_coll.begin()+ind ); m_coll.erase( m_coll.begin()+ind );
} }
void Add(TranslationOption *transOpt) { void Add(TranslationOption *transOpt) {

View File

@ -184,7 +184,7 @@ bool TreeInput::ProcessAndStripXMLTags(string &line, std::vector<XMLParseOutput>
Word *targetLHS = new Word(true); Word *targetLHS = new Word(true);
targetLHS->CreateFromString(Output, outputFactorOrder, targetLHSstr, true); targetLHS->CreateFromString(Output, outputFactorOrder, targetLHSstr, true);
UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL,
"Null factor left-hand-side"); "Null factor left-hand-side");
targetPhrase.SetTargetLHS(targetLHS); targetPhrase.SetTargetLHS(targetLHS);
// not tested // not tested
@ -289,7 +289,7 @@ void TreeInput::AddChartLabel(size_t startPos, size_t endPos, const Word &label
, const std::vector<FactorType>& /* factorOrder */) , const std::vector<FactorType>& /* factorOrder */)
{ {
UTIL_THROW_IF2(!label.IsNonTerminal(), UTIL_THROW_IF2(!label.IsNonTerminal(),
"Label must be a non-terminal"); "Label must be a non-terminal");
SourceLabelOverlap overlapType = StaticData::Instance().GetSourceLabelOverlap(); SourceLabelOverlap overlapType = StaticData::Instance().GetSourceLabelOverlap();
NonTerminalSet &list = GetLabelSet(startPos, endPos); NonTerminalSet &list = GetLabelSet(startPos, endPos);
@ -331,7 +331,7 @@ std::ostream& operator<<(std::ostream &out, const TreeInput &input)
for (iter = labelSet.begin(); iter != labelSet.end(); ++iter) { for (iter = labelSet.begin(); iter != labelSet.end(); ++iter) {
const Word &word = *iter; const Word &word = *iter;
UTIL_THROW_IF2(!word.IsNonTerminal(), UTIL_THROW_IF2(!word.IsNonTerminal(),
"Word must be a non-terminal"); "Word must be a non-terminal");
out << "[" << startPos <<"," << endPos << "]=" out << "[" << startPos <<"," << endPos << "]="
<< word << "(" << word.IsNonTerminal() << ") "; << word << "(" << word.IsNonTerminal() << ") ";
} }

View File

@ -200,7 +200,7 @@ Phrase TrellisPath::GetSurfacePhrase() const
FactorType factorType = outputFactor[i]; FactorType factorType = outputFactor[i];
const Factor *factor = targetPhrase.GetFactor(pos, factorType); const Factor *factor = targetPhrase.GetFactor(pos, factorType);
UTIL_THROW_IF2(factor == NULL, UTIL_THROW_IF2(factor == NULL,
"No factor " << factorType << " at position " << pos); "No factor " << factorType << " at position " << pos);
newWord[factorType] = factor; newWord[factorType] = factor;
} }
} }

View File

@ -108,30 +108,30 @@ enum DistortionOrientationOptions {
enum PhraseTableImplementation { enum PhraseTableImplementation {
Memory = 0 Memory = 0
,Binary = 1 ,Binary = 1
,OnDisk = 2 ,OnDisk = 2
//,GlueRule = 3 //,GlueRule = 3
//,Joshua = 4 //,Joshua = 4
//,MemorySourceLabel = 5 //,MemorySourceLabel = 5
,SCFG = 6 ,SCFG = 6
//,BerkeleyDb = 7 //,BerkeleyDb = 7
,SuffixArray = 8 ,SuffixArray = 8
,Hiero = 9 ,Hiero = 9
,ALSuffixArray = 10 ,ALSuffixArray = 10
,FuzzyMatch = 11 ,FuzzyMatch = 11
,Compact = 12 ,Compact = 12
,Interpolated = 13 ,Interpolated = 13
,DSuffixArray = 14 ,DSuffixArray = 14
,MemMappedSA = 15 ,MemMappedSA = 15
,DCacheBased = 32 ,DCacheBased = 32
}; };
enum InputTypeEnum { enum InputTypeEnum {
SentenceInput = 0 SentenceInput = 0
,ConfusionNetworkInput = 1 ,ConfusionNetworkInput = 1
,WordLatticeInput = 2 ,WordLatticeInput = 2
,TreeInputType = 3 ,TreeInputType = 3
,WordLatticeInput2 = 4 ,WordLatticeInput2 = 4
}; };
@ -145,27 +145,27 @@ enum XmlInputType {
enum DictionaryFind { enum DictionaryFind {
Best = 0 Best = 0
,All = 1 ,All = 1
}; };
enum SearchAlgorithm { enum SearchAlgorithm {
Normal = 0 Normal = 0
,CubePruning = 1 ,CubePruning = 1
,CubeGrowing = 2 ,CubeGrowing = 2
,ChartDecoding= 3 ,ChartDecoding= 3
,NormalBatch = 4 ,NormalBatch = 4
,ChartIncremental = 5 ,ChartIncremental = 5
}; };
enum SourceLabelOverlap { enum SourceLabelOverlap {
SourceLabelOverlapAdd = 0 SourceLabelOverlapAdd = 0
,SourceLabelOverlapReplace = 1 ,SourceLabelOverlapReplace = 1
,SourceLabelOverlapDiscard = 2 ,SourceLabelOverlapDiscard = 2
}; };
enum WordAlignmentSort { enum WordAlignmentSort {
NoSort = 0 NoSort = 0
,TargetOrder = 1 ,TargetOrder = 1
}; };
enum FormatType { enum FormatType {

View File

@ -316,7 +316,7 @@ inline float CalcTranslationScore(const std::vector<float> &probVector,
const std::vector<float> &weightT) const std::vector<float> &weightT)
{ {
UTIL_THROW_IF2(weightT.size() != probVector.size(), UTIL_THROW_IF2(weightT.size() != probVector.size(),
"Weight and score vector sizes not the same"); "Weight and score vector sizes not the same");
float rv=0.0; float rv=0.0;
for(float const *sb=&probVector[0],*se=sb+probVector.size(),*wb=&weightT[0]; for(float const *sb=&probVector[0],*se=sb+probVector.size(),*wb=&weightT[0];
sb!=se; ++sb, ++wb) sb!=se; ++sb, ++wb)

View File

@ -74,8 +74,8 @@ std::string Word::GetString(const vector<FactorType> factorType,bool endWithBlan
const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter(); const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter();
bool firstPass = true; bool firstPass = true;
for (unsigned int i = 0 ; i < factorType.size() ; i++) { for (unsigned int i = 0 ; i < factorType.size() ; i++) {
UTIL_THROW_IF2(factorType[i] >= MAX_NUM_FACTORS, UTIL_THROW_IF2(factorType[i] >= MAX_NUM_FACTORS,
"Trying to reference factor " << factorType[i] << ". Max factor is " << MAX_NUM_FACTORS); "Trying to reference factor " << factorType[i] << ". Max factor is " << MAX_NUM_FACTORS);
const Factor *factor = m_factorArray[factorType[i]]; const Factor *factor = m_factorArray[factorType[i]];
if (factor != NULL) { if (factor != NULL) {
@ -143,10 +143,10 @@ void Word::OnlyTheseFactors(const FactorMask &factors)
bool Word::IsEpsilon() const bool Word::IsEpsilon() const
{ {
const Factor *factor = m_factorArray[0]; const Factor *factor = m_factorArray[0];
int compare = factor->GetString().compare(EPSILON); int compare = factor->GetString().compare(EPSILON);
return compare == 0; return compare == 0;
} }
TO_STRING_BODY(Word); TO_STRING_BODY(Word);

View File

@ -13,7 +13,7 @@ namespace Moses
WordLattice::WordLattice() WordLattice::WordLattice()
{ {
UTIL_THROW_IF2(&InputFeature::Instance() == NULL, UTIL_THROW_IF2(&InputFeature::Instance() == NULL,
"Input feature must be specified"); "Input feature must be specified");
} }
size_t WordLattice::GetColumnIncrement(size_t i, size_t j) const size_t WordLattice::GetColumnIncrement(size_t i, size_t j) const
@ -216,9 +216,8 @@ WordLattice::CreateTranslationOptionCollection() const
if (StaticData::Instance().GetUseLegacyPT()) { if (StaticData::Instance().GetUseLegacyPT()) {
rv = new TranslationOptionCollectionConfusionNet(*this, maxNoTransOptPerCoverage, translationOptionThreshold); rv = new TranslationOptionCollectionConfusionNet(*this, maxNoTransOptPerCoverage, translationOptionThreshold);
} } else {
else { rv = new TranslationOptionCollectionLattice(*this, maxNoTransOptPerCoverage, translationOptionThreshold);
rv = new TranslationOptionCollectionLattice(*this, maxNoTransOptPerCoverage, translationOptionThreshold);
} }
assert(rv); assert(rv);

View File

@ -222,7 +222,7 @@ public:
//! converts bitmap into an integer ID, with an additional span covered //! converts bitmap into an integer ID, with an additional span covered
WordsBitmapID GetIDPlus( size_t startPos, size_t endPos ) const { WordsBitmapID GetIDPlus( size_t startPos, size_t endPos ) const {
assert(m_size < (1<<16)); assert(m_size < (1<<16));
size_t start = GetFirstGapPos(); size_t start = GetFirstGapPos();
if (start == NOT_FOUND) start = m_size; // nothing left if (start == NOT_FOUND) start = m_size; // nothing left