moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-08-16 15:00:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

Minor modification of the phrase properties framework.

Browse Source 
Properties can save memory by not storing the value string.
This commit is contained in:
Matthias Huck 2014-06-13 16:37:13 +01:00
parent a6f8a7c4c9
commit a5467d89c4
11 changed files with 41 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
moses-chart-cmd/IOWrapper.cpp
View File

@ -412,9 +412,10 @@ void IOWrapper::OutputTreeFragmentsTranslationOptions(std::ostream &out, Applica
OutputTranslationOption(out, applicationContext, hypo, sentence, translationId);
const TargetPhrase &currTarPhr = hypo->GetCurrTargetPhrase();
boost::shared_ptr<PhraseProperty> property;
out << " ||| ";
if (const PhraseProperty *property = currTarPhr.GetProperty("Tree")) {
if (currTarPhr.GetProperty("Tree", property)) {
out << " " << property->GetValueString();
} else {
out << " " << "noTreeInfo";
@ -438,9 +439,10 @@ void IOWrapper::OutputTreeFragmentsTranslationOptions(std::ostream &out, Applica
OutputTranslationOption(out, applicationContext, applied, sentence, translationId);
const TargetPhrase &currTarPhr = *static_cast<const TargetPhrase*>(applied->GetNote().vp);
boost::shared_ptr<PhraseProperty> property;
out << " ||| ";
if (const PhraseProperty *property = currTarPhr.GetProperty("Tree")) {
if (currTarPhr.GetProperty("Tree", property)) {
out << " " << property->GetValueString();
} else {
out << " " << "noTreeInfo";

7
moses/FF/TreeStructureFeature.cpp
View File

@ -271,9 +271,10 @@ FFState* TreeStructureFeature::EvaluateChart(const ChartHypothesis& cur_hypo
, int featureID /* used to index the state in the previous hypotheses */
, ScoreComponentCollection* accumulator) const
{
if (const PhraseProperty *property = cur_hypo.GetCurrTargetPhrase().GetProperty("Tree")) {
const std::string &tree = property->GetValueString();
TreePointer mytree (new InternalTree(tree));
boost::shared_ptr<PhraseProperty> property;
if (cur_hypo.GetCurrTargetPhrase().GetProperty("Tree", property)) {
const std::string *tree = property->GetValueString();
TreePointer mytree (new InternalTree(*tree));
if (m_labelset) {
AddNTLabels(mytree);

6
moses/PP/CountsPhraseProperty.cpp
View File

@ -1,14 +1,13 @@
#include "moses/PP/CountsPhraseProperty.h"
#include <sstream>
#include <assert.h>
#include "util/exception.hh"
namespace Moses
{
void CountsPhraseProperty::ProcessValue()
void CountsPhraseProperty::ProcessValue(const std::string &value)
{
std::istringstream tokenizer(m_value);
std::istringstream tokenizer(value);
if (! (tokenizer >> m_targetMarginal)) { // first token: countE
UTIL_THROW2("CountsPhraseProperty: Not able to read target marginal. Flawed property?");
@ -24,7 +23,6 @@ void CountsPhraseProperty::ProcessValue()
UTIL_THROW2("CountsPhraseProperty: Not able to read joint count. Flawed property?");
}
assert( m_jointCount > 0 );
};
} // namespace Moses

10
moses/PP/CountsPhraseProperty.h
View File

@ -2,6 +2,7 @@
#pragma once
#include "moses/PP/PhraseProperty.h"
#include "util/exception.hh"
#include <string>
#include <list>
@ -28,9 +29,9 @@ class CountsPhraseProperty : public PhraseProperty
{
public:
CountsPhraseProperty(const std::string &value) : PhraseProperty(value) {};
CountsPhraseProperty() {};
virtual void ProcessValue();
virtual void ProcessValue(const std::string &value);
size_t GetSourceMarginal() const {
return m_sourceMarginal;
@ -44,6 +45,11 @@ public:
return m_jointCount;
}
virtual const std::string *GetValueString() const {
UTIL_THROW2("CountsPhraseProperty: value string not available in this phrase property");
return NULL;
};
protected:
float m_sourceMarginal, m_targetMarginal, m_jointCount;

6
moses/PP/Factory.cpp
View File

@ -36,8 +36,8 @@ template <class P> class DefaultPhrasePropertyCreator : public PhrasePropertyCre
{
public:
boost::shared_ptr<PhraseProperty> CreateProperty(const std::string &value) {
P* property = new P(value);
property->ProcessValue();
P* property = new P();
property->ProcessValue(value);
return Create(property);
}
};
@ -54,7 +54,7 @@ PhrasePropertyFactory::PhrasePropertyFactory()
MOSES_PNAME2("Counts", CountsPhraseProperty);
MOSES_PNAME2("SourceLabels", SourceLabelsPhraseProperty);
MOSES_PNAME2("Tree",TreeStructurePhraseProperty);
MOSES_PNAME2("Tree", TreeStructurePhraseProperty);
}

8
moses/PP/PhraseProperty.h
View File

@ -11,15 +11,15 @@ namespace Moses
class PhraseProperty
{
public:
PhraseProperty(const std::string &value) : m_value(value) {};
PhraseProperty() {};
virtual void ProcessValue() {};
virtual void ProcessValue(const std::string &value) { m_value = new std::string(value); };
const std::string &GetValueString() const { return m_value; };
virtual const std::string *GetValueString() const { return m_value; };
protected:
const std::string m_value;
std::string *m_value;
};

5
moses/PP/SourceLabelsPhraseProperty.cpp
View File

@ -7,14 +7,13 @@
#include <queue>
#include <assert.h>
#include <limits>
#include "util/exception.hh"
namespace Moses
{
void SourceLabelsPhraseProperty::ProcessValue()
void SourceLabelsPhraseProperty::ProcessValue(const std::string &value)
{
std::istringstream tokenizer(m_value);
std::istringstream tokenizer(value);
if (! (tokenizer >> m_nNTs)) { // first token: number of non-terminals (incl. left-hand side)
UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read number of non-terminals. Flawed property?");

10
moses/PP/SourceLabelsPhraseProperty.h
View File

@ -2,6 +2,7 @@
#pragma once
#include "moses/PP/PhraseProperty.h"
#include "util/exception.hh"
#include <string>
#include <list>
@ -43,9 +44,9 @@ private:
class SourceLabelsPhraseProperty : public PhraseProperty
{
public:
SourceLabelsPhraseProperty(const std::string &value) : PhraseProperty(value) {};
SourceLabelsPhraseProperty() {};
virtual void ProcessValue();
virtual void ProcessValue(const std::string &value);
size_t GetNumberOfNonTerminals() const {
return m_nNTs;
@ -59,6 +60,11 @@ public:
return m_sourceLabelItems;
};
virtual const std::string *GetValueString() const {
UTIL_THROW2("SourceLabelsPhraseProperty: value string not available in this phrase property");
return NULL;
};
protected:
size_t m_nNTs;

2
moses/PP/TreeStructurePhraseProperty.h
View File

@ -10,7 +10,7 @@ namespace Moses
class TreeStructurePhraseProperty : public PhraseProperty
{
public:
TreeStructurePhraseProperty(const std::string &value) : PhraseProperty(value) {};
TreeStructurePhraseProperty() {};
};

8
moses/TargetPhrase.cpp
View File

@ -248,15 +248,15 @@ void TargetPhrase::SetProperty(const std::string &key, const std::string &value)
m_properties[key] = phrasePropertyFactory.ProduceProperty(key,value);
}
const PhraseProperty *TargetPhrase::GetProperty(const std::string &key) const
bool TargetPhrase::GetProperty(const std::string &key, boost::shared_ptr<PhraseProperty> &pp) const
{
std::map<std::string, boost::shared_ptr<PhraseProperty> >::const_iterator iter;
iter = m_properties.find(key);
if (iter != m_properties.end()) {
const boost::shared_ptr<PhraseProperty> &value = iter->second;
return value.get();
pp = iter->second;
return true;
}
return NULL;
return false;
}
void TargetPhrase::SetRuleSource(const Phrase &ruleSource) const

2
moses/TargetPhrase.h
View File

@ -137,7 +137,7 @@ public:
void SetProperties(const StringPiece &str);
void SetProperty(const std::string &key, const std::string &value);
const PhraseProperty *GetProperty(const std::string &key) const;
bool GetProperty(const std::string &key, boost::shared_ptr<PhraseProperty> &pp) const;
void Merge(const TargetPhrase &copy, const std::vector<FactorType>& factorVec);