mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-12-26 13:23:25 +03:00
Code cleanup and refactoring.
This commit is contained in:
parent
240b88c683
commit
29694af6e4
@ -51,7 +51,7 @@ int main(int argc, char **argv)
|
|||||||
std::stringstream ss;
|
std::stringstream ss;
|
||||||
ss << nscores;
|
ss << nscores;
|
||||||
PhraseDictionaryCompact pdc("PhraseDictionaryCompact input-factor=0 output-factor=0 num-features=" + ss.str() + " path=" + ttable);
|
PhraseDictionaryCompact pdc("PhraseDictionaryCompact input-factor=0 output-factor=0 num-features=" + ss.str() + " path=" + ttable);
|
||||||
AllOptions opts;
|
AllOptions::ptr opts(new AllOptions);
|
||||||
pdc.Load(opts);
|
pdc.Load(opts);
|
||||||
|
|
||||||
std::string line;
|
std::string line;
|
||||||
|
@ -163,11 +163,12 @@ int main(int argc, char const* argv[])
|
|||||||
}
|
}
|
||||||
|
|
||||||
StaticData& SD = const_cast<StaticData&>(StaticData::Instance());
|
StaticData& SD = const_cast<StaticData&>(StaticData::Instance());
|
||||||
LMBR_Options& lmbr = SD.options().lmbr;
|
boost::shared_ptr<AllOptions> opts(new AllOptions(*SD.options()));
|
||||||
MBR_Options& mbr = SD.options().mbr;
|
LMBR_Options& lmbr = opts->lmbr;
|
||||||
|
MBR_Options& mbr = opts->mbr;
|
||||||
lmbr.enabled = true;
|
lmbr.enabled = true;
|
||||||
|
|
||||||
boost::shared_ptr<IOWrapper> ioWrapper(new IOWrapper(SD.options()));
|
boost::shared_ptr<IOWrapper> ioWrapper(new IOWrapper(*opts));
|
||||||
if (!ioWrapper) {
|
if (!ioWrapper) {
|
||||||
throw runtime_error("Failed to initialise IOWrapper");
|
throw runtime_error("Failed to initialise IOWrapper");
|
||||||
}
|
}
|
||||||
|
@ -125,7 +125,7 @@ int main(int argc, char const** argv)
|
|||||||
IFVERBOSE(1) {
|
IFVERBOSE(1) {
|
||||||
PrintUserTime("Created input-output object");
|
PrintUserTime("Created input-output object");
|
||||||
}
|
}
|
||||||
AllOptions::ptr opts(new AllOptions(StaticData::Instance().options()));
|
AllOptions::ptr opts(new AllOptions(*StaticData::Instance().options()));
|
||||||
boost::shared_ptr<IOWrapper> ioWrapper(new IOWrapper(*opts));
|
boost::shared_ptr<IOWrapper> ioWrapper(new IOWrapper(*opts));
|
||||||
if (ioWrapper == NULL) {
|
if (ioWrapper == NULL) {
|
||||||
cerr << "Error; Failed to create IO object" << endl;
|
cerr << "Error; Failed to create IO object" << endl;
|
||||||
|
@ -139,7 +139,8 @@ BackwardsEdge::BackwardsEdge(const BitmapContainer &prevBitmapContainer
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Fetch the things we need for distortion cost computation.
|
// Fetch the things we need for distortion cost computation.
|
||||||
int maxDistortion = StaticData::Instance().GetMaxDistortion();
|
// int maxDistortion = StaticData::Instance().GetMaxDistortion();
|
||||||
|
int maxDistortion = itype.options()->reordering.max_distortion;
|
||||||
|
|
||||||
if (maxDistortion == -1) {
|
if (maxDistortion == -1) {
|
||||||
for (HypothesisSet::const_iterator iter = m_prevBitmapContainer.GetHypotheses().begin(); iter != m_prevBitmapContainer.GetHypotheses().end(); ++iter) {
|
for (HypothesisSet::const_iterator iter = m_prevBitmapContainer.GetHypotheses().begin(); iter != m_prevBitmapContainer.GetHypotheses().end(); ++iter) {
|
||||||
|
@ -102,7 +102,7 @@ ChartHypothesis::~ChartHypothesis()
|
|||||||
*/
|
*/
|
||||||
void ChartHypothesis::GetOutputPhrase(Phrase &outPhrase) const
|
void ChartHypothesis::GetOutputPhrase(Phrase &outPhrase) const
|
||||||
{
|
{
|
||||||
FactorType placeholderFactor = StaticData::Instance().options().input.placeholder_factor;
|
FactorType placeholderFactor = StaticData::Instance().options()->input.placeholder_factor;
|
||||||
|
|
||||||
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos) {
|
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos) {
|
||||||
const Word &word = GetCurrTargetPhrase().GetWord(pos);
|
const Word &word = GetCurrTargetPhrase().GetWord(pos);
|
||||||
@ -256,7 +256,7 @@ void ChartHypothesis::CleanupArcList()
|
|||||||
* However, may not be enough if only unique candidates are needed,
|
* However, may not be enough if only unique candidates are needed,
|
||||||
* so we'll keep all of arc list if nedd distinct n-best list
|
* so we'll keep all of arc list if nedd distinct n-best list
|
||||||
*/
|
*/
|
||||||
AllOptions const& opts = StaticData::Instance().options();
|
AllOptions const& opts = *StaticData::Instance().options();
|
||||||
size_t nBestSize = opts.nbest.nbest_size;
|
size_t nBestSize = opts.nbest.nbest_size;
|
||||||
bool distinctNBest = (opts.nbest.only_distinct
|
bool distinctNBest = (opts.nbest.only_distinct
|
||||||
|| opts.mbr.enabled
|
|| opts.mbr.enabled
|
||||||
@ -336,7 +336,7 @@ std::ostream& operator<<(std::ostream& out, const ChartHypothesis& hypo)
|
|||||||
out << "->" << hypo.GetWinningHypothesis()->GetId();
|
out << "->" << hypo.GetWinningHypothesis()->GetId();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (StaticData::Instance().GetIncludeLHSInSearchGraph()) {
|
if (hypo.GetManager().options()->output.include_lhs_in_search_graph) {
|
||||||
out << " " << hypo.GetTargetLHS() << "=>";
|
out << " " << hypo.GetTargetLHS() << "=>";
|
||||||
}
|
}
|
||||||
out << " " << hypo.GetCurrTargetPhrase()
|
out << " " << hypo.GetCurrTargetPhrase()
|
||||||
|
@ -82,7 +82,7 @@ void ChartKBestExtractor::Extract(
|
|||||||
// Generate the target-side yield of the derivation d.
|
// Generate the target-side yield of the derivation d.
|
||||||
Phrase ChartKBestExtractor::GetOutputPhrase(const Derivation &d)
|
Phrase ChartKBestExtractor::GetOutputPhrase(const Derivation &d)
|
||||||
{
|
{
|
||||||
FactorType placeholderFactor = StaticData::Instance().options().input.placeholder_factor;
|
FactorType placeholderFactor = StaticData::Instance().options()->input.placeholder_factor;
|
||||||
|
|
||||||
Phrase ret(ARRAY_SIZE_INCR);
|
Phrase ret(ARRAY_SIZE_INCR);
|
||||||
|
|
||||||
|
@ -44,17 +44,26 @@ ChartParserUnknown
|
|||||||
ChartParserUnknown::~ChartParserUnknown()
|
ChartParserUnknown::~ChartParserUnknown()
|
||||||
{
|
{
|
||||||
RemoveAllInColl(m_unksrcs);
|
RemoveAllInColl(m_unksrcs);
|
||||||
// RemoveAllInColl(m_cacheTargetPhraseCollection);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void ChartParserUnknown::Process(const Word &sourceWord, const Range &range, ChartParserCallback &to)
|
AllOptions::ptr const&
|
||||||
|
ChartParserUnknown::
|
||||||
|
options() const
|
||||||
|
{
|
||||||
|
return m_ttask.lock()->options();
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
ChartParserUnknown::
|
||||||
|
Process(const Word &sourceWord, const Range &range, ChartParserCallback &to)
|
||||||
{
|
{
|
||||||
// unknown word, add as trans opt
|
// unknown word, add as trans opt
|
||||||
const StaticData &staticData = StaticData::Instance();
|
const StaticData &staticData = StaticData::Instance();
|
||||||
const UnknownWordPenaltyProducer &unknownWordPenaltyProducer = UnknownWordPenaltyProducer::Instance();
|
const UnknownWordPenaltyProducer &unknownWordPenaltyProducer
|
||||||
|
= UnknownWordPenaltyProducer::Instance();
|
||||||
|
|
||||||
size_t isDigit = 0;
|
size_t isDigit = 0;
|
||||||
if (staticData.options().unk.drop) {
|
if (options()->unk.drop) {
|
||||||
const Factor *f = sourceWord[0]; // TODO hack. shouldn't know which factor is surface
|
const Factor *f = sourceWord[0]; // TODO hack. shouldn't know which factor is surface
|
||||||
const StringPiece s = f->GetString();
|
const StringPiece s = f->GetString();
|
||||||
isDigit = s.find_first_of("0123456789");
|
isDigit = s.find_first_of("0123456789");
|
||||||
@ -79,9 +88,9 @@ void ChartParserUnknown::Process(const Word &sourceWord, const Range &range, Cha
|
|||||||
}
|
}
|
||||||
|
|
||||||
//TranslationOption *transOpt;
|
//TranslationOption *transOpt;
|
||||||
if (! staticData.options().unk.drop || isDigit) {
|
if (! options()->unk.drop || isDigit) {
|
||||||
// loop
|
// loop
|
||||||
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
|
const UnknownLHSList &lhsList = options()->syntax.unknown_lhs; // staticData.GetUnknownLHS();
|
||||||
UnknownLHSList::const_iterator iterLHS;
|
UnknownLHSList::const_iterator iterLHS;
|
||||||
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
|
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
|
||||||
const string &targetLHSStr = iterLHS->first;
|
const string &targetLHSStr = iterLHS->first;
|
||||||
@ -91,7 +100,7 @@ void ChartParserUnknown::Process(const Word &sourceWord, const Range &range, Cha
|
|||||||
//const Word &sourceLHS = staticData.GetInputDefaultNonTerminal();
|
//const Word &sourceLHS = staticData.GetInputDefaultNonTerminal();
|
||||||
Word *targetLHS = new Word(true);
|
Word *targetLHS = new Word(true);
|
||||||
|
|
||||||
targetLHS->CreateFromString(Output, staticData.options().output.factor_order,
|
targetLHS->CreateFromString(Output, options()->output.factor_order,
|
||||||
targetLHSStr, true);
|
targetLHSStr, true);
|
||||||
UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS");
|
UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS");
|
||||||
|
|
||||||
@ -108,9 +117,8 @@ void ChartParserUnknown::Process(const Word &sourceWord, const Range &range, Cha
|
|||||||
targetPhrase->SetAlignmentInfo("0-0");
|
targetPhrase->SetAlignmentInfo("0-0");
|
||||||
targetPhrase->EvaluateInIsolation(*unksrc);
|
targetPhrase->EvaluateInIsolation(*unksrc);
|
||||||
|
|
||||||
AllOptions const& opts = staticData.options();
|
if (!options()->output.detailed_tree_transrep_filepath.empty() ||
|
||||||
if (!opts.output.detailed_tree_transrep_filepath.empty() ||
|
options()->nbest.print_trees || staticData.GetTreeStructure() != NULL) {
|
||||||
opts.nbest.print_trees || staticData.GetTreeStructure() != NULL) {
|
|
||||||
std::string prop = "[ ";
|
std::string prop = "[ ";
|
||||||
prop += (*targetLHS)[0]->GetString().as_string() + " ";
|
prop += (*targetLHS)[0]->GetString().as_string() + " ";
|
||||||
prop += sourceWord[0]->GetString().as_string() + " ]";
|
prop += sourceWord[0]->GetString().as_string() + " ]";
|
||||||
@ -126,14 +134,14 @@ void ChartParserUnknown::Process(const Word &sourceWord, const Range &range, Cha
|
|||||||
|
|
||||||
TargetPhrase *targetPhrase = new TargetPhrase(firstPt);
|
TargetPhrase *targetPhrase = new TargetPhrase(firstPt);
|
||||||
// loop
|
// loop
|
||||||
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
|
const UnknownLHSList &lhsList = options()->syntax.unknown_lhs;//staticData.GetUnknownLHS();
|
||||||
UnknownLHSList::const_iterator iterLHS;
|
UnknownLHSList::const_iterator iterLHS;
|
||||||
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
|
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
|
||||||
const string &targetLHSStr = iterLHS->first;
|
const string &targetLHSStr = iterLHS->first;
|
||||||
//float prob = iterLHS->second;
|
//float prob = iterLHS->second;
|
||||||
|
|
||||||
Word *targetLHS = new Word(true);
|
Word *targetLHS = new Word(true);
|
||||||
targetLHS->CreateFromString(Output, staticData.options().output.factor_order,
|
targetLHS->CreateFromString(Output, staticData.options()->output.factor_order,
|
||||||
targetLHSStr, true);
|
targetLHSStr, true);
|
||||||
UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS");
|
UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS");
|
||||||
|
|
||||||
@ -214,9 +222,7 @@ void ChartParser::Create(const Range &range, ChartParserCallback &to)
|
|||||||
if (range.GetNumWordsCovered() == 1
|
if (range.GetNumWordsCovered() == 1
|
||||||
&& range.GetStartPos() != 0
|
&& range.GetStartPos() != 0
|
||||||
&& range.GetStartPos() != m_source.GetSize()-1) {
|
&& range.GetStartPos() != m_source.GetSize()-1) {
|
||||||
bool always = m_ttask.lock()->options()->unk.always_create_direct_transopt;
|
bool always = options()->unk.always_create_direct_transopt;
|
||||||
// bool alwaysCreateDirectTranslationOption
|
|
||||||
// = StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
|
|
||||||
if (to.Empty() || always) {
|
if (to.Empty() || always) {
|
||||||
// create unknown words for 1 word coverage where we don't have any trans options
|
// create unknown words for 1 word coverage where we don't have any trans options
|
||||||
const Word &sourceWord = m_source.GetWord(range.GetStartPos());
|
const Word &sourceWord = m_source.GetWord(range.GetStartPos());
|
||||||
@ -291,4 +297,14 @@ long ChartParser::GetTranslationId() const
|
|||||||
{
|
{
|
||||||
return m_source.GetTranslationId();
|
return m_source.GetTranslationId();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
AllOptions::ptr const&
|
||||||
|
ChartParser::
|
||||||
|
options() const
|
||||||
|
{
|
||||||
|
return m_ttask.lock()->options();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
} // namespace Moses
|
} // namespace Moses
|
||||||
|
@ -57,6 +57,7 @@ public:
|
|||||||
private:
|
private:
|
||||||
std::vector<Phrase*> m_unksrcs;
|
std::vector<Phrase*> m_unksrcs;
|
||||||
std::list<TargetPhraseCollection::shared_ptr> m_cacheTargetPhraseCollection;
|
std::list<TargetPhraseCollection::shared_ptr> m_cacheTargetPhraseCollection;
|
||||||
|
AllOptions::ptr const& options() const;
|
||||||
};
|
};
|
||||||
|
|
||||||
class ChartParser
|
class ChartParser
|
||||||
@ -78,6 +79,8 @@ public:
|
|||||||
return m_unknown.GetUnknownSources();
|
return m_unknown.GetUnknownSources();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
AllOptions::ptr const& options() const;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
ChartParserUnknown m_unknown;
|
ChartParserUnknown m_unknown;
|
||||||
std::vector <DecodeGraph*> m_decodeGraphList;
|
std::vector <DecodeGraph*> m_decodeGraphList;
|
||||||
|
@ -66,9 +66,8 @@ ConfusionNet(AllOptions::ptr const& opts) : InputType(opts)
|
|||||||
{
|
{
|
||||||
stats.createOne();
|
stats.createOne();
|
||||||
|
|
||||||
const StaticData& SD = StaticData::Instance();
|
if (is_syntax(opts->search.algo)) {
|
||||||
if (SD.IsSyntax()) {
|
m_defaultLabelSet.insert(opts->syntax.input_default_non_terminal);
|
||||||
m_defaultLabelSet.insert(SD.GetInputDefaultNonTerminal());
|
|
||||||
}
|
}
|
||||||
UTIL_THROW_IF2(InputFeature::InstancePtr() == NULL, "Input feature must be specified");
|
UTIL_THROW_IF2(InputFeature::InstancePtr() == NULL, "Input feature must be specified");
|
||||||
}
|
}
|
||||||
@ -92,14 +91,14 @@ ConfusionNet(Sentence const& s) : InputType(s.options())
|
|||||||
|
|
||||||
bool
|
bool
|
||||||
ConfusionNet::
|
ConfusionNet::
|
||||||
ReadF(std::istream& in, const std::vector<FactorType>& factorOrder, int format)
|
ReadF(std::istream& in, int format)
|
||||||
{
|
{
|
||||||
VERBOSE(2, "read confusion net with format "<<format<<"\n");
|
VERBOSE(2, "read confusion net with format "<<format<<"\n");
|
||||||
switch(format) {
|
switch(format) {
|
||||||
case 0:
|
case 0:
|
||||||
return ReadFormat0(in,factorOrder);
|
return ReadFormat0(in);
|
||||||
case 1:
|
case 1:
|
||||||
return ReadFormat1(in,factorOrder);
|
return ReadFormat1(in);
|
||||||
default:
|
default:
|
||||||
std::cerr << "ERROR: unknown format '"<<format
|
std::cerr << "ERROR: unknown format '"<<format
|
||||||
<<"' in ConfusionNet::Read";
|
<<"' in ConfusionNet::Read";
|
||||||
@ -109,22 +108,20 @@ ReadF(std::istream& in, const std::vector<FactorType>& factorOrder, int format)
|
|||||||
|
|
||||||
int
|
int
|
||||||
ConfusionNet::
|
ConfusionNet::
|
||||||
Read(std::istream& in,
|
Read(std::istream& in)
|
||||||
const std::vector<FactorType>& factorOrder,
|
|
||||||
AllOptions const& opts)
|
|
||||||
{
|
{
|
||||||
int rv=ReadF(in,factorOrder,0);
|
int rv=ReadF(in,0);
|
||||||
if(rv) stats.collect(*this);
|
if(rv) stats.collect(*this);
|
||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool
|
bool
|
||||||
ConfusionNet::
|
ConfusionNet::
|
||||||
ReadFormat0(std::istream& in, const std::vector<FactorType>& factorOrder)
|
ReadFormat0(std::istream& in)
|
||||||
{
|
{
|
||||||
Clear();
|
Clear();
|
||||||
|
const std::vector<FactorType>& factorOrder = m_options->input.factor_order;
|
||||||
|
|
||||||
// const StaticData &staticData = StaticData::Instance();
|
|
||||||
const InputFeature *inputFeature = InputFeature::InstancePtr();
|
const InputFeature *inputFeature = InputFeature::InstancePtr();
|
||||||
size_t numInputScores = inputFeature->GetNumInputScores();
|
size_t numInputScores = inputFeature->GetNumInputScores();
|
||||||
size_t numRealWordCount = inputFeature->GetNumRealWordsInInput();
|
size_t numRealWordCount = inputFeature->GetNumRealWordsInInput();
|
||||||
@ -140,7 +137,6 @@ ReadFormat0(std::istream& in, const std::vector<FactorType>& factorOrder)
|
|||||||
Column col;
|
Column col;
|
||||||
while(is>>word) {
|
while(is>>word) {
|
||||||
Word w;
|
Word w;
|
||||||
// String2Word(word,w,factorOrder);
|
|
||||||
w.CreateFromString(Input,factorOrder,StringPiece(word),false,false);
|
w.CreateFromString(Input,factorOrder,StringPiece(word),false,false);
|
||||||
std::vector<float> probs(totalCount, 0.0);
|
std::vector<float> probs(totalCount, 0.0);
|
||||||
for(size_t i=0; i < numInputScores; i++) {
|
for(size_t i=0; i < numInputScores; i++) {
|
||||||
@ -179,9 +175,10 @@ ReadFormat0(std::istream& in, const std::vector<FactorType>& factorOrder)
|
|||||||
|
|
||||||
bool
|
bool
|
||||||
ConfusionNet::
|
ConfusionNet::
|
||||||
ReadFormat1(std::istream& in, const std::vector<FactorType>& factorOrder)
|
ReadFormat1(std::istream& in)
|
||||||
{
|
{
|
||||||
Clear();
|
Clear();
|
||||||
|
const std::vector<FactorType>& factorOrder = m_options->input.factor_order;
|
||||||
std::string line;
|
std::string line;
|
||||||
if(!getline(in,line)) return 0;
|
if(!getline(in,line)) return 0;
|
||||||
size_t s;
|
size_t s;
|
||||||
|
@ -30,8 +30,8 @@ protected:
|
|||||||
std::vector<Column> data;
|
std::vector<Column> data;
|
||||||
NonTerminalSet m_defaultLabelSet;
|
NonTerminalSet m_defaultLabelSet;
|
||||||
|
|
||||||
bool ReadFormat0(std::istream&,const std::vector<FactorType>& factorOrder);
|
bool ReadFormat0(std::istream&);
|
||||||
bool ReadFormat1(std::istream&,const std::vector<FactorType>& factorOrder);
|
bool ReadFormat1(std::istream&);
|
||||||
void String2Word(const std::string& s,Word& w,const std::vector<FactorType>& factorOrder);
|
void String2Word(const std::string& s,Word& w,const std::vector<FactorType>& factorOrder);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
@ -46,7 +46,8 @@ public:
|
|||||||
|
|
||||||
const Column& GetColumn(size_t i) const {
|
const Column& GetColumn(size_t i) const {
|
||||||
UTIL_THROW_IF2(i >= data.size(),
|
UTIL_THROW_IF2(i >= data.size(),
|
||||||
"Out of bounds. Trying to access " << i << " when vector only contains " << data.size());
|
"Out of bounds. Trying to access " << i
|
||||||
|
<< " when vector only contains " << data.size());
|
||||||
return data[i];
|
return data[i];
|
||||||
}
|
}
|
||||||
const Column& operator[](size_t i) const {
|
const Column& operator[](size_t i) const {
|
||||||
@ -64,11 +65,10 @@ public:
|
|||||||
data.clear();
|
data.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ReadF(std::istream&,const std::vector<FactorType>& factorOrder,int format=0);
|
bool ReadF(std::istream&, int format=0);
|
||||||
virtual void Print(std::ostream&) const;
|
virtual void Print(std::ostream&) const;
|
||||||
|
|
||||||
int Read(std::istream& in,const std::vector<FactorType>& factorOrder,
|
int Read(std::istream& in);
|
||||||
AllOptions const& opts);
|
|
||||||
|
|
||||||
Phrase GetSubString(const Range&) const; //TODO not defined
|
Phrase GetSubString(const Range&) const; //TODO not defined
|
||||||
std::string GetStringRep(const std::vector<FactorType> factorsToPrint) const; //TODO not defined
|
std::string GetStringRep(const std::vector<FactorType> factorsToPrint) const; //TODO not defined
|
||||||
|
@ -101,7 +101,7 @@ SimpleTranslationInterface::~SimpleTranslationInterface()
|
|||||||
//the simplified version of string input/output translation
|
//the simplified version of string input/output translation
|
||||||
string SimpleTranslationInterface::translate(const string &inputString)
|
string SimpleTranslationInterface::translate(const string &inputString)
|
||||||
{
|
{
|
||||||
boost::shared_ptr<Moses::IOWrapper> ioWrapper(new IOWrapper(StaticData::Instance().options()));
|
boost::shared_ptr<Moses::IOWrapper> ioWrapper(new IOWrapper(*StaticData::Instance().options()));
|
||||||
// main loop over set of input sentences
|
// main loop over set of input sentences
|
||||||
size_t sentEnd = inputString.rfind('\n'); //find the last \n, the input stream has to be appended with \n to be translated
|
size_t sentEnd = inputString.rfind('\n'); //find the last \n, the input stream has to be appended with \n to be translated
|
||||||
const string &newString = sentEnd != string::npos ? inputString : inputString + '\n';
|
const string &newString = sentEnd != string::npos ? inputString : inputString + '\n';
|
||||||
@ -180,7 +180,7 @@ batch_run()
|
|||||||
IFVERBOSE(1) PrintUserTime("Created input-output object");
|
IFVERBOSE(1) PrintUserTime("Created input-output object");
|
||||||
|
|
||||||
// set up read/writing class:
|
// set up read/writing class:
|
||||||
boost::shared_ptr<IOWrapper> ioWrapper(new IOWrapper(staticData.options()));
|
boost::shared_ptr<IOWrapper> ioWrapper(new IOWrapper(*staticData.options()));
|
||||||
UTIL_THROW_IF2(ioWrapper == NULL, "Error; Failed to create IO object"
|
UTIL_THROW_IF2(ioWrapper == NULL, "Error; Failed to create IO object"
|
||||||
<< " [" << HERE << "]");
|
<< " [" << HERE << "]");
|
||||||
|
|
||||||
|
@ -46,11 +46,12 @@ ConstrainedDecoding::ConstrainedDecoding(const std::string &line)
|
|||||||
ReadParameters();
|
ReadParameters();
|
||||||
}
|
}
|
||||||
|
|
||||||
void ConstrainedDecoding::Load(AllOptions const& opts)
|
void ConstrainedDecoding::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
const StaticData &staticData = StaticData::Instance();
|
const StaticData &staticData = StaticData::Instance();
|
||||||
bool addBeginEndWord
|
bool addBeginEndWord
|
||||||
= ((opts.search.algo == CYKPlus) || (opts.search.algo == ChartIncremental));
|
= ((opts->search.algo == CYKPlus) || (opts->search.algo == ChartIncremental));
|
||||||
|
|
||||||
for(size_t i = 0; i < m_paths.size(); ++i) {
|
for(size_t i = 0; i < m_paths.size(); ++i) {
|
||||||
InputFileStream constraintFile(m_paths[i]);
|
InputFileStream constraintFile(m_paths[i]);
|
||||||
@ -62,10 +63,10 @@ void ConstrainedDecoding::Load(AllOptions const& opts)
|
|||||||
Phrase phrase(0);
|
Phrase phrase(0);
|
||||||
if (vecStr.size() == 1) {
|
if (vecStr.size() == 1) {
|
||||||
sentenceID++;
|
sentenceID++;
|
||||||
phrase.CreateFromString(Output, opts.output.factor_order, vecStr[0], NULL);
|
phrase.CreateFromString(Output, opts->output.factor_order, vecStr[0], NULL);
|
||||||
} else if (vecStr.size() == 2) {
|
} else if (vecStr.size() == 2) {
|
||||||
sentenceID = Scan<long>(vecStr[0]);
|
sentenceID = Scan<long>(vecStr[0]);
|
||||||
phrase.CreateFromString(Output, opts.output.factor_order, vecStr[1], NULL);
|
phrase.CreateFromString(Output, opts->output.factor_order, vecStr[1], NULL);
|
||||||
} else {
|
} else {
|
||||||
UTIL_THROW(util::Exception, "Reference file not loaded");
|
UTIL_THROW(util::Exception, "Reference file not loaded");
|
||||||
}
|
}
|
||||||
|
@ -36,7 +36,7 @@ class ConstrainedDecoding : public StatefulFeatureFunction
|
|||||||
public:
|
public:
|
||||||
ConstrainedDecoding(const std::string &line);
|
ConstrainedDecoding(const std::string &line);
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
bool IsUseable(const FactorMask &mask) const {
|
bool IsUseable(const FactorMask &mask) const {
|
||||||
return true;
|
return true;
|
||||||
|
@ -1,7 +1,6 @@
|
|||||||
#include "CountNonTerms.h"
|
#include "CountNonTerms.h"
|
||||||
#include "moses/Util.h"
|
#include "moses/Util.h"
|
||||||
#include "moses/TargetPhrase.h"
|
#include "moses/TargetPhrase.h"
|
||||||
#include "moses/StaticData.h"
|
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
|
||||||
@ -21,8 +20,6 @@ void CountNonTerms::EvaluateInIsolation(const Phrase &sourcePhrase
|
|||||||
, ScoreComponentCollection &scoreBreakdown
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
, ScoreComponentCollection &estimatedScores) const
|
, ScoreComponentCollection &estimatedScores) const
|
||||||
{
|
{
|
||||||
const StaticData &staticData = StaticData::Instance();
|
|
||||||
|
|
||||||
vector<float> scores(m_numScoreComponents, 0);
|
vector<float> scores(m_numScoreComponents, 0);
|
||||||
size_t indScore = 0;
|
size_t indScore = 0;
|
||||||
|
|
||||||
@ -39,7 +36,7 @@ void CountNonTerms::EvaluateInIsolation(const Phrase &sourcePhrase
|
|||||||
if (m_targetSyntax) {
|
if (m_targetSyntax) {
|
||||||
for (size_t i = 0; i < targetPhrase.GetSize(); ++i) {
|
for (size_t i = 0; i < targetPhrase.GetSize(); ++i) {
|
||||||
const Word &word = targetPhrase.GetWord(i);
|
const Word &word = targetPhrase.GetWord(i);
|
||||||
if (word.IsNonTerminal() && word != staticData.GetOutputDefaultNonTerminal()) {
|
if (word.IsNonTerminal() && word != m_options->syntax.output_default_non_terminal) {
|
||||||
++scores[indScore];
|
++scores[indScore];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -49,7 +46,7 @@ void CountNonTerms::EvaluateInIsolation(const Phrase &sourcePhrase
|
|||||||
if (m_sourceSyntax) {
|
if (m_sourceSyntax) {
|
||||||
for (size_t i = 0; i < sourcePhrase.GetSize(); ++i) {
|
for (size_t i = 0; i < sourcePhrase.GetSize(); ++i) {
|
||||||
const Word &word = sourcePhrase.GetWord(i);
|
const Word &word = sourcePhrase.GetWord(i);
|
||||||
if (word.IsNonTerminal() && word != staticData.GetInputDefaultNonTerminal()) {
|
if (word.IsNonTerminal() && word != m_options->syntax.input_default_non_terminal) {
|
||||||
++scores[indScore];
|
++scores[indScore];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -72,5 +69,12 @@ void CountNonTerms::SetParameter(const std::string& key, const std::string& valu
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
CountNonTerms::
|
||||||
|
Load(AllOptions::ptr const& opts)
|
||||||
|
{
|
||||||
|
m_options = opts;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -7,6 +7,8 @@ namespace Moses
|
|||||||
|
|
||||||
class CountNonTerms : public StatelessFeatureFunction
|
class CountNonTerms : public StatelessFeatureFunction
|
||||||
{
|
{
|
||||||
|
Word m_input_default_nonterminal;
|
||||||
|
Word m_output_default_nonterminal;
|
||||||
public:
|
public:
|
||||||
CountNonTerms(const std::string &line);
|
CountNonTerms(const std::string &line);
|
||||||
bool IsUseable(const FactorMask &mask) const {
|
bool IsUseable(const FactorMask &mask) const {
|
||||||
@ -41,6 +43,7 @@ public:
|
|||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
|
void Load(AllOptions::ptr const& opts);
|
||||||
protected:
|
protected:
|
||||||
bool m_all, m_sourceSyntax, m_targetSyntax;
|
bool m_all, m_sourceSyntax, m_targetSyntax;
|
||||||
};
|
};
|
||||||
|
@ -52,8 +52,9 @@ void CoveredReferenceFeature::EvaluateWithSourceContext(const InputType &input
|
|||||||
estimatedScores->Assign(this, scores);
|
estimatedScores->Assign(this, scores);
|
||||||
}
|
}
|
||||||
|
|
||||||
void CoveredReferenceFeature::Load(AllOptions const& opts)
|
void CoveredReferenceFeature::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
InputFileStream refFile(m_path);
|
InputFileStream refFile(m_path);
|
||||||
std::string line;
|
std::string line;
|
||||||
const StaticData &staticData = StaticData::Instance();
|
const StaticData &staticData = StaticData::Instance();
|
||||||
|
@ -44,7 +44,7 @@ public:
|
|||||||
ReadParameters();
|
ReadParameters();
|
||||||
}
|
}
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
bool IsUseable(const FactorMask &mask) const {
|
bool IsUseable(const FactorMask &mask) const {
|
||||||
return true;
|
return true;
|
||||||
|
@ -16,8 +16,9 @@ DeleteRules::DeleteRules(const std::string &line)
|
|||||||
ReadParameters();
|
ReadParameters();
|
||||||
}
|
}
|
||||||
|
|
||||||
void DeleteRules::Load(AllOptions const& opts)
|
void DeleteRules::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
std::vector<FactorType> factorOrder;
|
std::vector<FactorType> factorOrder;
|
||||||
factorOrder.push_back(0); // unfactored for now
|
factorOrder.push_back(0); // unfactored for now
|
||||||
|
|
||||||
|
@ -15,7 +15,7 @@ protected:
|
|||||||
public:
|
public:
|
||||||
DeleteRules(const std::string &line);
|
DeleteRules(const std::string &line);
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
bool IsUseable(const FactorMask &mask) const {
|
bool IsUseable(const FactorMask &mask) const {
|
||||||
return true;
|
return true;
|
||||||
|
@ -323,8 +323,9 @@ void DynamicCacheBasedLanguageModel::Clear()
|
|||||||
m_cache.clear();
|
m_cache.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
void DynamicCacheBasedLanguageModel::Load(AllOptions const& opts)
|
void DynamicCacheBasedLanguageModel::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
// SetPreComputedScores();
|
// SetPreComputedScores();
|
||||||
VERBOSE(2,"DynamicCacheBasedLanguageModel::Load()" << std::endl);
|
VERBOSE(2,"DynamicCacheBasedLanguageModel::Load()" << std::endl);
|
||||||
Load(m_initfiles);
|
Load(m_initfiles);
|
||||||
|
@ -119,7 +119,7 @@ public:
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
void Load(const std::string filestr);
|
void Load(const std::string filestr);
|
||||||
void Execute(std::string command);
|
void Execute(std::string command);
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
@ -47,6 +47,7 @@ protected:
|
|||||||
size_t m_index; // index into vector covering ALL feature function values
|
size_t m_index; // index into vector covering ALL feature function values
|
||||||
std::vector<bool> m_tuneableComponents;
|
std::vector<bool> m_tuneableComponents;
|
||||||
size_t m_numTuneableComponents;
|
size_t m_numTuneableComponents;
|
||||||
|
AllOptions::ptr m_options;
|
||||||
//In case there's multiple producers with the same description
|
//In case there's multiple producers with the same description
|
||||||
static std::multiset<std::string> description_counts;
|
static std::multiset<std::string> description_counts;
|
||||||
|
|
||||||
@ -70,7 +71,13 @@ public:
|
|||||||
virtual ~FeatureFunction();
|
virtual ~FeatureFunction();
|
||||||
|
|
||||||
//! override to load model files
|
//! override to load model files
|
||||||
virtual void Load(AllOptions const& opts) {
|
virtual void Load(AllOptions::ptr const& opts) {
|
||||||
|
m_options = opts;
|
||||||
|
}
|
||||||
|
|
||||||
|
AllOptions::ptr const&
|
||||||
|
options() const {
|
||||||
|
return m_options;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void ResetDescriptionCounts() {
|
static void ResetDescriptionCounts() {
|
||||||
|
@ -51,8 +51,9 @@ GlobalLexicalModel::~GlobalLexicalModel()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void GlobalLexicalModel::Load(AllOptions const& opts)
|
void GlobalLexicalModel::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
FactorCollection &factorCollection = FactorCollection::Instance();
|
FactorCollection &factorCollection = FactorCollection::Instance();
|
||||||
const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter();
|
const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter();
|
||||||
|
|
||||||
|
@ -57,7 +57,7 @@ private:
|
|||||||
std::vector<FactorType> m_inputFactorsVec, m_outputFactorsVec;
|
std::vector<FactorType> m_inputFactorsVec, m_outputFactorsVec;
|
||||||
std::string m_filePath;
|
std::string m_filePath;
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
float ScorePhrase( const TargetPhrase& targetPhrase ) const;
|
float ScorePhrase( const TargetPhrase& targetPhrase ) const;
|
||||||
float GetFromCacheOrScorePhrase( const TargetPhrase& targetPhrase ) const;
|
float GetFromCacheOrScorePhrase( const TargetPhrase& targetPhrase ) const;
|
||||||
|
@ -19,8 +19,8 @@ HyperParameterAsWeight::HyperParameterAsWeight(const std::string &line)
|
|||||||
|
|
||||||
vector<float> weights = staticData.GetWeights(this);
|
vector<float> weights = staticData.GetWeights(this);
|
||||||
|
|
||||||
staticData.m_options.search.stack_size = weights[0] * 1000;
|
staticData.m_options->search.stack_size = weights[0] * 1000;
|
||||||
staticData.m_options.search.beam_width = weights[1] * 10;
|
staticData.m_options->search.beam_width = weights[1] * 10;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -23,9 +23,9 @@ InputFeature::InputFeature(const std::string &line)
|
|||||||
s_instance = this;
|
s_instance = this;
|
||||||
}
|
}
|
||||||
|
|
||||||
void InputFeature::Load(AllOptions const& opts)
|
void InputFeature::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
const PhraseDictionary *pt = PhraseDictionary::GetColl()[0];
|
const PhraseDictionary *pt = PhraseDictionary::GetColl()[0];
|
||||||
const PhraseDictionaryTreeAdaptor *ptBin = dynamic_cast<const PhraseDictionaryTreeAdaptor*>(pt);
|
const PhraseDictionaryTreeAdaptor *ptBin = dynamic_cast<const PhraseDictionaryTreeAdaptor*>(pt);
|
||||||
|
|
||||||
|
@ -23,7 +23,7 @@ public:
|
|||||||
|
|
||||||
InputFeature(const std::string &line);
|
InputFeature(const std::string &line);
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
|
@ -8,7 +8,7 @@ InternalTree::InternalTree(const std::string & line, size_t start, size_t len, c
|
|||||||
{
|
{
|
||||||
|
|
||||||
std::vector<FactorType> const& oFactors
|
std::vector<FactorType> const& oFactors
|
||||||
= StaticData::Instance().options().output.factor_order;
|
= StaticData::Instance().options()->output.factor_order;
|
||||||
if (len > 0) {
|
if (len > 0) {
|
||||||
m_value.CreateFromString(Output, oFactors, StringPiece(line).substr(start, len),
|
m_value.CreateFromString(Output, oFactors, StringPiece(line).substr(start, len),
|
||||||
nonterminal);
|
nonterminal);
|
||||||
@ -22,7 +22,7 @@ InternalTree::InternalTree(const std::string & line, const bool nonterminal)
|
|||||||
|
|
||||||
if (found == line.npos) {
|
if (found == line.npos) {
|
||||||
m_value.CreateFromString(Output,
|
m_value.CreateFromString(Output,
|
||||||
StaticData::Instance().options().output.factor_order,
|
StaticData::Instance().options()->output.factor_order,
|
||||||
line, nonterminal);
|
line, nonterminal);
|
||||||
} else {
|
} else {
|
||||||
AddSubTree(line, 0);
|
AddSubTree(line, 0);
|
||||||
@ -50,7 +50,7 @@ size_t InternalTree::AddSubTree(const std::string & line, size_t pos)
|
|||||||
} else {
|
} else {
|
||||||
if (len > 0) {
|
if (len > 0) {
|
||||||
m_value.CreateFromString(Output,
|
m_value.CreateFromString(Output,
|
||||||
StaticData::Instance().options().output.factor_order,
|
StaticData::Instance().options()->output.factor_order,
|
||||||
StringPiece(line).substr(oldpos, len), false);
|
StringPiece(line).substr(oldpos, len), false);
|
||||||
has_value = true;
|
has_value = true;
|
||||||
}
|
}
|
||||||
@ -59,7 +59,7 @@ size_t InternalTree::AddSubTree(const std::string & line, size_t pos)
|
|||||||
} else if (token == ' ' || token == ']') {
|
} else if (token == ' ' || token == ']') {
|
||||||
if (len > 0 && !has_value) {
|
if (len > 0 && !has_value) {
|
||||||
m_value.CreateFromString(Output,
|
m_value.CreateFromString(Output,
|
||||||
StaticData::Instance().options().output.factor_order,
|
StaticData::Instance().options()->output.factor_order,
|
||||||
StringPiece(line).substr(oldpos, len), true);
|
StringPiece(line).substr(oldpos, len), true);
|
||||||
has_value = true;
|
has_value = true;
|
||||||
} else if (len > 0) {
|
} else if (len > 0) {
|
||||||
@ -90,7 +90,7 @@ std::string InternalTree::GetString(bool start) const
|
|||||||
ret += "[";
|
ret += "[";
|
||||||
}
|
}
|
||||||
|
|
||||||
ret += m_value.GetString(StaticData::Instance().options().output.factor_order, false);
|
ret += m_value.GetString(StaticData::Instance().options()->output.factor_order, false);
|
||||||
for (std::vector<TreePointer>::const_iterator it = m_children.begin(); it != m_children.end(); ++it) {
|
for (std::vector<TreePointer>::const_iterator it = m_children.begin(); it != m_children.end(); ++it) {
|
||||||
ret += (*it)->GetString(false);
|
ret += (*it)->GetString(false);
|
||||||
}
|
}
|
||||||
|
@ -84,8 +84,9 @@ LexicalReordering::
|
|||||||
|
|
||||||
void
|
void
|
||||||
LexicalReordering::
|
LexicalReordering::
|
||||||
Load(AllOptions const& opts)
|
Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
typedef LexicalReorderingTable LRTable;
|
typedef LexicalReorderingTable LRTable;
|
||||||
if (m_filePath.size())
|
if (m_filePath.size())
|
||||||
m_table.reset(LRTable::LoadAvailable(m_filePath, m_factorsF,
|
m_table.reset(LRTable::LoadAvailable(m_filePath, m_factorsF,
|
||||||
|
@ -33,7 +33,7 @@ class LexicalReordering : public StatefulFeatureFunction
|
|||||||
public:
|
public:
|
||||||
LexicalReordering(const std::string &line);
|
LexicalReordering(const std::string &line);
|
||||||
virtual ~LexicalReordering();
|
virtual ~LexicalReordering();
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
virtual
|
virtual
|
||||||
bool
|
bool
|
||||||
|
@ -159,8 +159,9 @@ void Model1Feature::SetParameter(const std::string& key, const std::string& valu
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void Model1Feature::Load(AllOptions const& opts)
|
void Model1Feature::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
FEATUREVERBOSE(2, GetScoreProducerDescription() << ": Loading source vocabulary from file " << m_fileNameVcbS << " ...");
|
FEATUREVERBOSE(2, GetScoreProducerDescription() << ": Loading source vocabulary from file " << m_fileNameVcbS << " ...");
|
||||||
Model1Vocabulary vcbS;
|
Model1Vocabulary vcbS;
|
||||||
vcbS.Load(m_fileNameVcbS);
|
vcbS.Load(m_fileNameVcbS);
|
||||||
|
@ -99,7 +99,7 @@ private:
|
|||||||
Model1LexicalTable m_model1;
|
Model1LexicalTable m_model1;
|
||||||
const Factor* m_emptyWord;
|
const Factor* m_emptyWord;
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
// cache
|
// cache
|
||||||
mutable boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> > m_cache;
|
mutable boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> > m_cache;
|
||||||
|
@ -35,8 +35,9 @@ void OpSequenceModel :: readLanguageModel(const char *lmFile)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void OpSequenceModel::Load(AllOptions const& opts)
|
void OpSequenceModel::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
readLanguageModel(m_lmPath.c_str());
|
readLanguageModel(m_lmPath.c_str());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -25,7 +25,7 @@ public:
|
|||||||
~OpSequenceModel();
|
~OpSequenceModel();
|
||||||
|
|
||||||
void readLanguageModel(const char *);
|
void readLanguageModel(const char *);
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
FFState* EvaluateWhenApplied(
|
FFState* EvaluateWhenApplied(
|
||||||
const Hypothesis& cur_hypo,
|
const Hypothesis& cur_hypo,
|
||||||
|
@ -75,8 +75,9 @@ void PhraseOrientationFeature::SetParameter(const std::string& key, const std::s
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void PhraseOrientationFeature::Load(AllOptions const& opts)
|
void PhraseOrientationFeature::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
if ( !m_filenameTargetWordList.empty() ) {
|
if ( !m_filenameTargetWordList.empty() ) {
|
||||||
LoadWordList(m_filenameTargetWordList,m_targetWordList);
|
LoadWordList(m_filenameTargetWordList,m_targetWordList);
|
||||||
m_useTargetWordList = true;
|
m_useTargetWordList = true;
|
||||||
|
@ -289,7 +289,7 @@ public:
|
|||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
|
@ -65,8 +65,9 @@ void PhrasePairFeature::SetParameter(const std::string& key, const std::string&
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void PhrasePairFeature::Load(AllOptions const& opts)
|
void PhrasePairFeature::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
if (m_domainTrigger) {
|
if (m_domainTrigger) {
|
||||||
// domain trigger terms for each input document
|
// domain trigger terms for each input document
|
||||||
ifstream inFileSource(m_filePathSource.c_str());
|
ifstream inFileSource(m_filePathSource.c_str());
|
||||||
|
@ -44,7 +44,7 @@ class PhrasePairFeature: public StatelessFeatureFunction
|
|||||||
public:
|
public:
|
||||||
PhrasePairFeature(const std::string &line);
|
PhrasePairFeature(const std::string &line);
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
bool IsUseable(const FactorMask &mask) const;
|
bool IsUseable(const FactorMask &mask) const;
|
||||||
|
@ -14,11 +14,11 @@ RuleScope::RuleScope(const std::string &line)
|
|||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
bool IsAmbiguous(const Word &word, bool sourceSyntax)
|
// bool IsAmbiguous(const Word &word, bool sourceSyntax)
|
||||||
{
|
// {
|
||||||
const Word &inputDefaultNonTerminal = StaticData::Instance().GetInputDefaultNonTerminal();
|
// const Word &inputDefaultNonTerminal = StaticData::Instance().GetInputDefaultNonTerminal();
|
||||||
return word.IsNonTerminal() && (!sourceSyntax || word == inputDefaultNonTerminal);
|
// return word.IsNonTerminal() && (!sourceSyntax || word == inputDefaultNonTerminal);
|
||||||
}
|
// }
|
||||||
|
|
||||||
void RuleScope::EvaluateInIsolation(const Phrase &source
|
void RuleScope::EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
|
@ -52,8 +52,8 @@ bool SoftMatchingFeature::Load(const std::string& filePath)
|
|||||||
}
|
}
|
||||||
|
|
||||||
Word LHS, RHS;
|
Word LHS, RHS;
|
||||||
LHS.CreateFromString(Output, SD.options().output.factor_order, tokens[0], true);
|
LHS.CreateFromString(Output, SD.options()->output.factor_order, tokens[0], true);
|
||||||
RHS.CreateFromString(Output, SD.options().output.factor_order, tokens[1], true);
|
RHS.CreateFromString(Output, SD.options()->output.factor_order, tokens[1], true);
|
||||||
|
|
||||||
m_softMatches[RHS[0]->GetId()].push_back(LHS);
|
m_softMatches[RHS[0]->GetId()].push_back(LHS);
|
||||||
GetOrSetFeatureName(RHS, LHS);
|
GetOrSetFeatureName(RHS, LHS);
|
||||||
@ -125,7 +125,7 @@ const std::string& SoftMatchingFeature::GetOrSetFeatureName(const Word& RHS, con
|
|||||||
#endif
|
#endif
|
||||||
std::string &name = m_nameCache[RHS[0]->GetId()][LHS[0]->GetId()];
|
std::string &name = m_nameCache[RHS[0]->GetId()][LHS[0]->GetId()];
|
||||||
const std::vector<FactorType> & oFactors
|
const std::vector<FactorType> & oFactors
|
||||||
= StaticData::Instance().options().output.factor_order;
|
= StaticData::Instance().options()->output.factor_order;
|
||||||
std::string LHS_string = LHS.GetString(oFactors, false);
|
std::string LHS_string = LHS.GetString(oFactors, false);
|
||||||
std::string RHS_string = RHS.GetString(oFactors, false);
|
std::string RHS_string = RHS.GetString(oFactors, false);
|
||||||
name = LHS_string + "->" + RHS_string;
|
name = LHS_string + "->" + RHS_string;
|
||||||
|
@ -88,8 +88,9 @@ void SoftSourceSyntacticConstraintsFeature::SetParameter(const std::string& key,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void SoftSourceSyntacticConstraintsFeature::Load(AllOptions const& opts)
|
void SoftSourceSyntacticConstraintsFeature::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
// don't change the loading order!
|
// don't change the loading order!
|
||||||
LoadSourceLabelSet();
|
LoadSourceLabelSet();
|
||||||
if (!m_coreSourceLabelSetFile.empty()) {
|
if (!m_coreSourceLabelSetFile.empty()) {
|
||||||
@ -98,6 +99,7 @@ void SoftSourceSyntacticConstraintsFeature::Load(AllOptions const& opts)
|
|||||||
if (!m_targetSourceLHSJointCountFile.empty()) {
|
if (!m_targetSourceLHSJointCountFile.empty()) {
|
||||||
LoadTargetSourceLeftHandSideJointCountFile();
|
LoadTargetSourceLeftHandSideJointCountFile();
|
||||||
}
|
}
|
||||||
|
// m_output_default_nonterminal = opts->syntax.output_default_non_terminal;
|
||||||
}
|
}
|
||||||
|
|
||||||
void SoftSourceSyntacticConstraintsFeature::LoadSourceLabelSet()
|
void SoftSourceSyntacticConstraintsFeature::LoadSourceLabelSet()
|
||||||
@ -311,8 +313,8 @@ void SoftSourceSyntacticConstraintsFeature::EvaluateWithSourceContext(const Inpu
|
|||||||
std::vector<float> newScores(m_numScoreComponents,0);
|
std::vector<float> newScores(m_numScoreComponents,0);
|
||||||
|
|
||||||
const TreeInput& treeInput = static_cast<const TreeInput&>(input);
|
const TreeInput& treeInput = static_cast<const TreeInput&>(input);
|
||||||
const StaticData& staticData = StaticData::Instance();
|
// const StaticData& staticData = StaticData::Instance();
|
||||||
const Word& outputDefaultNonTerminal = staticData.GetOutputDefaultNonTerminal();
|
// const Word& outputDefaultNonTerminal = staticData.GetOutputDefaultNonTerminal();
|
||||||
|
|
||||||
size_t nNTs = 1;
|
size_t nNTs = 1;
|
||||||
bool treeInputMismatchLHSBinary = true;
|
bool treeInputMismatchLHSBinary = true;
|
||||||
@ -365,7 +367,7 @@ void SoftSourceSyntacticConstraintsFeature::EvaluateWithSourceContext(const Inpu
|
|||||||
|
|
||||||
for (NonTerminalSet::const_iterator treeInputLabelsIt = treeInputLabels.begin();
|
for (NonTerminalSet::const_iterator treeInputLabelsIt = treeInputLabels.begin();
|
||||||
treeInputLabelsIt != treeInputLabels.end(); ++treeInputLabelsIt) {
|
treeInputLabelsIt != treeInputLabels.end(); ++treeInputLabelsIt) {
|
||||||
if (*treeInputLabelsIt != outputDefaultNonTerminal) {
|
if (*treeInputLabelsIt != m_options->syntax.output_default_non_terminal) {
|
||||||
boost::unordered_map<const Factor*,size_t>::const_iterator foundTreeInputLabel
|
boost::unordered_map<const Factor*,size_t>::const_iterator foundTreeInputLabel
|
||||||
= m_sourceLabelIndexesByFactor.find((*treeInputLabelsIt)[0]);
|
= m_sourceLabelIndexesByFactor.find((*treeInputLabelsIt)[0]);
|
||||||
if (foundTreeInputLabel != m_sourceLabelIndexesByFactor.end()) {
|
if (foundTreeInputLabel != m_sourceLabelIndexesByFactor.end()) {
|
||||||
@ -387,7 +389,7 @@ void SoftSourceSyntacticConstraintsFeature::EvaluateWithSourceContext(const Inpu
|
|||||||
|
|
||||||
for (NonTerminalSet::const_iterator treeInputLabelsIt = treeInputLabels.begin();
|
for (NonTerminalSet::const_iterator treeInputLabelsIt = treeInputLabels.begin();
|
||||||
treeInputLabelsIt != treeInputLabels.end(); ++treeInputLabelsIt) {
|
treeInputLabelsIt != treeInputLabels.end(); ++treeInputLabelsIt) {
|
||||||
if (*treeInputLabelsIt != outputDefaultNonTerminal) {
|
if (*treeInputLabelsIt != m_options->syntax.output_default_non_terminal) {
|
||||||
boost::unordered_map<const Factor*,size_t>::const_iterator foundTreeInputLabel
|
boost::unordered_map<const Factor*,size_t>::const_iterator foundTreeInputLabel
|
||||||
= m_sourceLabelIndexesByFactor.find((*treeInputLabelsIt)[0]);
|
= m_sourceLabelIndexesByFactor.find((*treeInputLabelsIt)[0]);
|
||||||
if (foundTreeInputLabel != m_sourceLabelIndexesByFactor.end()) {
|
if (foundTreeInputLabel != m_sourceLabelIndexesByFactor.end()) {
|
||||||
@ -568,7 +570,9 @@ void SoftSourceSyntacticConstraintsFeature::EvaluateWithSourceContext(const Inpu
|
|||||||
}
|
}
|
||||||
if ( treeInputLabelsLHS.size() == 0 ) {
|
if ( treeInputLabelsLHS.size() == 0 ) {
|
||||||
scoreBreakdown.PlusEquals(this,
|
scoreBreakdown.PlusEquals(this,
|
||||||
"LHSPAIR_" + targetLHS->GetString().as_string() + "_" + outputDefaultNonTerminal[0]->GetString().as_string(),
|
"LHSPAIR_" + targetLHS->GetString().as_string() + "_"
|
||||||
|
+ m_options->syntax.output_default_non_terminal[0]
|
||||||
|
->GetString().as_string(),
|
||||||
1);
|
1);
|
||||||
if (!m_targetSourceLHSJointCountFile.empty()) {
|
if (!m_targetSourceLHSJointCountFile.empty()) {
|
||||||
t2sLabelsScore = TransformScore(m_floor);
|
t2sLabelsScore = TransformScore(m_floor);
|
||||||
|
@ -31,7 +31,7 @@ public:
|
|||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
void EvaluateInIsolation(const Phrase &source
|
void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
@ -101,6 +101,7 @@ protected:
|
|||||||
std::pair<float,float> GetLabelPairProbabilities(const Factor* target,
|
std::pair<float,float> GetLabelPairProbabilities(const Factor* target,
|
||||||
const size_t source) const;
|
const size_t source) const;
|
||||||
|
|
||||||
|
// Word m_output_default_nonterminal;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -47,11 +47,12 @@ void SourceGHKMTreeInputMatchFeature::EvaluateWithSourceContext(const InputType
|
|||||||
const Word& lhsLabel = targetPhrase.GetTargetLHS();
|
const Word& lhsLabel = targetPhrase.GetTargetLHS();
|
||||||
|
|
||||||
const StaticData& staticData = StaticData::Instance();
|
const StaticData& staticData = StaticData::Instance();
|
||||||
const Word& outputDefaultNonTerminal = staticData.GetOutputDefaultNonTerminal();
|
|
||||||
|
|
||||||
std::vector<float> newScores(m_numScoreComponents,0.0); // m_numScoreComponents == 2 // first fires for matches, second for mismatches
|
std::vector<float> newScores(m_numScoreComponents,0.0);
|
||||||
|
// m_numScoreComponents == 2 // first fires for matches, second for mismatches
|
||||||
|
|
||||||
if ( (treeInputLabels.find(lhsLabel) != treeInputLabels.end()) && (lhsLabel != outputDefaultNonTerminal) ) {
|
if ( (treeInputLabels.find(lhsLabel) != treeInputLabels.end())
|
||||||
|
&& (lhsLabel != m_options->syntax.output_default_non_terminal) ) {
|
||||||
// match
|
// match
|
||||||
newScores[0] = 1.0;
|
newScores[0] = 1.0;
|
||||||
} else {
|
} else {
|
||||||
@ -62,6 +63,13 @@ void SourceGHKMTreeInputMatchFeature::EvaluateWithSourceContext(const InputType
|
|||||||
scoreBreakdown.PlusEquals(this, newScores);
|
scoreBreakdown.PlusEquals(this, newScores);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
SourceGHKMTreeInputMatchFeature::
|
||||||
|
Load(AllOptions::ptr const& opts)
|
||||||
|
{
|
||||||
|
m_options = opts;
|
||||||
|
// m_output_default_nonterminal = opts->syntax.output_default_non_terminal;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "StatelessFeatureFunction.h"
|
#include "StatelessFeatureFunction.h"
|
||||||
|
#include "moses/parameters/AllOptions.h"
|
||||||
|
|
||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
@ -8,6 +9,7 @@ namespace Moses
|
|||||||
// assumes that source-side syntax labels are stored in the target non-terminal field of the rules
|
// assumes that source-side syntax labels are stored in the target non-terminal field of the rules
|
||||||
class SourceGHKMTreeInputMatchFeature : public StatelessFeatureFunction
|
class SourceGHKMTreeInputMatchFeature : public StatelessFeatureFunction
|
||||||
{
|
{
|
||||||
|
// Word m_output_default_nonterminal;
|
||||||
public:
|
public:
|
||||||
SourceGHKMTreeInputMatchFeature(const std::string &line);
|
SourceGHKMTreeInputMatchFeature(const std::string &line);
|
||||||
|
|
||||||
@ -40,6 +42,7 @@ public:
|
|||||||
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
void EvaluateWhenApplied(const ChartHypothesis &hypo,
|
||||||
ScoreComponentCollection* accumulator) const {};
|
ScoreComponentCollection* accumulator) const {};
|
||||||
|
|
||||||
|
void Load(AllOptions::ptr const& opts);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -36,8 +36,9 @@ void SourceWordDeletionFeature::SetParameter(const std::string& key, const std::
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void SourceWordDeletionFeature::Load(AllOptions const& opts)
|
void SourceWordDeletionFeature::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
if (m_filename.empty())
|
if (m_filename.empty())
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
@ -23,7 +23,7 @@ private:
|
|||||||
public:
|
public:
|
||||||
SourceWordDeletionFeature(const std::string &line);
|
SourceWordDeletionFeature(const std::string &line);
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
bool IsUseable(const FactorMask &mask) const;
|
bool IsUseable(const FactorMask &mask) const;
|
||||||
|
|
||||||
|
@ -48,8 +48,9 @@ void TargetBigramFeature::SetParameter(const std::string& key, const std::string
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void TargetBigramFeature::Load(AllOptions const& opts)
|
void TargetBigramFeature::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
if (m_filePath == "*")
|
if (m_filePath == "*")
|
||||||
return ; //allow all
|
return ; //allow all
|
||||||
ifstream inFile(m_filePath.c_str());
|
ifstream inFile(m_filePath.c_str());
|
||||||
|
@ -34,7 +34,7 @@ class TargetBigramFeature : public StatefulFeatureFunction
|
|||||||
public:
|
public:
|
||||||
TargetBigramFeature(const std::string &line);
|
TargetBigramFeature(const std::string &line);
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
bool IsUseable(const FactorMask &mask) const;
|
bool IsUseable(const FactorMask &mask) const;
|
||||||
|
|
||||||
|
@ -74,8 +74,9 @@ void TargetNgramFeature::SetParameter(const std::string& key, const std::string&
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void TargetNgramFeature::Load(AllOptions const& opts)
|
void TargetNgramFeature::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
if (m_file == "") return; //allow all, for now
|
if (m_file == "") return; //allow all, for now
|
||||||
|
|
||||||
if (m_file == "*") return; //allow all
|
if (m_file == "*") return; //allow all
|
||||||
|
@ -203,7 +203,7 @@ class TargetNgramFeature : public StatefulFeatureFunction
|
|||||||
public:
|
public:
|
||||||
TargetNgramFeature(const std::string &line);
|
TargetNgramFeature(const std::string &line);
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
bool IsUseable(const FactorMask &mask) const;
|
bool IsUseable(const FactorMask &mask) const;
|
||||||
|
|
||||||
|
@ -34,8 +34,9 @@ void TargetWordInsertionFeature::SetParameter(const std::string& key, const std:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void TargetWordInsertionFeature::Load(AllOptions const& opts)
|
void TargetWordInsertionFeature::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
if (m_filename.empty())
|
if (m_filename.empty())
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
@ -25,7 +25,7 @@ public:
|
|||||||
|
|
||||||
bool IsUseable(const FactorMask &mask) const;
|
bool IsUseable(const FactorMask &mask) const;
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
virtual void EvaluateInIsolation(const Phrase &source
|
virtual void EvaluateInIsolation(const Phrase &source
|
||||||
, const TargetPhrase &targetPhrase
|
, const TargetPhrase &targetPhrase
|
||||||
|
@ -8,8 +8,9 @@
|
|||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
|
|
||||||
void TreeStructureFeature::Load(AllOptions const& opts)
|
void TreeStructureFeature::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
|
|
||||||
// syntactic constraints can be hooked in here.
|
// syntactic constraints can be hooked in here.
|
||||||
m_constraints = NULL;
|
m_constraints = NULL;
|
||||||
|
@ -74,7 +74,7 @@ public:
|
|||||||
int /* featureID - used to index the state in the previous hypotheses */,
|
int /* featureID - used to index the state in the previous hypotheses */,
|
||||||
ScoreComponentCollection* accumulator) const;
|
ScoreComponentCollection* accumulator) const;
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -87,8 +87,9 @@ void WordTranslationFeature::SetParameter(const std::string& key, const std::str
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void WordTranslationFeature::Load(AllOptions const& opts)
|
void WordTranslationFeature::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
// load word list for restricted feature set
|
// load word list for restricted feature set
|
||||||
if (m_filePathSource.empty()) {
|
if (m_filePathSource.empty()) {
|
||||||
return;
|
return;
|
||||||
|
@ -40,7 +40,7 @@ public:
|
|||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
bool IsUseable(const FactorMask &mask) const;
|
bool IsUseable(const FactorMask &mask) const;
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
void EvaluateWithSourceContext(const InputType &input
|
void EvaluateWithSourceContext(const InputType &input
|
||||||
, const InputPath &inputPath
|
, const InputPath &inputPath
|
||||||
|
@ -18,9 +18,7 @@ namespace Moses
|
|||||||
|
|
||||||
//! populate this InputType with data from in stream
|
//! populate this InputType with data from in stream
|
||||||
int ForestInput::
|
int ForestInput::
|
||||||
Read(std::istream &in,
|
Read(std::istream &in)
|
||||||
std::vector<FactorType> const& factorOrder,
|
|
||||||
AllOptions const& opts)
|
|
||||||
{
|
{
|
||||||
using Syntax::F2S::Forest;
|
using Syntax::F2S::Forest;
|
||||||
|
|
||||||
@ -48,7 +46,7 @@ Read(std::istream &in,
|
|||||||
std::getline(in, line);
|
std::getline(in, line);
|
||||||
} else {
|
} else {
|
||||||
do {
|
do {
|
||||||
ParseHyperedgeLine(line, factorOrder);
|
ParseHyperedgeLine(line);
|
||||||
std::getline(in, line);
|
std::getline(in, line);
|
||||||
} while (line != "");
|
} while (line != "");
|
||||||
}
|
}
|
||||||
@ -58,7 +56,7 @@ Read(std::istream &in,
|
|||||||
// not sure ForestInput needs to.
|
// not sure ForestInput needs to.
|
||||||
std::stringstream strme;
|
std::stringstream strme;
|
||||||
strme << "<s> " << sentence << " </s>" << std::endl;
|
strme << "<s> " << sentence << " </s>" << std::endl;
|
||||||
Sentence::Read(strme, factorOrder, opts);
|
Sentence::Read(strme);
|
||||||
|
|
||||||
// Find the maximum end position of any vertex (0 if forest is empty).
|
// Find the maximum end position of any vertex (0 if forest is empty).
|
||||||
std::size_t maxEnd = FindMaxEnd(*m_forest);
|
std::size_t maxEnd = FindMaxEnd(*m_forest);
|
||||||
@ -70,6 +68,9 @@ Read(std::istream &in,
|
|||||||
assert(topVertices.size() >= 1);
|
assert(topVertices.size() >= 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
const std::vector<FactorType>& factorOrder = m_options->input.factor_order;
|
||||||
|
|
||||||
// Add <s> vertex.
|
// Add <s> vertex.
|
||||||
Forest::Vertex *startSymbol = NULL;
|
Forest::Vertex *startSymbol = NULL;
|
||||||
{
|
{
|
||||||
@ -122,7 +123,9 @@ Read(std::istream &in,
|
|||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
Syntax::F2S::Forest::Vertex *ForestInput::AddOrDeleteVertex(Forest::Vertex *v)
|
Syntax::F2S::Forest::Vertex*
|
||||||
|
ForestInput::
|
||||||
|
AddOrDeleteVertex(Forest::Vertex *v)
|
||||||
{
|
{
|
||||||
std::pair<VertexSet::iterator, bool> ret = m_vertexSet.insert(v);
|
std::pair<VertexSet::iterator, bool> ret = m_vertexSet.insert(v);
|
||||||
if (ret.second) {
|
if (ret.second) {
|
||||||
@ -172,14 +175,16 @@ void ForestInput::FindTopVertices(Forest &forest,
|
|||||||
std::back_inserter(topVertices));
|
std::back_inserter(topVertices));
|
||||||
}
|
}
|
||||||
|
|
||||||
void ForestInput::ParseHyperedgeLine(
|
void
|
||||||
const std::string &line, const std::vector<FactorType>& factorOrder)
|
ForestInput::
|
||||||
|
ParseHyperedgeLine(const std::string &line)
|
||||||
{
|
{
|
||||||
|
const std::vector<FactorType>& factorOrder = m_options->input.factor_order;
|
||||||
using Syntax::F2S::Forest;
|
using Syntax::F2S::Forest;
|
||||||
|
|
||||||
const util::AnyCharacter delimiter(" \t");
|
const util::AnyCharacter delimiter(" \t");
|
||||||
util::TokenIter<util::AnyCharacter, true> p(line, delimiter);
|
util::TokenIter<util::AnyCharacter, true> p(line, delimiter);
|
||||||
Forest::Vertex *v = AddOrDeleteVertex(ParseVertex(*p, factorOrder));
|
Forest::Vertex *v = AddOrDeleteVertex(ParseVertex(*p));
|
||||||
Forest::Hyperedge *e = new Forest::Hyperedge();
|
Forest::Hyperedge *e = new Forest::Hyperedge();
|
||||||
e->head = v;
|
e->head = v;
|
||||||
++p;
|
++p;
|
||||||
@ -188,7 +193,7 @@ void ForestInput::ParseHyperedgeLine(
|
|||||||
//throw Exception("");
|
//throw Exception("");
|
||||||
}
|
}
|
||||||
for (++p; *p != "|||"; ++p) {
|
for (++p; *p != "|||"; ++p) {
|
||||||
v = ParseVertex(*p, factorOrder);
|
v = ParseVertex(*p);
|
||||||
if (!v->pvertex.symbol.IsNonTerminal()) {
|
if (!v->pvertex.symbol.IsNonTerminal()) {
|
||||||
// Egret does not give start/end for terminals.
|
// Egret does not give start/end for terminals.
|
||||||
v->pvertex.span = Range(e->head->pvertex.span.GetStartPos(),
|
v->pvertex.span = Range(e->head->pvertex.span.GetStartPos(),
|
||||||
@ -203,11 +208,11 @@ void ForestInput::ParseHyperedgeLine(
|
|||||||
e->head->incoming.push_back(e);
|
e->head->incoming.push_back(e);
|
||||||
}
|
}
|
||||||
|
|
||||||
Syntax::F2S::Forest::Vertex *ForestInput::ParseVertex(
|
Syntax::F2S::Forest::Vertex*
|
||||||
const StringPiece &s, const std::vector<FactorType>& factorOrder)
|
ForestInput::ParseVertex(const StringPiece &s)
|
||||||
{
|
{
|
||||||
using Syntax::F2S::Forest;
|
using Syntax::F2S::Forest;
|
||||||
|
const std::vector<FactorType>& factorOrder = m_options->input.factor_order;
|
||||||
Word symbol;
|
Word symbol;
|
||||||
std::size_t pos = s.rfind('[');
|
std::size_t pos = s.rfind('[');
|
||||||
if (pos == std::string::npos) {
|
if (pos == std::string::npos) {
|
||||||
|
@ -29,9 +29,7 @@ public:
|
|||||||
|
|
||||||
//! populate this InputType with data from in stream
|
//! populate this InputType with data from in stream
|
||||||
virtual int
|
virtual int
|
||||||
Read(std::istream& in,
|
Read(std::istream& in);
|
||||||
std::vector<FactorType> const& factorOrder,
|
|
||||||
AllOptions const& opts);
|
|
||||||
|
|
||||||
//! Output debugging info to stream out
|
//! Output debugging info to stream out
|
||||||
virtual void Print(std::ostream&) const;
|
virtual void Print(std::ostream&) const;
|
||||||
@ -76,11 +74,9 @@ private:
|
|||||||
|
|
||||||
void FindTopVertices(Forest &, std::vector<Forest::Vertex *> &);
|
void FindTopVertices(Forest &, std::vector<Forest::Vertex *> &);
|
||||||
|
|
||||||
void ParseHyperedgeLine(const std::string &,
|
void ParseHyperedgeLine(const std::string &);
|
||||||
const std::vector<FactorType> &);
|
|
||||||
|
|
||||||
Forest::Vertex *ParseVertex(const StringPiece &,
|
Forest::Vertex *ParseVertex(const StringPiece &);
|
||||||
const std::vector<FactorType> &);
|
|
||||||
|
|
||||||
boost::shared_ptr<Forest> m_forest;
|
boost::shared_ptr<Forest> m_forest;
|
||||||
Forest::Vertex *m_rootVertex;
|
Forest::Vertex *m_rootVertex;
|
||||||
|
@ -44,8 +44,9 @@ GenerationDictionary::GenerationDictionary(const std::string &line)
|
|||||||
ReadParameters();
|
ReadParameters();
|
||||||
}
|
}
|
||||||
|
|
||||||
void GenerationDictionary::Load(AllOptions const& opts)
|
void GenerationDictionary::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
FactorCollection &factorCollection = FactorCollection::Instance();
|
FactorCollection &factorCollection = FactorCollection::Instance();
|
||||||
|
|
||||||
const size_t numFeatureValuesInConfig = this->GetNumScoreComponents();
|
const size_t numFeatureValuesInConfig = this->GetNumScoreComponents();
|
||||||
|
@ -62,7 +62,7 @@ public:
|
|||||||
virtual ~GenerationDictionary();
|
virtual ~GenerationDictionary();
|
||||||
|
|
||||||
//! load data file
|
//! load data file
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
/** number of unique input entries in the generation table.
|
/** number of unique input entries in the generation table.
|
||||||
* NOT the number of lines in the generation table
|
* NOT the number of lines in the generation table
|
||||||
|
@ -56,7 +56,7 @@ WriteHypos(const ChartHypothesisCollection& hypos,
|
|||||||
ChartHypothesisCollection::const_iterator iter;
|
ChartHypothesisCollection::const_iterator iter;
|
||||||
for (iter = hypos.begin() ; iter != hypos.end() ; ++iter) {
|
for (iter = hypos.begin() ; iter != hypos.end() ; ++iter) {
|
||||||
ChartHypothesis &mainHypo = **iter;
|
ChartHypothesis &mainHypo = **iter;
|
||||||
if (StaticData::Instance().options().output.DontPruneSearchGraph ||
|
if (StaticData::Instance().options()->output.DontPruneSearchGraph ||
|
||||||
reachable.find(mainHypo.GetId()) != reachable.end()) {
|
reachable.find(mainHypo.GetId()) != reachable.end()) {
|
||||||
(*m_out) << m_lineNumber << " " << mainHypo << endl;
|
(*m_out) << m_lineNumber << " " << mainHypo << endl;
|
||||||
}
|
}
|
||||||
@ -90,7 +90,7 @@ WriteHypos(const ChartHypothesisCollection& hypos,
|
|||||||
ChartHypothesisCollection::const_iterator iter;
|
ChartHypothesisCollection::const_iterator iter;
|
||||||
for (iter = hypos.begin() ; iter != hypos.end() ; ++iter) {
|
for (iter = hypos.begin() ; iter != hypos.end() ; ++iter) {
|
||||||
const ChartHypothesis* mainHypo = *iter;
|
const ChartHypothesis* mainHypo = *iter;
|
||||||
if (!StaticData::Instance().options().output.DontPruneSearchGraph &&
|
if (!StaticData::Instance().options()->output.DontPruneSearchGraph &&
|
||||||
reachable.find(mainHypo->GetId()) == reachable.end()) {
|
reachable.find(mainHypo->GetId()) == reachable.end()) {
|
||||||
//Ignore non reachable nodes
|
//Ignore non reachable nodes
|
||||||
continue;
|
continue;
|
||||||
|
@ -78,20 +78,19 @@ IOWrapper::IOWrapper(AllOptions const& opts)
|
|||||||
Parameter const& P = staticData.GetParameter();
|
Parameter const& P = staticData.GetParameter();
|
||||||
|
|
||||||
// context buffering for context-sensitive decoding
|
// context buffering for context-sensitive decoding
|
||||||
m_look_ahead = staticData.options().context.look_ahead;
|
m_look_ahead = staticData.options()->context.look_ahead;
|
||||||
m_look_back = staticData.options().context.look_back;
|
m_look_back = staticData.options()->context.look_back;
|
||||||
|
m_inputType = staticData.options()->input.input_type;
|
||||||
m_inputType = staticData.options().input.input_type;
|
|
||||||
|
|
||||||
UTIL_THROW_IF2((m_look_ahead || m_look_back) && m_inputType != SentenceInput,
|
UTIL_THROW_IF2((m_look_ahead || m_look_back) && m_inputType != SentenceInput,
|
||||||
"Context-sensitive decoding currently works only with sentence input.");
|
"Context-sensitive decoding currently works only with sentence input.");
|
||||||
|
|
||||||
m_currentLine = staticData.GetStartTranslationId();
|
m_currentLine = staticData.GetStartTranslationId();
|
||||||
|
|
||||||
m_inputFactorOrder = &staticData.options().input.factor_order;
|
m_inputFactorOrder = &staticData.options()->input.factor_order;
|
||||||
|
|
||||||
size_t nBestSize = staticData.options().nbest.nbest_size;
|
size_t nBestSize = staticData.options()->nbest.nbest_size;
|
||||||
string nBestFilePath = staticData.options().nbest.output_file_path;
|
string nBestFilePath = staticData.options()->nbest.output_file_path;
|
||||||
|
|
||||||
staticData.GetParameter().SetParameter<string>(m_inputFilePath, "input-file", "");
|
staticData.GetParameter().SetParameter<string>(m_inputFilePath, "input-file", "");
|
||||||
if (m_inputFilePath.empty()) {
|
if (m_inputFilePath.empty()) {
|
||||||
@ -130,8 +129,8 @@ IOWrapper::IOWrapper(AllOptions const& opts)
|
|||||||
P.SetParameter<string>(path, "output-word-graph", "");
|
P.SetParameter<string>(path, "output-word-graph", "");
|
||||||
if (path.size()) m_wordGraphCollector.reset(new OutputCollector(path));
|
if (path.size()) m_wordGraphCollector.reset(new OutputCollector(path));
|
||||||
|
|
||||||
size_t latticeSamplesSize = staticData.options().output.lattice_sample_size;
|
size_t latticeSamplesSize = staticData.options()->output.lattice_sample_size;
|
||||||
string latticeSamplesFile = staticData.options().output.lattice_sample_filepath;
|
string latticeSamplesFile = staticData.options()->output.lattice_sample_filepath;
|
||||||
if (latticeSamplesSize) {
|
if (latticeSamplesSize) {
|
||||||
m_latticeSamplesCollector.reset(new OutputCollector(latticeSamplesFile));
|
m_latticeSamplesCollector.reset(new OutputCollector(latticeSamplesFile));
|
||||||
if (m_latticeSamplesCollector->OutputIsCout()) {
|
if (m_latticeSamplesCollector->OutputIsCout()) {
|
||||||
|
@ -219,7 +219,6 @@ boost::shared_ptr<InputType>
|
|||||||
IOWrapper::
|
IOWrapper::
|
||||||
BufferInput()
|
BufferInput()
|
||||||
{
|
{
|
||||||
AllOptions const& opts = StaticData::Instance().options();
|
|
||||||
boost::shared_ptr<itype> source;
|
boost::shared_ptr<itype> source;
|
||||||
boost::shared_ptr<InputType> ret;
|
boost::shared_ptr<InputType> ret;
|
||||||
if (m_future_input.size()) {
|
if (m_future_input.size()) {
|
||||||
@ -228,13 +227,13 @@ BufferInput()
|
|||||||
m_buffered_ahead -= ret->GetSize();
|
m_buffered_ahead -= ret->GetSize();
|
||||||
} else {
|
} else {
|
||||||
source.reset(new itype(m_options));
|
source.reset(new itype(m_options));
|
||||||
if (!source->Read(*m_inputStream, *m_inputFactorOrder, opts))
|
if (!source->Read(*m_inputStream))
|
||||||
return ret;
|
return ret;
|
||||||
ret = source;
|
ret = source;
|
||||||
}
|
}
|
||||||
while (m_buffered_ahead < m_look_ahead) {
|
while (m_buffered_ahead < m_look_ahead) {
|
||||||
source.reset(new itype(m_options));
|
source.reset(new itype(m_options));
|
||||||
if (!source->Read(*m_inputStream, *m_inputFactorOrder, opts))
|
if (!source->Read(*m_inputStream))
|
||||||
break;
|
break;
|
||||||
m_future_input.push_back(source);
|
m_future_input.push_back(source);
|
||||||
m_buffered_ahead += source->GetSize();
|
m_buffered_ahead += source->GetSize();
|
||||||
|
@ -208,7 +208,7 @@ Manager::Manager(ttasksptr const& ttask)
|
|||||||
: BaseManager(ttask)
|
: BaseManager(ttask)
|
||||||
, cells_(m_source, ChartCellBaseFactory(), parser_)
|
, cells_(m_source, ChartCellBaseFactory(), parser_)
|
||||||
, parser_(ttask, cells_)
|
, parser_(ttask, cells_)
|
||||||
, n_best_(search::NBestConfig(StaticData::Instance().options().nbest.nbest_size))
|
, n_best_(search::NBestConfig(StaticData::Instance().options()->nbest.nbest_size))
|
||||||
{ }
|
{ }
|
||||||
|
|
||||||
Manager::~Manager()
|
Manager::~Manager()
|
||||||
@ -232,8 +232,8 @@ PopulateBest(const Model &model, const std::vector<lm::WordIndex> &words, Best &
|
|||||||
const StaticData &data = StaticData::Instance();
|
const StaticData &data = StaticData::Instance();
|
||||||
const float lm_weight = data.GetWeights(&abstract)[0];
|
const float lm_weight = data.GetWeights(&abstract)[0];
|
||||||
const float oov_weight = abstract.OOVFeatureEnabled() ? data.GetWeights(&abstract)[1] : 0.0;
|
const float oov_weight = abstract.OOVFeatureEnabled() ? data.GetWeights(&abstract)[1] : 0.0;
|
||||||
size_t cpl = data.options().cube.pop_limit;
|
size_t cpl = data.options()->cube.pop_limit;
|
||||||
size_t nbs = data.options().nbest.nbest_size;
|
size_t nbs = data.options()->nbest.nbest_size;
|
||||||
search::Config config(lm_weight * log_10, cpl, search::NBestConfig(nbs));
|
search::Config config(lm_weight * log_10, cpl, search::NBestConfig(nbs));
|
||||||
search::Context<Model> context(config, model);
|
search::Context<Model> context(config, model);
|
||||||
|
|
||||||
@ -261,7 +261,7 @@ PopulateBest(const Model &model, const std::vector<lm::WordIndex> &words, Best &
|
|||||||
|
|
||||||
template <class Model> void Manager::LMCallback(const Model &model, const std::vector<lm::WordIndex> &words)
|
template <class Model> void Manager::LMCallback(const Model &model, const std::vector<lm::WordIndex> &words)
|
||||||
{
|
{
|
||||||
std::size_t nbest = StaticData::Instance().options().nbest.nbest_size;
|
std::size_t nbest = StaticData::Instance().options()->nbest.nbest_size;
|
||||||
if (nbest <= 1) {
|
if (nbest <= 1) {
|
||||||
search::History ret = PopulateBest(model, words, single_best_);
|
search::History ret = PopulateBest(model, words, single_best_);
|
||||||
if (ret) {
|
if (ret) {
|
||||||
|
@ -32,6 +32,7 @@ namespace Moses
|
|||||||
InputType::InputType(AllOptions::ptr const& opts, long translationId)
|
InputType::InputType(AllOptions::ptr const& opts, long translationId)
|
||||||
: m_options(opts)
|
: m_options(opts)
|
||||||
, m_translationId(translationId)
|
, m_translationId(translationId)
|
||||||
|
, m_reorderingConstraint(opts->reordering.max_distortion)
|
||||||
{
|
{
|
||||||
m_frontSpanCoveredLength = 0;
|
m_frontSpanCoveredLength = 0;
|
||||||
m_sourceCompleted.resize(0);
|
m_sourceCompleted.resize(0);
|
||||||
|
@ -190,9 +190,10 @@ public:
|
|||||||
|
|
||||||
//! populate this InputType with data from in stream
|
//! populate this InputType with data from in stream
|
||||||
virtual int
|
virtual int
|
||||||
Read(std::istream& in,
|
Read(std::istream& in) = 0;
|
||||||
std::vector<FactorType> const& factorOrder,
|
// ,
|
||||||
AllOptions const& opts) =0;
|
// std::vector<FactorType> const& factorOrder,
|
||||||
|
// AllOptions const& opts) =0;
|
||||||
|
|
||||||
//! Output debugging info to stream out
|
//! Output debugging info to stream out
|
||||||
virtual void Print(std::ostream&) const =0;
|
virtual void Print(std::ostream&) const =0;
|
||||||
|
@ -20,8 +20,9 @@ BilingualLM::BilingualLM(const std::string &line)
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void BilingualLM::Load(AllOptions const& opts)
|
void BilingualLM::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
ReadParameters();
|
ReadParameters();
|
||||||
loadModel();
|
loadModel();
|
||||||
}
|
}
|
||||||
|
@ -117,7 +117,7 @@ public:
|
|||||||
return new BilingualLMState(0);
|
return new BilingualLMState(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
FFState* EvaluateWhenApplied(
|
FFState* EvaluateWhenApplied(
|
||||||
const Hypothesis& cur_hypo,
|
const Hypothesis& cur_hypo,
|
||||||
|
@ -28,7 +28,7 @@ public:
|
|||||||
LanguageModelDALM(const std::string &line);
|
LanguageModelDALM(const std::string &line);
|
||||||
virtual ~LanguageModelDALM();
|
virtual ~LanguageModelDALM();
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const;
|
virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const;
|
||||||
|
|
||||||
|
@ -96,7 +96,7 @@ bool LanguageModelIRST::IsUseable(const FactorMask &mask) const
|
|||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
void LanguageModelIRST::Load(AllOptions const& opts)
|
void LanguageModelIRST::Load(AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
FactorCollection &factorCollection = FactorCollection::Instance();
|
FactorCollection &factorCollection = FactorCollection::Instance();
|
||||||
|
|
||||||
|
@ -88,7 +88,7 @@ public:
|
|||||||
|
|
||||||
bool IsUseable(const FactorMask &mask) const;
|
bool IsUseable(const FactorMask &mask) const;
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
const FFState *EmptyHypothesisState(const InputType &/*input*/) const;
|
const FFState *EmptyHypothesisState(const InputType &/*input*/) const;
|
||||||
|
|
||||||
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL) const;
|
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL) const;
|
||||||
|
@ -54,7 +54,7 @@ protected:
|
|||||||
public:
|
public:
|
||||||
LanguageModelMaxEntSRI(const std::string &line);
|
LanguageModelMaxEntSRI(const std::string &line);
|
||||||
~LanguageModelMaxEntSRI();
|
~LanguageModelMaxEntSRI();
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0) const;
|
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0) const;
|
||||||
};
|
};
|
||||||
|
@ -27,7 +27,7 @@ public:
|
|||||||
|
|
||||||
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0) const;
|
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0) const;
|
||||||
|
|
||||||
virtual void Load(AllOptions const& opts);
|
virtual void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -208,7 +208,7 @@ public:
|
|||||||
int /* featureID - used to index the state in the previous hypotheses */,
|
int /* featureID - used to index the state in the previous hypotheses */,
|
||||||
ScoreComponentCollection* accumulator) const;
|
ScoreComponentCollection* accumulator) const;
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
// Iterator-class that yields all children of a node; if child is virtual node of binarized tree, its children are yielded instead.
|
// Iterator-class that yields all children of a node; if child is virtual node of binarized tree, its children are yielded instead.
|
||||||
class UnbinarizedChildren
|
class UnbinarizedChildren
|
||||||
|
@ -39,7 +39,7 @@ public:
|
|||||||
LanguageModelRandLM(const std::string &line);
|
LanguageModelRandLM(const std::string &line);
|
||||||
~LanguageModelRandLM();
|
~LanguageModelRandLM();
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL) const;
|
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL) const;
|
||||||
void InitializeForInput(ttasksptr const& ttask);
|
void InitializeForInput(ttasksptr const& ttask);
|
||||||
void CleanUpAfterSentenceProcessing(const InputType& source);
|
void CleanUpAfterSentenceProcessing(const InputType& source);
|
||||||
|
@ -54,7 +54,7 @@ protected:
|
|||||||
public:
|
public:
|
||||||
LanguageModelSRI(const std::string &line);
|
LanguageModelSRI(const std::string &line);
|
||||||
~LanguageModelSRI();
|
~LanguageModelSRI();
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0) const;
|
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0) const;
|
||||||
};
|
};
|
||||||
|
@ -24,7 +24,7 @@ public:
|
|||||||
|
|
||||||
void SetParameter(const std::string& key, const std::string& value);
|
void SetParameter(const std::string& key, const std::string& value);
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
virtual LMResult GetValue(
|
virtual LMResult GetValue(
|
||||||
const std::vector<const Word*> &contextFactor,
|
const std::vector<const Word*> &contextFactor,
|
||||||
|
@ -38,9 +38,8 @@ MockHypothesisGuard
|
|||||||
m_uwp("UnknownWordPenalty"), m_dist("Distortion")
|
m_uwp("UnknownWordPenalty"), m_dist("Distortion")
|
||||||
{
|
{
|
||||||
BOOST_CHECK_EQUAL(alignments.size(), targetSegments.size());
|
BOOST_CHECK_EQUAL(alignments.size(), targetSegments.size());
|
||||||
std::vector<Moses::FactorType> factors(1,0);
|
AllOptions::ptr opts(new AllOptions(*StaticData::Instance().options()));
|
||||||
AllOptions::ptr opts(new AllOptions(StaticData::Instance().options()));
|
m_sentence.reset(new Sentence(opts, 0, sourceSentence));
|
||||||
m_sentence.reset(new Sentence(opts,0, sourceSentence, &factors));
|
|
||||||
m_ttask = TranslationTask::create(m_sentence);
|
m_ttask = TranslationTask::create(m_sentence);
|
||||||
m_manager.reset(new Manager(m_ttask));
|
m_manager.reset(new Manager(m_ttask));
|
||||||
|
|
||||||
@ -59,16 +58,14 @@ MockHypothesisGuard
|
|||||||
for (; ti != targetSegments.end() && ai != alignments.end(); ++ti,++ai) {
|
for (; ti != targetSegments.end() && ai != alignments.end(); ++ti,++ai) {
|
||||||
Hypothesis* prevHypo = m_hypothesis;
|
Hypothesis* prevHypo = m_hypothesis;
|
||||||
Range range(ai->first,ai->second);
|
Range range(ai->first,ai->second);
|
||||||
const Bitmap &newBitmap = bitmaps.GetBitmap(prevHypo->GetWordsBitmap(),
|
const Bitmap &newBitmap = bitmaps.GetBitmap(prevHypo->GetWordsBitmap(), range);
|
||||||
range);
|
|
||||||
|
|
||||||
m_targetPhrases.push_back(TargetPhrase(NULL));
|
m_targetPhrases.push_back(TargetPhrase(NULL));
|
||||||
// m_targetPhrases.back().CreateFromString(Input, factors, *ti, "|", NULL);
|
vector<FactorType> const& factors = opts->output.factor_order;
|
||||||
m_targetPhrases.back().CreateFromString(Input, factors, *ti, NULL);
|
m_targetPhrases.back().CreateFromString(Input, factors, *ti, NULL);
|
||||||
m_toptions.push_back(new TranslationOption
|
m_toptions.push_back(new TranslationOption
|
||||||
(range,m_targetPhrases.back()));
|
(range,m_targetPhrases.back()));
|
||||||
m_hypothesis = new Hypothesis(*prevHypo, *m_toptions.back(), newBitmap, m_manager->GetNextHypoId());
|
m_hypothesis = new Hypothesis(*prevHypo, *m_toptions.back(), newBitmap,
|
||||||
|
m_manager->GetNextHypoId());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -119,7 +119,7 @@ Phrase::
|
|||||||
GetStringRep(vector<FactorType> const& factorsToPrint,
|
GetStringRep(vector<FactorType> const& factorsToPrint,
|
||||||
AllOptions const* opts) const
|
AllOptions const* opts) const
|
||||||
{
|
{
|
||||||
if (!opts) opts = &StaticData::Instance().options();
|
if (!opts) opts = StaticData::Instance().options().get();
|
||||||
bool markUnk = opts->unk.mark;
|
bool markUnk = opts->unk.mark;
|
||||||
util::StringStream strme;
|
util::StringStream strme;
|
||||||
for (size_t pos = 0 ; pos < GetSize() ; pos++) {
|
for (size_t pos = 0 ; pos < GetSize() ; pos++) {
|
||||||
|
@ -187,7 +187,9 @@ bool ReorderingConstraint::Check( const Bitmap &bitmap, size_t startPos, size_t
|
|||||||
|
|
||||||
|
|
||||||
// check, if we are setting us up for a dead end due to distortion limits
|
// check, if we are setting us up for a dead end due to distortion limits
|
||||||
size_t distortionLimit = (size_t)StaticData::Instance().GetMaxDistortion();
|
|
||||||
|
// size_t distortionLimit = (size_t)StaticData::Instance().GetMaxDistortion();
|
||||||
|
size_t distortionLimit = m_max_distortion;
|
||||||
if (startPos != firstGapPos && endZone-firstGapPos >= distortionLimit) {
|
if (startPos != firstGapPos && endZone-firstGapPos >= distortionLimit) {
|
||||||
VERBOSE(3," dead end due to distortion limit" << std::endl);
|
VERBOSE(3," dead end due to distortion limit" << std::endl);
|
||||||
return false;
|
return false;
|
||||||
|
@ -53,11 +53,16 @@ protected:
|
|||||||
size_t *m_localWall; /**< flag for each word if it is a local wall */
|
size_t *m_localWall; /**< flag for each word if it is a local wall */
|
||||||
std::vector< std::vector< size_t > > m_zone; /** zones that limit reordering */
|
std::vector< std::vector< size_t > > m_zone; /** zones that limit reordering */
|
||||||
bool m_active; /**< flag indicating, if there are any active constraints */
|
bool m_active; /**< flag indicating, if there are any active constraints */
|
||||||
|
int m_max_distortion;
|
||||||
public:
|
public:
|
||||||
|
|
||||||
//! create ReorderingConstraint of length size and initialise to zero
|
//! create ReorderingConstraint of length size and initialise to zero
|
||||||
ReorderingConstraint() :m_wall(NULL),m_localWall(NULL),m_active(false) {}
|
ReorderingConstraint(int max_distortion)
|
||||||
|
: m_wall(NULL)
|
||||||
|
, m_localWall(NULL)
|
||||||
|
, m_active(false)
|
||||||
|
, m_max_distortion(max_distortion)
|
||||||
|
{}
|
||||||
|
|
||||||
//! destructer
|
//! destructer
|
||||||
~ReorderingConstraint() {
|
~ReorderingConstraint() {
|
||||||
|
@ -43,7 +43,7 @@ RuleCube::RuleCube(const ChartTranslationOptions &transOpt,
|
|||||||
{
|
{
|
||||||
RuleCubeItem *item = new RuleCubeItem(transOpt, allChartCells);
|
RuleCubeItem *item = new RuleCubeItem(transOpt, allChartCells);
|
||||||
m_covered.insert(item);
|
m_covered.insert(item);
|
||||||
if (StaticData::Instance().options().cube.lazy_scoring) {
|
if (StaticData::Instance().options()->cube.lazy_scoring) {
|
||||||
item->EstimateScore();
|
item->EstimateScore();
|
||||||
} else {
|
} else {
|
||||||
item->CreateHypothesis(transOpt, manager);
|
item->CreateHypothesis(transOpt, manager);
|
||||||
@ -91,7 +91,7 @@ void RuleCube::CreateNeighbor(const RuleCubeItem &item, int dimensionIndex,
|
|||||||
if (!result.second) {
|
if (!result.second) {
|
||||||
delete newItem; // already seen it
|
delete newItem; // already seen it
|
||||||
} else {
|
} else {
|
||||||
if (StaticData::Instance().options().cube.lazy_scoring) {
|
if (StaticData::Instance().options()->cube.lazy_scoring) {
|
||||||
newItem->EstimateScore();
|
newItem->EstimateScore();
|
||||||
} else {
|
} else {
|
||||||
newItem->CreateHypothesis(m_transOpt, manager);
|
newItem->CreateHypothesis(m_transOpt, manager);
|
||||||
|
@ -50,7 +50,7 @@ ChartHypothesis *RuleCubeQueue::Pop()
|
|||||||
// pop the most promising item from the cube and get the corresponding
|
// pop the most promising item from the cube and get the corresponding
|
||||||
// hypothesis
|
// hypothesis
|
||||||
RuleCubeItem *item = cube->Pop(m_manager);
|
RuleCubeItem *item = cube->Pop(m_manager);
|
||||||
if (StaticData::Instance().options().cube.lazy_scoring) {
|
if (StaticData::Instance().options()->cube.lazy_scoring) {
|
||||||
item->CreateHypothesis(cube->GetTranslationOption(), m_manager);
|
item->CreateHypothesis(cube->GetTranslationOption(), m_manager);
|
||||||
}
|
}
|
||||||
ChartHypothesis *hypo = item->ReleaseHypothesis();
|
ChartHypothesis *hypo = item->ReleaseHypothesis();
|
||||||
|
@ -43,9 +43,8 @@ namespace Moses
|
|||||||
Sentence::
|
Sentence::
|
||||||
Sentence(AllOptions::ptr const& opts) : Phrase(0) , InputType(opts)
|
Sentence(AllOptions::ptr const& opts) : Phrase(0) , InputType(opts)
|
||||||
{
|
{
|
||||||
const StaticData& SD = StaticData::Instance();
|
if (is_syntax(opts->search.algo))
|
||||||
if (SD.IsSyntax())
|
m_defaultLabelSet.insert(opts->syntax.input_default_non_terminal);
|
||||||
m_defaultLabelSet.insert(SD.GetInputDefaultNonTerminal());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Sentence::
|
Sentence::
|
||||||
@ -146,36 +145,30 @@ aux_interpret_dlt(string& line) // whatever DLT means ... --- UG
|
|||||||
|
|
||||||
void
|
void
|
||||||
Sentence::
|
Sentence::
|
||||||
aux_interpret_xml(AllOptions const& opts, std::string& line, std::vector<size_t> & xmlWalls,
|
aux_interpret_xml(std::string& line, std::vector<size_t> & xmlWalls,
|
||||||
std::vector<std::pair<size_t, std::string> >& placeholders)
|
std::vector<std::pair<size_t, std::string> >& placeholders)
|
||||||
{
|
{
|
||||||
// parse XML markup in translation line
|
// parse XML markup in translation line
|
||||||
|
|
||||||
const StaticData &SD = StaticData::Instance();
|
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
if (opts.input.xml_policy != XmlPassThrough) {
|
if (m_options->input.xml_policy != XmlPassThrough) {
|
||||||
int offset = SD.IsSyntax() ? 1 : 0;
|
bool OK = ProcessAndStripXMLTags(*m_options, line,
|
||||||
bool OK = ProcessAndStripXMLTags(opts, line, m_xmlOptions,
|
m_xmlOptions,
|
||||||
m_reorderingConstraint,
|
m_reorderingConstraint,
|
||||||
xmlWalls, placeholders, offset,
|
xmlWalls, placeholders);
|
||||||
SD.GetXmlBrackets().first,
|
|
||||||
SD.GetXmlBrackets().second);
|
|
||||||
UTIL_THROW_IF2(!OK, "Unable to parse XML in line: " << line);
|
UTIL_THROW_IF2(!OK, "Unable to parse XML in line: " << line);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
Sentence::
|
Sentence::
|
||||||
init(AllOptions::ptr const& opts, string line, std::vector<FactorType> const& factorOrder)
|
init(string line)
|
||||||
{
|
{
|
||||||
using namespace std;
|
using namespace std;
|
||||||
const StaticData &SD = StaticData::Instance();
|
|
||||||
|
|
||||||
m_frontSpanCoveredLength = 0;
|
m_frontSpanCoveredLength = 0;
|
||||||
m_sourceCompleted.resize(0);
|
m_sourceCompleted.resize(0);
|
||||||
|
|
||||||
if (SD.ContinuePartialTranslation())
|
if (m_options->input.continue_partial_translation)
|
||||||
aux_init_partial_translation(line);
|
aux_init_partial_translation(line);
|
||||||
|
|
||||||
line = Trim(line);
|
line = Trim(line);
|
||||||
@ -183,28 +176,28 @@ init(AllOptions::ptr const& opts, string line, std::vector<FactorType> const& fa
|
|||||||
aux_interpret_dlt(line); // some poorly documented cache-based stuff
|
aux_interpret_dlt(line); // some poorly documented cache-based stuff
|
||||||
|
|
||||||
// if sentences is specified as "<passthrough tag1=""/>"
|
// if sentences is specified as "<passthrough tag1=""/>"
|
||||||
if (SD.options().output.PrintPassThrough ||
|
if (m_options->output.PrintPassThrough ||m_options->nbest.include_passthrough) {
|
||||||
SD.options().nbest.include_passthrough) {
|
|
||||||
string pthru = PassthroughSGML(line,"passthrough");
|
string pthru = PassthroughSGML(line,"passthrough");
|
||||||
this->SetPassthroughInformation(pthru);
|
this->SetPassthroughInformation(pthru);
|
||||||
}
|
}
|
||||||
|
|
||||||
vector<size_t> xmlWalls;
|
vector<size_t> xmlWalls;
|
||||||
vector<pair<size_t, string> >placeholders;
|
vector<pair<size_t, string> >placeholders;
|
||||||
aux_interpret_xml(*opts, line, xmlWalls, placeholders);
|
aux_interpret_xml(line, xmlWalls, placeholders);
|
||||||
|
|
||||||
Phrase::CreateFromString(Input, factorOrder, line, NULL);
|
Phrase::CreateFromString(Input, m_options->input.factor_order, line, NULL);
|
||||||
|
|
||||||
ProcessPlaceholders(placeholders);
|
ProcessPlaceholders(placeholders);
|
||||||
|
|
||||||
if (SD.IsSyntax()) InitStartEndWord();
|
if (is_syntax(m_options->search.algo))
|
||||||
|
InitStartEndWord();
|
||||||
|
|
||||||
// now that we have final word positions in phrase (from
|
// now that we have final word positions in phrase (from
|
||||||
// CreateFromString), we can make input phrase objects to go with
|
// CreateFromString), we can make input phrase objects to go with
|
||||||
// our XmlOptions and create TranslationOptions
|
// our XmlOptions and create TranslationOptions
|
||||||
|
|
||||||
// only fill the vector if we are parsing XML
|
// only fill the vector if we are parsing XML
|
||||||
if (opts->input.xml_policy != XmlPassThrough) {
|
if (m_options->input.xml_policy != XmlPassThrough) {
|
||||||
m_xmlCoverageMap.assign(GetSize(), false);
|
m_xmlCoverageMap.assign(GetSize(), false);
|
||||||
BOOST_FOREACH(XmlOption const* o, m_xmlOptions) {
|
BOOST_FOREACH(XmlOption const* o, m_xmlOptions) {
|
||||||
Range const& r = o->range;
|
Range const& r = o->range;
|
||||||
@ -217,7 +210,7 @@ init(AllOptions::ptr const& opts, string line, std::vector<FactorType> const& fa
|
|||||||
m_reorderingConstraint.InitializeWalls(GetSize());
|
m_reorderingConstraint.InitializeWalls(GetSize());
|
||||||
|
|
||||||
// set reordering walls, if "-monotone-at-punction" is set
|
// set reordering walls, if "-monotone-at-punction" is set
|
||||||
if (SD.UseReorderingConstraint() && GetSize()) {
|
if (m_options->reordering.monotone_at_punct && GetSize()) {
|
||||||
Range r(0, GetSize()-1);
|
Range r(0, GetSize()-1);
|
||||||
m_reorderingConstraint.SetMonotoneAtPunctuation(GetSubString(r));
|
m_reorderingConstraint.SetMonotoneAtPunctuation(GetSubString(r));
|
||||||
}
|
}
|
||||||
@ -232,14 +225,12 @@ init(AllOptions::ptr const& opts, string line, std::vector<FactorType> const& fa
|
|||||||
|
|
||||||
int
|
int
|
||||||
Sentence::
|
Sentence::
|
||||||
Read(std::istream& in,
|
Read(std::istream& in)
|
||||||
const std::vector<FactorType>& factorOrder,
|
|
||||||
AllOptions const& opts)
|
|
||||||
{
|
{
|
||||||
std::string line;
|
std::string line;
|
||||||
if (getline(in, line, '\n').eof())
|
if (getline(in, line, '\n').eof())
|
||||||
return 0;
|
return 0;
|
||||||
init(m_options, line, factorOrder);
|
init(line);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -247,7 +238,7 @@ void
|
|||||||
Sentence::
|
Sentence::
|
||||||
ProcessPlaceholders(const std::vector< std::pair<size_t, std::string> > &placeholders)
|
ProcessPlaceholders(const std::vector< std::pair<size_t, std::string> > &placeholders)
|
||||||
{
|
{
|
||||||
FactorType placeholderFactor = StaticData::Instance().options().input.placeholder_factor;
|
FactorType placeholderFactor = m_options->input.placeholder_factor;
|
||||||
if (placeholderFactor == NOT_FOUND) {
|
if (placeholderFactor == NOT_FOUND) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -325,7 +316,7 @@ void Sentence::GetXmlTranslationOptions(std::vector <TranslationOption*> &list,
|
|||||||
|
|
||||||
std::vector <ChartTranslationOptions*>
|
std::vector <ChartTranslationOptions*>
|
||||||
Sentence::
|
Sentence::
|
||||||
GetXmlChartTranslationOptions(AllOptions const& opts) const
|
GetXmlChartTranslationOptions() const
|
||||||
{
|
{
|
||||||
std::vector <ChartTranslationOptions*> ret;
|
std::vector <ChartTranslationOptions*> ret;
|
||||||
|
|
||||||
@ -333,7 +324,7 @@ GetXmlChartTranslationOptions(AllOptions const& opts) const
|
|||||||
// this code is a copy of the 1 in Sentence.
|
// this code is a copy of the 1 in Sentence.
|
||||||
|
|
||||||
//only fill the vector if we are parsing XML
|
//only fill the vector if we are parsing XML
|
||||||
if (opts.input.xml_policy != XmlPassThrough ) {
|
if (m_options->input.xml_policy != XmlPassThrough ) {
|
||||||
//TODO: needed to handle exclusive
|
//TODO: needed to handle exclusive
|
||||||
//for (size_t i=0; i<GetSize(); i++) {
|
//for (size_t i=0; i<GetSize(); i++) {
|
||||||
// m_xmlCoverageMap.push_back(false);
|
// m_xmlCoverageMap.push_back(false);
|
||||||
@ -374,12 +365,10 @@ CreateFromString(vector<FactorType> const& FOrder, string const& phraseString)
|
|||||||
}
|
}
|
||||||
|
|
||||||
Sentence::
|
Sentence::
|
||||||
Sentence(AllOptions::ptr const& opts, size_t const transId,
|
Sentence(AllOptions::ptr const& opts, size_t const transId, string stext)
|
||||||
string stext, vector<FactorType> const* IFO)
|
|
||||||
: InputType(opts, transId)
|
: InputType(opts, transId)
|
||||||
{
|
{
|
||||||
if (IFO) init(opts,stext, *IFO);
|
init(stext);
|
||||||
else init(opts, stext, opts->input.factor_order);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -64,8 +64,8 @@ protected:
|
|||||||
|
|
||||||
public:
|
public:
|
||||||
Sentence(AllOptions::ptr const& opts);
|
Sentence(AllOptions::ptr const& opts);
|
||||||
Sentence(AllOptions::ptr const& opts, size_t const transId, std::string stext,
|
Sentence(AllOptions::ptr const& opts, size_t const transId, std::string stext);
|
||||||
std::vector<FactorType> const* IFO = NULL);
|
// std::vector<FactorType> const* IFO = NULL);
|
||||||
// Sentence(size_t const transId, std::string const& stext);
|
// Sentence(size_t const transId, std::string const& stext);
|
||||||
~Sentence();
|
~Sentence();
|
||||||
|
|
||||||
@ -94,11 +94,11 @@ public:
|
|||||||
//! populates vector argument with XML force translation options for the specific range passed
|
//! populates vector argument with XML force translation options for the specific range passed
|
||||||
void GetXmlTranslationOptions(std::vector<TranslationOption*> &list) const;
|
void GetXmlTranslationOptions(std::vector<TranslationOption*> &list) const;
|
||||||
void GetXmlTranslationOptions(std::vector<TranslationOption*> &list, size_t startPos, size_t endPos) const;
|
void GetXmlTranslationOptions(std::vector<TranslationOption*> &list, size_t startPos, size_t endPos) const;
|
||||||
std::vector<ChartTranslationOptions*> GetXmlChartTranslationOptions(AllOptions const& opts) const;
|
std::vector<ChartTranslationOptions*> GetXmlChartTranslationOptions() const;
|
||||||
|
|
||||||
virtual int
|
virtual int
|
||||||
Read(std::istream& in, const std::vector<FactorType>& factorOrder,
|
Read(std::istream& in);
|
||||||
AllOptions const& opts);
|
// , const std::vector<FactorType>& factorOrder, AllOptions const& opts);
|
||||||
|
|
||||||
void Print(std::ostream& out) const;
|
void Print(std::ostream& out) const;
|
||||||
|
|
||||||
@ -115,9 +115,7 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void
|
void init(std::string line);
|
||||||
init(AllOptions::ptr const& opts, std::string line,
|
|
||||||
std::vector<FactorType> const& factorOrder);
|
|
||||||
|
|
||||||
std::vector<std::map<std::string,std::string> > const&
|
std::vector<std::map<std::string,std::string> > const&
|
||||||
GetDltMeta() const {
|
GetDltMeta() const {
|
||||||
@ -139,7 +137,7 @@ private:
|
|||||||
|
|
||||||
void
|
void
|
||||||
aux_interpret_xml
|
aux_interpret_xml
|
||||||
(AllOptions const& opts, std::string& line, std::vector<size_t> & xmlWalls,
|
(std::string& line, std::vector<size_t> & xmlWalls,
|
||||||
std::vector<std::pair<size_t, std::string> >& placeholders);
|
std::vector<std::pair<size_t, std::string> >& placeholders);
|
||||||
|
|
||||||
void
|
void
|
||||||
|
@ -61,16 +61,11 @@ bool g_mosesDebug = false;
|
|||||||
StaticData StaticData::s_instance;
|
StaticData StaticData::s_instance;
|
||||||
|
|
||||||
StaticData::StaticData()
|
StaticData::StaticData()
|
||||||
: m_sourceStartPosMattersForRecombination(false)
|
: m_options(new AllOptions)
|
||||||
, m_requireSortingAfterSourceContext(false)
|
, m_requireSortingAfterSourceContext(false)
|
||||||
// , m_isAlwaysCreateDirectTranslationOption(false)
|
|
||||||
, m_currentWeightSetting("default")
|
, m_currentWeightSetting("default")
|
||||||
, m_treeStructure(NULL)
|
, m_treeStructure(NULL)
|
||||||
{
|
{
|
||||||
m_xmlBrackets.first="<";
|
|
||||||
m_xmlBrackets.second=">";
|
|
||||||
|
|
||||||
// memory pools
|
|
||||||
Phrase::InitializeMemPool();
|
Phrase::InitializeMemPool();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -123,34 +118,6 @@ StaticData
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
|
||||||
StaticData
|
|
||||||
::ini_input_options()
|
|
||||||
{
|
|
||||||
const PARAM_VEC *params;
|
|
||||||
|
|
||||||
m_parameter->SetParameter(m_continuePartialTranslation,
|
|
||||||
"continue-partial-translation", false );
|
|
||||||
|
|
||||||
// specify XML tags opening and closing brackets for XML option
|
|
||||||
params = m_parameter->GetParam("xml-brackets");
|
|
||||||
if (params && params->size()) {
|
|
||||||
std::vector<std::string> brackets = Tokenize(params->at(0));
|
|
||||||
if(brackets.size()!=2) {
|
|
||||||
cerr << "invalid xml-brackets value, must specify exactly 2 blank-delimited strings for XML tags opening and closing brackets" << endl;
|
|
||||||
exit(1);
|
|
||||||
}
|
|
||||||
m_xmlBrackets.first= brackets[0];
|
|
||||||
m_xmlBrackets.second=brackets[1];
|
|
||||||
VERBOSE(1,"XML tags opening and closing brackets for XML input are: "
|
|
||||||
<< m_xmlBrackets.first << " and " << m_xmlBrackets.second << endl);
|
|
||||||
}
|
|
||||||
|
|
||||||
m_parameter->SetParameter(m_defaultNonTermOnlyForEmptyRange,
|
|
||||||
"default-non-term-for-empty-range-only", false );
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
bool
|
bool
|
||||||
StaticData
|
StaticData
|
||||||
::ini_output_options()
|
::ini_output_options()
|
||||||
@ -161,8 +128,6 @@ StaticData
|
|||||||
m_parameter->SetParameter(m_verboseLevel, "verbose", (size_t) 1);
|
m_parameter->SetParameter(m_verboseLevel, "verbose", (size_t) 1);
|
||||||
|
|
||||||
|
|
||||||
m_parameter->SetParameter(m_includeLHSInSearchGraph,
|
|
||||||
"include-lhs-in-search-graph", false );
|
|
||||||
|
|
||||||
m_parameter->SetParameter<string>(m_outputUnknownsFile,
|
m_parameter->SetParameter<string>(m_outputUnknownsFile,
|
||||||
"output-unknowns", "");
|
"output-unknowns", "");
|
||||||
@ -218,7 +183,9 @@ bool StaticData::LoadData(Parameter *parameter)
|
|||||||
|
|
||||||
const PARAM_VEC *params;
|
const PARAM_VEC *params;
|
||||||
|
|
||||||
m_options.init(*parameter);
|
m_options->init(*parameter);
|
||||||
|
if (is_syntax(m_options->search.algo))
|
||||||
|
m_options->syntax.LoadNonTerminals(*parameter, FactorCollection::Instance());
|
||||||
|
|
||||||
if (IsSyntax())
|
if (IsSyntax())
|
||||||
LoadChartDecodingParameters();
|
LoadChartDecodingParameters();
|
||||||
@ -229,7 +196,6 @@ bool StaticData::LoadData(Parameter *parameter)
|
|||||||
m_parameter->SetParameter<string>(m_factorDelimiter, "factor-delimiter", "|");
|
m_parameter->SetParameter<string>(m_factorDelimiter, "factor-delimiter", "|");
|
||||||
m_parameter->SetParameter<size_t>(m_lmcache_cleanup_threshold, "clean-lm-cache", 1);
|
m_parameter->SetParameter<size_t>(m_lmcache_cleanup_threshold, "clean-lm-cache", 1);
|
||||||
|
|
||||||
ini_input_options();
|
|
||||||
m_bookkeeping_options.init(*parameter);
|
m_bookkeeping_options.init(*parameter);
|
||||||
if (!ini_output_options()) return false;
|
if (!ini_output_options()) return false;
|
||||||
|
|
||||||
@ -876,7 +842,7 @@ StaticData
|
|||||||
|
|
||||||
// FIXME Does this make sense for F2S? Perhaps it should be changed once
|
// FIXME Does this make sense for F2S? Perhaps it should be changed once
|
||||||
// FIXME the pipeline uses RuleTable consistently.
|
// FIXME the pipeline uses RuleTable consistently.
|
||||||
SearchAlgorithm algo = m_options.search.algo;
|
SearchAlgorithm algo = m_options->search.algo;
|
||||||
if (algo == SyntaxS2T || algo == SyntaxT2S ||
|
if (algo == SyntaxS2T || algo == SyntaxT2S ||
|
||||||
algo == SyntaxT2S_SCFG || algo == SyntaxF2S) {
|
algo == SyntaxT2S_SCFG || algo == SyntaxF2S) {
|
||||||
// Automatically override PhraseDictionary{Memory,Scope3}. This will
|
// Automatically override PhraseDictionary{Memory,Scope3}. This will
|
||||||
|
@ -72,7 +72,7 @@ private:
|
|||||||
static StaticData s_instance;
|
static StaticData s_instance;
|
||||||
protected:
|
protected:
|
||||||
Parameter *m_parameter;
|
Parameter *m_parameter;
|
||||||
AllOptions m_options;
|
boost::shared_ptr<AllOptions> m_options;
|
||||||
|
|
||||||
mutable ScoreComponentCollection m_allWeights;
|
mutable ScoreComponentCollection m_allWeights;
|
||||||
|
|
||||||
@ -95,31 +95,22 @@ protected:
|
|||||||
size_t m_latticeSamplesSize;
|
size_t m_latticeSamplesSize;
|
||||||
|
|
||||||
std::string m_latticeSamplesFilePath;
|
std::string m_latticeSamplesFilePath;
|
||||||
// bool m_dropUnknown; //! false = treat unknown words as unknowns, and translate them as themselves; true = drop (ignore) them
|
// bool m_wordDeletionEnabled;
|
||||||
// bool m_markUnknown; //! false = treat unknown words as unknowns, and translate them as themselves; true = mark and (ignore) them
|
|
||||||
// std::string m_unknownWordPrefix;
|
|
||||||
// std::string m_unknownWordSuffix;
|
|
||||||
bool m_wordDeletionEnabled;
|
|
||||||
|
|
||||||
// bool m_disableDiscarding;
|
|
||||||
bool m_printAllDerivations;
|
bool m_printAllDerivations;
|
||||||
bool m_printTranslationOptions;
|
bool m_printTranslationOptions;
|
||||||
|
|
||||||
bool m_sourceStartPosMattersForRecombination;
|
// bool m_sourceStartPosMattersForRecombination;
|
||||||
bool m_requireSortingAfterSourceContext;
|
bool m_requireSortingAfterSourceContext;
|
||||||
|
|
||||||
mutable size_t m_verboseLevel;
|
mutable size_t m_verboseLevel;
|
||||||
|
|
||||||
std::string m_factorDelimiter; //! by default, |, but it can be changed
|
std::string m_factorDelimiter; //! by default, |, but it can be changed
|
||||||
|
|
||||||
// XmlInputType m_xmlInputType; //! method for handling sentence XML input
|
|
||||||
std::pair<std::string,std::string> m_xmlBrackets; //! strings to use as XML tags' opening and closing brackets. Default are "<" and ">"
|
std::pair<std::string,std::string> m_xmlBrackets; //! strings to use as XML tags' opening and closing brackets. Default are "<" and ">"
|
||||||
|
|
||||||
size_t m_lmcache_cleanup_threshold; //! number of translations after which LM claenup is performed (0=never, N=after N translations; default is 1)
|
size_t m_lmcache_cleanup_threshold; //! number of translations after which LM claenup is performed (0=never, N=after N translations; default is 1)
|
||||||
|
|
||||||
// bool m_isAlwaysCreateDirectTranslationOption;
|
|
||||||
//! constructor. only the 1 static variable can be created
|
|
||||||
|
|
||||||
bool m_includeLHSInSearchGraph; //! include LHS of rules in search graph
|
bool m_includeLHSInSearchGraph; //! include LHS of rules in search graph
|
||||||
std::string m_outputUnknownsFile; //! output unknowns in this file
|
std::string m_outputUnknownsFile; //! output unknowns in this file
|
||||||
|
|
||||||
@ -144,7 +135,6 @@ protected:
|
|||||||
std::map< std::string, std::set< std::string > > m_weightSettingIgnoreFF; // feature function
|
std::map< std::string, std::set< std::string > > m_weightSettingIgnoreFF; // feature function
|
||||||
std::map< std::string, std::set< size_t > > m_weightSettingIgnoreDP; // decoding path
|
std::map< std::string, std::set< size_t > > m_weightSettingIgnoreDP; // decoding path
|
||||||
|
|
||||||
// FactorType m_placeHolderFactor;
|
|
||||||
bool m_useLegacyPT;
|
bool m_useLegacyPT;
|
||||||
bool m_defaultNonTermOnlyForEmptyRange;
|
bool m_defaultNonTermOnlyForEmptyRange;
|
||||||
S2TParsingAlgorithm m_s2tParsingAlgorithm;
|
S2TParsingAlgorithm m_s2tParsingAlgorithm;
|
||||||
@ -174,7 +164,6 @@ protected:
|
|||||||
|
|
||||||
const StatefulFeatureFunction* m_treeStructure;
|
const StatefulFeatureFunction* m_treeStructure;
|
||||||
|
|
||||||
void ini_input_options();
|
|
||||||
void ini_oov_options();
|
void ini_oov_options();
|
||||||
bool ini_output_options();
|
bool ini_output_options();
|
||||||
bool ini_performance_options();
|
bool ini_performance_options();
|
||||||
@ -182,9 +171,6 @@ protected:
|
|||||||
void initialize_features();
|
void initialize_features();
|
||||||
public:
|
public:
|
||||||
|
|
||||||
// bool IsAlwaysCreateDirectTranslationOption() const {
|
|
||||||
// return m_isAlwaysCreateDirectTranslationOption;
|
|
||||||
// }
|
|
||||||
//! destructor
|
//! destructor
|
||||||
~StaticData();
|
~StaticData();
|
||||||
|
|
||||||
@ -219,51 +205,35 @@ public:
|
|||||||
return *m_parameter;
|
return *m_parameter;
|
||||||
}
|
}
|
||||||
|
|
||||||
AllOptions const&
|
AllOptions::ptr const
|
||||||
options() const {
|
options() const {
|
||||||
return m_options;
|
return m_options;
|
||||||
}
|
}
|
||||||
|
|
||||||
AllOptions&
|
// AllOptions&
|
||||||
options() {
|
// options() {
|
||||||
return m_options;
|
// return m_options;
|
||||||
}
|
// }
|
||||||
|
|
||||||
inline bool
|
// inline bool
|
||||||
GetSourceStartPosMattersForRecombination() const {
|
// GetSourceStartPosMattersForRecombination() const {
|
||||||
return m_sourceStartPosMattersForRecombination;
|
// return m_sourceStartPosMattersForRecombination;
|
||||||
}
|
// }
|
||||||
|
|
||||||
bool
|
|
||||||
IsWordDeletionEnabled() const {
|
|
||||||
// return m_wordDeletionEnabled;
|
|
||||||
return m_options.unk.word_deletion_enabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
int
|
|
||||||
GetMaxDistortion() const {
|
|
||||||
return m_options.reordering.max_distortion;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool
|
|
||||||
UseReorderingConstraint() const {
|
|
||||||
return m_reorderingConstraint;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool
|
bool
|
||||||
UseEarlyDiscarding() const {
|
UseEarlyDiscarding() const {
|
||||||
return m_options.search.early_discarding_threshold
|
return m_options->search.early_discarding_threshold
|
||||||
!= -std::numeric_limits<float>::infinity();
|
!= -std::numeric_limits<float>::infinity();
|
||||||
}
|
}
|
||||||
|
|
||||||
bool
|
bool
|
||||||
UseEarlyDistortionCost() const {
|
UseEarlyDistortionCost() const {
|
||||||
return m_options.reordering.use_early_distortion_cost;
|
return m_options->reordering.use_early_distortion_cost;
|
||||||
}
|
}
|
||||||
|
|
||||||
float
|
float
|
||||||
GetTranslationOptionThreshold() const {
|
GetTranslationOptionThreshold() const {
|
||||||
return m_options.search.trans_opt_threshold;
|
return m_options->search.trans_opt_threshold;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t
|
size_t
|
||||||
@ -288,7 +258,7 @@ public:
|
|||||||
|
|
||||||
bool IsSyntax(SearchAlgorithm algo = DefaultSearchAlgorithm) const {
|
bool IsSyntax(SearchAlgorithm algo = DefaultSearchAlgorithm) const {
|
||||||
if (algo == DefaultSearchAlgorithm)
|
if (algo == DefaultSearchAlgorithm)
|
||||||
algo = m_options.search.algo;
|
algo = m_options->search.algo;
|
||||||
|
|
||||||
return (algo == CYKPlus || algo == ChartIncremental ||
|
return (algo == CYKPlus || algo == ChartIncremental ||
|
||||||
algo == SyntaxS2T || algo == SyntaxT2S ||
|
algo == SyntaxS2T || algo == SyntaxT2S ||
|
||||||
@ -333,37 +303,14 @@ public:
|
|||||||
return m_outputUnknownsFile;
|
return m_outputUnknownsFile;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool GetIncludeLHSInSearchGraph() const {
|
// bool GetIncludeLHSInSearchGraph() const {
|
||||||
return m_includeLHSInSearchGraph;
|
// return m_includeLHSInSearchGraph;
|
||||||
}
|
|
||||||
|
|
||||||
std::pair<std::string,std::string> GetXmlBrackets() const {
|
|
||||||
return m_xmlBrackets;
|
|
||||||
}
|
|
||||||
|
|
||||||
// bool PrintTranslationOptions() const {
|
|
||||||
// return m_printTranslationOptions;
|
|
||||||
// }
|
|
||||||
|
|
||||||
// bool PrintAllDerivations() const {
|
|
||||||
// return m_printAllDerivations;
|
|
||||||
// }
|
// }
|
||||||
|
|
||||||
const UnknownLHSList &GetUnknownLHS() const {
|
const UnknownLHSList &GetUnknownLHS() const {
|
||||||
return m_unknownLHS;
|
return m_unknownLHS;
|
||||||
}
|
}
|
||||||
|
|
||||||
const Word &GetInputDefaultNonTerminal() const {
|
|
||||||
return m_inputDefaultNonTerminal;
|
|
||||||
}
|
|
||||||
const Word &GetOutputDefaultNonTerminal() const {
|
|
||||||
return m_outputDefaultNonTerminal;
|
|
||||||
}
|
|
||||||
|
|
||||||
SourceLabelOverlap GetSourceLabelOverlap() const {
|
|
||||||
return m_sourceLabelOverlap;
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t GetRuleLimit() const {
|
size_t GetRuleLimit() const {
|
||||||
return m_ruleLimit;
|
return m_ruleLimit;
|
||||||
}
|
}
|
||||||
@ -473,7 +420,6 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
//sentence (and thread) specific initialisationn and cleanup
|
//sentence (and thread) specific initialisationn and cleanup
|
||||||
// void InitializeForInput(const InputType& source, ttaskptr const& ttask) const;
|
|
||||||
void InitializeForInput(ttasksptr const& ttask) const;
|
void InitializeForInput(ttasksptr const& ttask) const;
|
||||||
void CleanUpAfterSentenceProcessing(ttasksptr const& ttask) const;
|
void CleanUpAfterSentenceProcessing(ttasksptr const& ttask) const;
|
||||||
|
|
||||||
@ -525,10 +471,6 @@ public:
|
|||||||
return m_defaultNonTermOnlyForEmptyRange;
|
return m_defaultNonTermOnlyForEmptyRange;
|
||||||
}
|
}
|
||||||
|
|
||||||
// S2TParsingAlgorithm GetS2TParsingAlgorithm() const {
|
|
||||||
// return m_s2tParsingAlgorithm;
|
|
||||||
// }
|
|
||||||
|
|
||||||
bool RequireSortingAfterSourceContext() const {
|
bool RequireSortingAfterSourceContext() const {
|
||||||
return m_requireSortingAfterSourceContext;
|
return m_requireSortingAfterSourceContext;
|
||||||
}
|
}
|
||||||
|
@ -3,9 +3,8 @@
|
|||||||
#include <sstream>
|
#include <sstream>
|
||||||
|
|
||||||
#include "moses/FF/UnknownWordPenaltyProducer.h"
|
#include "moses/FF/UnknownWordPenaltyProducer.h"
|
||||||
#include "moses/StaticData.h"
|
|
||||||
#include "util/string_stream.hh"
|
#include "util/string_stream.hh"
|
||||||
|
#include "moses/parameters/AllOptions.h"
|
||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
namespace Syntax
|
namespace Syntax
|
||||||
@ -14,13 +13,13 @@ namespace F2S
|
|||||||
{
|
{
|
||||||
|
|
||||||
GlueRuleSynthesizer::
|
GlueRuleSynthesizer::
|
||||||
GlueRuleSynthesizer(HyperTree &trie, const std::vector<FactorType> &iFactors)
|
GlueRuleSynthesizer(Moses::AllOptions const& opts, HyperTree &trie)
|
||||||
: m_hyperTree(trie)
|
: m_input_default_nonterminal(opts.syntax.input_default_non_terminal)
|
||||||
|
, m_output_default_nonterminal(opts.syntax.output_default_non_terminal)
|
||||||
|
, m_hyperTree(trie)
|
||||||
{
|
{
|
||||||
// const std::vector<FactorType> &inputFactorOrder =
|
|
||||||
// StaticData::Instance().GetInputFactorOrder();
|
|
||||||
Word *lhs = NULL;
|
Word *lhs = NULL;
|
||||||
m_dummySourcePhrase.CreateFromString(Input, iFactors, "hello", &lhs);
|
m_dummySourcePhrase.CreateFromString(Input, opts.input.factor_order, "hello", &lhs);
|
||||||
delete lhs;
|
delete lhs;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -47,11 +46,10 @@ void GlueRuleSynthesizer::SynthesizeHyperPath(const Forest::Hyperedge &e,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
|
TargetPhrase*
|
||||||
const Forest::Hyperedge &e)
|
GlueRuleSynthesizer::
|
||||||
|
SynthesizeTargetPhrase(const Forest::Hyperedge &e)
|
||||||
{
|
{
|
||||||
const StaticData &staticData = StaticData::Instance();
|
|
||||||
|
|
||||||
const UnknownWordPenaltyProducer &unknownWordPenaltyProducer =
|
const UnknownWordPenaltyProducer &unknownWordPenaltyProducer =
|
||||||
UnknownWordPenaltyProducer::Instance();
|
UnknownWordPenaltyProducer::Instance();
|
||||||
|
|
||||||
@ -61,7 +59,7 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
|
|||||||
for (std::size_t i = 0; i < e.tail.size(); ++i) {
|
for (std::size_t i = 0; i < e.tail.size(); ++i) {
|
||||||
const Word &symbol = e.tail[i]->pvertex.symbol;
|
const Word &symbol = e.tail[i]->pvertex.symbol;
|
||||||
if (symbol.IsNonTerminal()) {
|
if (symbol.IsNonTerminal()) {
|
||||||
targetPhrase->AddWord(staticData.GetOutputDefaultNonTerminal());
|
targetPhrase->AddWord(m_output_default_nonterminal);
|
||||||
} else {
|
} else {
|
||||||
// TODO Check this
|
// TODO Check this
|
||||||
Word &targetWord = targetPhrase->AddWord();
|
Word &targetWord = targetPhrase->AddWord();
|
||||||
@ -75,7 +73,7 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
|
|||||||
float score = LOWEST_SCORE;
|
float score = LOWEST_SCORE;
|
||||||
targetPhrase->GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, score);
|
targetPhrase->GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, score);
|
||||||
targetPhrase->EvaluateInIsolation(m_dummySourcePhrase);
|
targetPhrase->EvaluateInIsolation(m_dummySourcePhrase);
|
||||||
Word *targetLhs = new Word(staticData.GetOutputDefaultNonTerminal());
|
Word *targetLhs = new Word(m_output_default_nonterminal);
|
||||||
targetPhrase->SetTargetLHS(targetLhs);
|
targetPhrase->SetTargetLHS(targetLhs);
|
||||||
targetPhrase->SetAlignmentInfo(alignmentSS.str());
|
targetPhrase->SetAlignmentInfo(alignmentSS.str());
|
||||||
|
|
||||||
|
@ -9,6 +9,7 @@
|
|||||||
|
|
||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
|
class AllOptions;
|
||||||
namespace Syntax
|
namespace Syntax
|
||||||
{
|
{
|
||||||
namespace F2S
|
namespace F2S
|
||||||
@ -16,8 +17,10 @@ namespace F2S
|
|||||||
|
|
||||||
class GlueRuleSynthesizer : public HyperTreeCreator
|
class GlueRuleSynthesizer : public HyperTreeCreator
|
||||||
{
|
{
|
||||||
|
Word m_input_default_nonterminal;
|
||||||
|
Word m_output_default_nonterminal;
|
||||||
public:
|
public:
|
||||||
GlueRuleSynthesizer(HyperTree &, std::vector<FactorType> const& iFactors);
|
GlueRuleSynthesizer(Moses::AllOptions const& opts, HyperTree &);
|
||||||
|
|
||||||
// Synthesize the minimal, monotone rule that can be applied to the given
|
// Synthesize the minimal, monotone rule that can be applied to the given
|
||||||
// hyperedge and add it to the rule trie.
|
// hyperedge and add it to the rule trie.
|
||||||
|
@ -60,9 +60,9 @@ void Manager<RuleMatcher>::Decode()
|
|||||||
const StaticData &staticData = StaticData::Instance();
|
const StaticData &staticData = StaticData::Instance();
|
||||||
|
|
||||||
// Get various pruning-related constants.
|
// Get various pruning-related constants.
|
||||||
const std::size_t popLimit = staticData.options().cube.pop_limit;
|
const std::size_t popLimit = staticData.options()->cube.pop_limit;
|
||||||
const std::size_t ruleLimit = staticData.GetRuleLimit();
|
const std::size_t ruleLimit = staticData.GetRuleLimit();
|
||||||
const std::size_t stackLimit = staticData.options().search.stack_size;
|
const std::size_t stackLimit = staticData.options()->search.stack_size;
|
||||||
|
|
||||||
// Initialize the stacks.
|
// Initialize the stacks.
|
||||||
InitializeStacks();
|
InitializeStacks();
|
||||||
@ -74,8 +74,7 @@ void Manager<RuleMatcher>::Decode()
|
|||||||
RuleMatcherCallback callback(m_stackMap, ruleLimit);
|
RuleMatcherCallback callback(m_stackMap, ruleLimit);
|
||||||
|
|
||||||
// Create a glue rule synthesizer.
|
// Create a glue rule synthesizer.
|
||||||
GlueRuleSynthesizer glueRuleSynthesizer(*m_glueRuleTrie,
|
GlueRuleSynthesizer glueRuleSynthesizer(*options(), *m_glueRuleTrie);
|
||||||
options()->input.factor_order);
|
|
||||||
|
|
||||||
// Sort the input forest's vertices into bottom-up topological order.
|
// Sort the input forest's vertices into bottom-up topological order.
|
||||||
std::vector<const Forest::Vertex *> sortedVertices;
|
std::vector<const Forest::Vertex *> sortedVertices;
|
||||||
@ -256,7 +255,7 @@ void Manager<RuleMatcher>::ExtractKBest(
|
|||||||
// with 0 being 'unlimited.' This actually sets a large-ish limit in case
|
// with 0 being 'unlimited.' This actually sets a large-ish limit in case
|
||||||
// too many translations are identical.
|
// too many translations are identical.
|
||||||
const StaticData &staticData = StaticData::Instance();
|
const StaticData &staticData = StaticData::Instance();
|
||||||
const std::size_t nBestFactor = staticData.options().nbest.factor;
|
const std::size_t nBestFactor = staticData.options()->nbest.factor;
|
||||||
std::size_t numDerivations = (nBestFactor == 0) ? k*1000 : k*nBestFactor;
|
std::size_t numDerivations = (nBestFactor == 0) ? k*1000 : k*nBestFactor;
|
||||||
|
|
||||||
// Extract the derivations.
|
// Extract the derivations.
|
||||||
|
@ -75,7 +75,7 @@ void KBestExtractor::Extract(
|
|||||||
// Generate the target-side yield of the derivation d.
|
// Generate the target-side yield of the derivation d.
|
||||||
Phrase KBestExtractor::GetOutputPhrase(const Derivation &d)
|
Phrase KBestExtractor::GetOutputPhrase(const Derivation &d)
|
||||||
{
|
{
|
||||||
FactorType placeholderFactor = StaticData::Instance().options().input.placeholder_factor;
|
FactorType placeholderFactor = StaticData::Instance().options()->input.placeholder_factor;
|
||||||
|
|
||||||
Phrase ret(ARRAY_SIZE_INCR);
|
Phrase ret(ARRAY_SIZE_INCR);
|
||||||
|
|
||||||
|
@ -24,34 +24,35 @@ RuleTableFF::RuleTableFF(const std::string &line)
|
|||||||
s_instances.push_back(this);
|
s_instances.push_back(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
void RuleTableFF::Load(Moses::AllOptions const& opts)
|
void RuleTableFF::Load(Moses::AllOptions::ptr const& opts)
|
||||||
{
|
{
|
||||||
|
m_options = opts;
|
||||||
SetFeaturesToApply();
|
SetFeaturesToApply();
|
||||||
|
|
||||||
if (opts.search.algo == SyntaxF2S || opts.search.algo == SyntaxT2S) {
|
if (opts->search.algo == SyntaxF2S || opts->search.algo == SyntaxT2S) {
|
||||||
F2S::HyperTree *trie = new F2S::HyperTree(this);
|
F2S::HyperTree *trie = new F2S::HyperTree(this);
|
||||||
F2S::HyperTreeLoader loader;
|
F2S::HyperTreeLoader loader;
|
||||||
loader.Load(opts, m_input, m_output, m_filePath, *this, *trie, m_sourceTerminalSet);
|
loader.Load(*opts, m_input, m_output, m_filePath, *this, *trie, m_sourceTerminalSet);
|
||||||
m_table = trie;
|
m_table = trie;
|
||||||
} else if (opts.search.algo == SyntaxS2T) {
|
} else if (opts->search.algo == SyntaxS2T) {
|
||||||
S2TParsingAlgorithm algorithm = opts.syntax.s2t_parsing_algo; // staticData.GetS2TParsingAlgorithm();
|
S2TParsingAlgorithm algorithm = opts->syntax.s2t_parsing_algo; // staticData.GetS2TParsingAlgorithm();
|
||||||
if (algorithm == RecursiveCYKPlus) {
|
if (algorithm == RecursiveCYKPlus) {
|
||||||
S2T::RuleTrieCYKPlus *trie = new S2T::RuleTrieCYKPlus(this);
|
S2T::RuleTrieCYKPlus *trie = new S2T::RuleTrieCYKPlus(this);
|
||||||
S2T::RuleTrieLoader loader;
|
S2T::RuleTrieLoader loader;
|
||||||
loader.Load(opts,m_input, m_output, m_filePath, *this, *trie);
|
loader.Load(*opts,m_input, m_output, m_filePath, *this, *trie);
|
||||||
m_table = trie;
|
m_table = trie;
|
||||||
} else if (algorithm == Scope3) {
|
} else if (algorithm == Scope3) {
|
||||||
S2T::RuleTrieScope3 *trie = new S2T::RuleTrieScope3(this);
|
S2T::RuleTrieScope3 *trie = new S2T::RuleTrieScope3(this);
|
||||||
S2T::RuleTrieLoader loader;
|
S2T::RuleTrieLoader loader;
|
||||||
loader.Load(opts, m_input, m_output, m_filePath, *this, *trie);
|
loader.Load(*opts, m_input, m_output, m_filePath, *this, *trie);
|
||||||
m_table = trie;
|
m_table = trie;
|
||||||
} else {
|
} else {
|
||||||
UTIL_THROW2("ERROR: unhandled S2T parsing algorithm");
|
UTIL_THROW2("ERROR: unhandled S2T parsing algorithm");
|
||||||
}
|
}
|
||||||
} else if (opts.search.algo == SyntaxT2S_SCFG) {
|
} else if (opts->search.algo == SyntaxT2S_SCFG) {
|
||||||
T2S::RuleTrie *trie = new T2S::RuleTrie(this);
|
T2S::RuleTrie *trie = new T2S::RuleTrie(this);
|
||||||
T2S::RuleTrieLoader loader;
|
T2S::RuleTrieLoader loader;
|
||||||
loader.Load(opts, m_input, m_output, m_filePath, *this, *trie);
|
loader.Load(*opts, m_input, m_output, m_filePath, *this, *trie);
|
||||||
m_table = trie;
|
m_table = trie;
|
||||||
} else {
|
} else {
|
||||||
UTIL_THROW2(
|
UTIL_THROW2(
|
||||||
|
@ -27,7 +27,7 @@ public:
|
|||||||
// FIXME Delete m_table?
|
// FIXME Delete m_table?
|
||||||
~RuleTableFF() {}
|
~RuleTableFF() {}
|
||||||
|
|
||||||
void Load(AllOptions const& opts);
|
void Load(AllOptions::ptr const& opts);
|
||||||
|
|
||||||
const RuleTable *GetTable() const {
|
const RuleTable *GetTable() const {
|
||||||
return m_table;
|
return m_table;
|
||||||
|
@ -163,9 +163,9 @@ void Manager<Parser>::Decode()
|
|||||||
const StaticData &staticData = StaticData::Instance();
|
const StaticData &staticData = StaticData::Instance();
|
||||||
|
|
||||||
// Get various pruning-related constants.
|
// Get various pruning-related constants.
|
||||||
const std::size_t popLimit = staticData.options().cube.pop_limit;
|
const std::size_t popLimit = staticData.options()->cube.pop_limit;
|
||||||
const std::size_t ruleLimit = staticData.GetRuleLimit();
|
const std::size_t ruleLimit = staticData.GetRuleLimit();
|
||||||
const std::size_t stackLimit = staticData.options().search.stack_size;
|
const std::size_t stackLimit = staticData.options()->search.stack_size;
|
||||||
|
|
||||||
// Initialise the PChart and SChart.
|
// Initialise the PChart and SChart.
|
||||||
InitializeCharts();
|
InitializeCharts();
|
||||||
@ -303,7 +303,7 @@ void Manager<Parser>::ExtractKBest(
|
|||||||
// with 0 being 'unlimited.' This actually sets a large-ish limit in case
|
// with 0 being 'unlimited.' This actually sets a large-ish limit in case
|
||||||
// too many translations are identical.
|
// too many translations are identical.
|
||||||
const StaticData &staticData = StaticData::Instance();
|
const StaticData &staticData = StaticData::Instance();
|
||||||
const std::size_t nBestFactor = staticData.options().nbest.factor;
|
const std::size_t nBestFactor = staticData.options()->nbest.factor;
|
||||||
std::size_t numDerivations = (nBestFactor == 0) ? k*1000 : k*nBestFactor;
|
std::size_t numDerivations = (nBestFactor == 0) ? k*1000 : k*nBestFactor;
|
||||||
|
|
||||||
// Extract the derivations.
|
// Extract the derivations.
|
||||||
|
@ -57,7 +57,7 @@ OovHandler<RuleTrie>::SynthesizeTargetLhs(const std::string &lhsStr)
|
|||||||
{
|
{
|
||||||
Word *targetLhs = new Word(true);
|
Word *targetLhs = new Word(true);
|
||||||
targetLhs->CreateFromString(Output,
|
targetLhs->CreateFromString(Output,
|
||||||
StaticData::Instance().options().output.factor_order,
|
StaticData::Instance().options()->output.factor_order,
|
||||||
lhsStr, true);
|
lhsStr, true);
|
||||||
UTIL_THROW_IF2(targetLhs->GetFactor(0) == NULL, "Null factor for target LHS");
|
UTIL_THROW_IF2(targetLhs->GetFactor(0) == NULL, "Null factor for target LHS");
|
||||||
return targetLhs;
|
return targetLhs;
|
||||||
@ -83,7 +83,7 @@ TargetPhrase *OovHandler<RuleTrie>::SynthesizeTargetPhrase(
|
|||||||
targetPhrase->EvaluateInIsolation(srcPhrase);
|
targetPhrase->EvaluateInIsolation(srcPhrase);
|
||||||
targetPhrase->SetTargetLHS(&targetLhs);
|
targetPhrase->SetTargetLHS(&targetLhs);
|
||||||
targetPhrase->SetAlignmentInfo("0-0");
|
targetPhrase->SetAlignmentInfo("0-0");
|
||||||
if (!SD.options().output.detailed_tree_transrep_filepath.empty() ||
|
if (!SD.options()->output.detailed_tree_transrep_filepath.empty() ||
|
||||||
SD.GetTreeStructure() != NULL) {
|
SD.GetTreeStructure() != NULL) {
|
||||||
std::string value = "[ " + targetLhs[0]->GetString().as_string() + " " +
|
std::string value = "[ " + targetLhs[0]->GetString().as_string() + " " +
|
||||||
oov[0]->GetString().as_string() + " ]";
|
oov[0]->GetString().as_string() + " ]";
|
||||||
@ -96,7 +96,7 @@ TargetPhrase *OovHandler<RuleTrie>::SynthesizeTargetPhrase(
|
|||||||
template<typename RuleTrie>
|
template<typename RuleTrie>
|
||||||
bool OovHandler<RuleTrie>::ShouldDrop(const Word &oov)
|
bool OovHandler<RuleTrie>::ShouldDrop(const Word &oov)
|
||||||
{
|
{
|
||||||
if (!StaticData::Instance().options().unk.drop) {
|
if (!StaticData::Instance().options()->unk.drop) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
const Factor *f = oov[0]; // TODO hack. shouldn't know which factor is surface
|
const Factor *f = oov[0]; // TODO hack. shouldn't know which factor is surface
|
||||||
|
@ -11,7 +11,7 @@ namespace Syntax
|
|||||||
|
|
||||||
Phrase GetOneBestTargetYield(const SHyperedge &h)
|
Phrase GetOneBestTargetYield(const SHyperedge &h)
|
||||||
{
|
{
|
||||||
FactorType placeholderFactor = StaticData::Instance().options().input.placeholder_factor;
|
FactorType placeholderFactor = StaticData::Instance().options()->input.placeholder_factor;
|
||||||
|
|
||||||
Phrase ret(ARRAY_SIZE_INCR);
|
Phrase ret(ARRAY_SIZE_INCR);
|
||||||
|
|
||||||
|
@ -3,7 +3,7 @@
|
|||||||
#include <sstream>
|
#include <sstream>
|
||||||
|
|
||||||
#include "moses/FF/UnknownWordPenaltyProducer.h"
|
#include "moses/FF/UnknownWordPenaltyProducer.h"
|
||||||
#include "moses/StaticData.h"
|
#include <boost/scoped_ptr.hpp>
|
||||||
|
|
||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
@ -12,7 +12,9 @@ namespace Syntax
|
|||||||
namespace T2S
|
namespace T2S
|
||||||
{
|
{
|
||||||
|
|
||||||
void GlueRuleSynthesizer::SynthesizeRule(const InputTree::Node &node)
|
void
|
||||||
|
GlueRuleSynthesizer::
|
||||||
|
SynthesizeRule(const InputTree::Node &node)
|
||||||
{
|
{
|
||||||
const Word &sourceLhs = node.pvertex.symbol;
|
const Word &sourceLhs = node.pvertex.symbol;
|
||||||
boost::scoped_ptr<Phrase> sourceRhs(SynthesizeSourcePhrase(node));
|
boost::scoped_ptr<Phrase> sourceRhs(SynthesizeSourcePhrase(node));
|
||||||
@ -22,7 +24,9 @@ void GlueRuleSynthesizer::SynthesizeRule(const InputTree::Node &node)
|
|||||||
tpc->Add(tp);
|
tpc->Add(tp);
|
||||||
}
|
}
|
||||||
|
|
||||||
Phrase *GlueRuleSynthesizer::SynthesizeSourcePhrase(const InputTree::Node &node)
|
Phrase*
|
||||||
|
GlueRuleSynthesizer::
|
||||||
|
SynthesizeSourcePhrase(const InputTree::Node &node)
|
||||||
{
|
{
|
||||||
Phrase *phrase = new Phrase(node.children.size());
|
Phrase *phrase = new Phrase(node.children.size());
|
||||||
for (std::vector<InputTree::Node*>::const_iterator p = node.children.begin();
|
for (std::vector<InputTree::Node*>::const_iterator p = node.children.begin();
|
||||||
@ -37,11 +41,10 @@ Phrase *GlueRuleSynthesizer::SynthesizeSourcePhrase(const InputTree::Node &node)
|
|||||||
return phrase;
|
return phrase;
|
||||||
}
|
}
|
||||||
|
|
||||||
TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
|
TargetPhrase*
|
||||||
const InputTree::Node &node, const Phrase &sourceRhs)
|
GlueRuleSynthesizer::
|
||||||
|
SynthesizeTargetPhrase(const InputTree::Node &node, const Phrase &sourceRhs)
|
||||||
{
|
{
|
||||||
const StaticData &staticData = StaticData::Instance();
|
|
||||||
|
|
||||||
const UnknownWordPenaltyProducer &unknownWordPenaltyProducer =
|
const UnknownWordPenaltyProducer &unknownWordPenaltyProducer =
|
||||||
UnknownWordPenaltyProducer::Instance();
|
UnknownWordPenaltyProducer::Instance();
|
||||||
|
|
||||||
@ -51,7 +54,7 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
|
|||||||
for (std::size_t i = 0; i < node.children.size(); ++i) {
|
for (std::size_t i = 0; i < node.children.size(); ++i) {
|
||||||
const Word &symbol = node.children[i]->pvertex.symbol;
|
const Word &symbol = node.children[i]->pvertex.symbol;
|
||||||
if (symbol.IsNonTerminal()) {
|
if (symbol.IsNonTerminal()) {
|
||||||
targetPhrase->AddWord(staticData.GetOutputDefaultNonTerminal());
|
targetPhrase->AddWord(m_output_default_nonterminal);
|
||||||
} else {
|
} else {
|
||||||
// TODO Check this
|
// TODO Check this
|
||||||
Word &targetWord = targetPhrase->AddWord();
|
Word &targetWord = targetPhrase->AddWord();
|
||||||
@ -65,7 +68,7 @@ TargetPhrase *GlueRuleSynthesizer::SynthesizeTargetPhrase(
|
|||||||
float score = LOWEST_SCORE;
|
float score = LOWEST_SCORE;
|
||||||
targetPhrase->GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, score);
|
targetPhrase->GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, score);
|
||||||
targetPhrase->EvaluateInIsolation(sourceRhs);
|
targetPhrase->EvaluateInIsolation(sourceRhs);
|
||||||
Word *targetLhs = new Word(staticData.GetOutputDefaultNonTerminal());
|
Word *targetLhs = new Word(m_output_default_nonterminal);
|
||||||
targetPhrase->SetTargetLHS(targetLhs);
|
targetPhrase->SetTargetLHS(targetLhs);
|
||||||
targetPhrase->SetAlignmentInfo(alignmentSS.str());
|
targetPhrase->SetAlignmentInfo(alignmentSS.str());
|
||||||
|
|
||||||
|
@ -16,8 +16,12 @@ namespace T2S
|
|||||||
|
|
||||||
class GlueRuleSynthesizer : public RuleTrieCreator
|
class GlueRuleSynthesizer : public RuleTrieCreator
|
||||||
{
|
{
|
||||||
|
Word m_output_default_nonterminal;
|
||||||
public:
|
public:
|
||||||
GlueRuleSynthesizer(RuleTrie &trie) : m_ruleTrie(trie) {}
|
GlueRuleSynthesizer(RuleTrie &trie, Word dflt_nonterm)
|
||||||
|
: m_ruleTrie(trie)
|
||||||
|
, m_output_default_nonterminal(dflt_nonterm)
|
||||||
|
{}
|
||||||
|
|
||||||
// Synthesize the minimal, montone rule that can be applied to the given node
|
// Synthesize the minimal, montone rule that can be applied to the given node
|
||||||
// and add it to the rule trie.
|
// and add it to the rule trie.
|
||||||
|
@ -111,7 +111,8 @@ void Manager<RuleMatcher>::Decode()
|
|||||||
F2S::RuleMatcherCallback callback(m_stackMap, ruleLimit);
|
F2S::RuleMatcherCallback callback(m_stackMap, ruleLimit);
|
||||||
|
|
||||||
// Create a glue rule synthesizer.
|
// Create a glue rule synthesizer.
|
||||||
GlueRuleSynthesizer glueRuleSynthesizer(*m_glueRuleTrie);
|
Word dflt_nonterm = options()->syntax.output_default_non_terminal;
|
||||||
|
GlueRuleSynthesizer glueRuleSynthesizer(*m_glueRuleTrie, dflt_nonterm);
|
||||||
|
|
||||||
// Visit each node of the input tree in post-order.
|
// Visit each node of the input tree in post-order.
|
||||||
for (std::vector<InputTree::Node>::const_iterator p =
|
for (std::vector<InputTree::Node>::const_iterator p =
|
||||||
@ -215,7 +216,7 @@ void Manager<RuleMatcher>::ExtractKBest(
|
|||||||
// with 0 being 'unlimited.' This actually sets a large-ish limit in case
|
// with 0 being 'unlimited.' This actually sets a large-ish limit in case
|
||||||
// too many translations are identical.
|
// too many translations are identical.
|
||||||
const StaticData &staticData = StaticData::Instance();
|
const StaticData &staticData = StaticData::Instance();
|
||||||
const std::size_t nBestFactor = staticData.options().nbest.factor;
|
const std::size_t nBestFactor = staticData.options()->nbest.factor;
|
||||||
std::size_t numDerivations = (nBestFactor == 0) ? k*1000 : k*nBestFactor;
|
std::size_t numDerivations = (nBestFactor == 0) ? k*1000 : k*nBestFactor;
|
||||||
|
|
||||||
// Extract the derivations.
|
// Extract the derivations.
|
||||||
|
@ -47,9 +47,7 @@ void TabbedSentence::CreateFromString(const std::vector<FactorType> &factorOrder
|
|||||||
|
|
||||||
int
|
int
|
||||||
TabbedSentence::
|
TabbedSentence::
|
||||||
Read(std::istream& in,
|
Read(std::istream& in)
|
||||||
std::vector<FactorType> const& factorOrder,
|
|
||||||
AllOptions const& opts)
|
|
||||||
{
|
{
|
||||||
TabbedColumns allColumns;
|
TabbedColumns allColumns;
|
||||||
|
|
||||||
@ -60,17 +58,14 @@ Read(std::istream& in,
|
|||||||
boost::split(allColumns, line, boost::is_any_of("\t"));
|
boost::split(allColumns, line, boost::is_any_of("\t"));
|
||||||
|
|
||||||
if(allColumns.size() < 2) {
|
if(allColumns.size() < 2) {
|
||||||
std::stringstream dummyStream;
|
Sentence::init(line);
|
||||||
dummyStream << line << std::endl;
|
|
||||||
return Sentence::Read(dummyStream, factorOrder, opts);
|
|
||||||
} else {
|
} else {
|
||||||
m_columns.resize(allColumns.size() - 1);
|
m_columns.resize(allColumns.size() - 1);
|
||||||
std::copy(allColumns.begin() + 1, allColumns.end(), m_columns.begin());
|
std::copy(allColumns.begin() + 1, allColumns.end(), m_columns.begin());
|
||||||
|
Sentence::init(allColumns[0]);
|
||||||
std::stringstream dummyStream;
|
|
||||||
dummyStream << allColumns[0] << std::endl;
|
|
||||||
return Sentence::Read(dummyStream, factorOrder, opts);
|
|
||||||
}
|
}
|
||||||
|
return 1;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user