This commit is contained in:
Hieu Hoang 2015-05-02 11:45:24 +01:00
parent a4a7c14593
commit cc8c6b7b10
95 changed files with 2349 additions and 2409 deletions

View File

@ -94,8 +94,7 @@ void BleuScorer::setReferenceFiles(const vector<string>& referenceFiles)
mert::VocabularyFactory::GetVocabulary()->clear(); mert::VocabularyFactory::GetVocabulary()->clear();
//load reference data //load reference data
for (size_t i = 0; i < referenceFiles.size(); ++i) for (size_t i = 0; i < referenceFiles.size(); ++i) {
{
TRACE_ERR("Loading reference from " << referenceFiles[i] << endl); TRACE_ERR("Loading reference from " << referenceFiles[i] << endl);
ifstream ifs(referenceFiles[i].c_str()); ifstream ifs(referenceFiles[i].c_str());
@ -153,8 +152,7 @@ void BleuScorer::ProcessReferenceLine(const std::string& line, Reference* ref) c
bool BleuScorer::GetNextReferenceFromStreams(std::vector<boost::shared_ptr<std::ifstream> >& referenceStreams, Reference& ref) const bool BleuScorer::GetNextReferenceFromStreams(std::vector<boost::shared_ptr<std::ifstream> >& referenceStreams, Reference& ref) const
{ {
for (vector<boost::shared_ptr<ifstream> >::iterator ifs=referenceStreams.begin(); ifs!=referenceStreams.end(); ++ifs) for (vector<boost::shared_ptr<ifstream> >::iterator ifs=referenceStreams.begin(); ifs!=referenceStreams.end(); ++ifs) {
{
if (!(*ifs)) return false; if (!(*ifs)) return false;
string line; string line;
if (!getline(**ifs, line)) return false; if (!getline(**ifs, line)) return false;
@ -309,8 +307,7 @@ vector<float> BleuScorer::ScoreNbestList(const string& scoreFile, const string&
vector<FeatureDataIterator> featureDataIters; vector<FeatureDataIterator> featureDataIters;
vector<ScoreDataIterator> scoreDataIters; vector<ScoreDataIterator> scoreDataIters;
for (size_t i = 0; i < featureFiles.size(); ++i) for (size_t i = 0; i < featureFiles.size(); ++i) {
{
featureDataIters.push_back(FeatureDataIterator(featureFiles[i])); featureDataIters.push_back(FeatureDataIterator(featureFiles[i]));
scoreDataIters.push_back(ScoreDataIterator(scoreFiles[i])); scoreDataIters.push_back(ScoreDataIterator(scoreFiles[i]));
} }
@ -318,8 +315,7 @@ vector<float> BleuScorer::ScoreNbestList(const string& scoreFile, const string&
vector<pair<size_t,size_t> > hypotheses; vector<pair<size_t,size_t> > hypotheses;
UTIL_THROW_IF2(featureDataIters[0] == FeatureDataIterator::end(), UTIL_THROW_IF2(featureDataIters[0] == FeatureDataIterator::end(),
"At the end of feature data iterator"); "At the end of feature data iterator");
for (size_t i = 0; i < featureFiles.size(); ++i) for (size_t i = 0; i < featureFiles.size(); ++i) {
{
UTIL_THROW_IF2(featureDataIters[i] == FeatureDataIterator::end(), UTIL_THROW_IF2(featureDataIters[i] == FeatureDataIterator::end(),
"Feature file " << i << " ended prematurely"); "Feature file " << i << " ended prematurely");
UTIL_THROW_IF2(scoreDataIters[i] == ScoreDataIterator::end(), UTIL_THROW_IF2(scoreDataIters[i] == ScoreDataIterator::end(),

View File

@ -13,7 +13,8 @@
using namespace std; using namespace std;
using namespace MosesTuning; using namespace MosesTuning;
BOOST_AUTO_TEST_CASE(viterbi_simple_lattice) { BOOST_AUTO_TEST_CASE(viterbi_simple_lattice)
{
Vocab vocab; Vocab vocab;
WordVec words; WordVec words;
string wordStrings[] = string wordStrings[] =
@ -244,7 +245,8 @@ BOOST_AUTO_TEST_CASE(viterbi_3branch_lattice)
BOOST_CHECK_EQUAL(6, hopeHypo.bleuStats[8]); BOOST_CHECK_EQUAL(6, hopeHypo.bleuStats[8]);
} }
BOOST_AUTO_TEST_CASE(viterbi_full_hypergraph) { BOOST_AUTO_TEST_CASE(viterbi_full_hypergraph)
{
Vocab vocab; Vocab vocab;
//References //References
ReferenceSet references; ReferenceSet references;

View File

@ -11,7 +11,8 @@ how many of the features are really "dense". This is because in hg mira
all features (sparse and dense) are to get rolled in to SparseVector all features (sparse and dense) are to get rolled in to SparseVector
*/ */
BOOST_AUTO_TEST_CASE(from_sparse) { BOOST_AUTO_TEST_CASE(from_sparse)
{
SparseVector sp; SparseVector sp;
sp.set("dense0", 0.2); sp.set("dense0", 0.2);
sp.set("dense1", 0.3); sp.set("dense1", 0.3);

View File

@ -32,8 +32,7 @@ int main(int argc, char **argv)
// initialize reference streams // initialize reference streams
std::vector<boost::shared_ptr<std::ifstream> > refStreams; std::vector<boost::shared_ptr<std::ifstream> > refStreams;
for (std::vector<std::string>::const_iterator refFile=refFiles.begin(); refFile!=refFiles.end(); ++refFile) for (std::vector<std::string>::const_iterator refFile=refFiles.begin(); refFile!=refFiles.end(); ++refFile) {
{
TRACE_ERR("Loading reference from " << *refFile << std::endl); TRACE_ERR("Loading reference from " << *refFile << std::endl);
boost::shared_ptr<std::ifstream> ifs(new std::ifstream(refFile->c_str())); boost::shared_ptr<std::ifstream> ifs(new std::ifstream(refFile->c_str()));
UTIL_THROW_IF2(!ifs, "Cannot open " << *refFile); UTIL_THROW_IF2(!ifs, "Cannot open " << *refFile);
@ -44,8 +43,7 @@ int main(int argc, char **argv)
std::string nbestLine; std::string nbestLine;
int sid = -1; int sid = -1;
Reference ref; Reference ref;
while ( getline(std::cin, nbestLine) ) while ( getline(std::cin, nbestLine) ) {
{
std::vector<std::string> items; std::vector<std::string> items;
Moses::TokenizeMultiCharSeparator(items, nbestLine, " ||| "); Moses::TokenizeMultiCharSeparator(items, nbestLine, " ||| ");
int sidCurrent = Moses::Scan<int>(items[0]); int sidCurrent = Moses::Scan<int>(items[0]);

View File

@ -34,8 +34,7 @@ int main(int argc, char **argv)
// initialize reference streams // initialize reference streams
vector<boost::shared_ptr<ifstream> > refStreams; vector<boost::shared_ptr<ifstream> > refStreams;
for (vector<string>::const_iterator refFile=refFiles.begin(); refFile!=refFiles.end(); ++refFile) for (vector<string>::const_iterator refFile=refFiles.begin(); refFile!=refFiles.end(); ++refFile) {
{
TRACE_ERR("Loading reference from " << *refFile << endl); TRACE_ERR("Loading reference from " << *refFile << endl);
boost::shared_ptr<ifstream> ifs(new ifstream(refFile->c_str())); boost::shared_ptr<ifstream> ifs(new ifstream(refFile->c_str()));
UTIL_THROW_IF2(!ifs, "Cannot open " << *refFile); UTIL_THROW_IF2(!ifs, "Cannot open " << *refFile);
@ -45,8 +44,7 @@ int main(int argc, char **argv)
// load sentences, preparing statistics, score // load sentences, preparing statistics, score
string hypothesisLine; string hypothesisLine;
size_t sid = 0; size_t sid = 0;
while (getline(std::cin, hypothesisLine)) while (getline(std::cin, hypothesisLine)) {
{
Reference ref; Reference ref;
if (!scorer.GetNextReferenceFromStreams(refStreams, ref)) { if (!scorer.GetNextReferenceFromStreams(refStreams, ref)) {
UTIL_THROW2("Missing references"); UTIL_THROW2("Missing references");

View File

@ -177,8 +177,7 @@ int main(int argc, char* argv[])
const vector<float>& scale_grid = grid.getGrid(lmbr_scale); const vector<float>& scale_grid = grid.getGrid(lmbr_scale);
boost::shared_ptr<InputType> source; boost::shared_ptr<InputType> source;
while((source = ioWrapper->ReadInput()) != NULL) while((source = ioWrapper->ReadInput()) != NULL) {
{
// set up task of translating one sentence // set up task of translating one sentence
boost::shared_ptr<TranslationTask> ttask; boost::shared_ptr<TranslationTask> ttask;
ttask = TranslationTask::create(source, ioWrapper); ttask = TranslationTask::create(source, ioWrapper);
@ -187,17 +186,13 @@ int main(int argc, char* argv[])
TrellisPathList nBestList; TrellisPathList nBestList;
manager.CalcNBest(nBestSize, nBestList,true); manager.CalcNBest(nBestSize, nBestList,true);
//grid search //grid search
BOOST_FOREACH(float const& p, pgrid) BOOST_FOREACH(float const& p, pgrid) {
{
SD.SetLatticeMBRPrecision(p); SD.SetLatticeMBRPrecision(p);
BOOST_FOREACH(float const& r, rgrid) BOOST_FOREACH(float const& r, rgrid) {
{
SD.SetLatticeMBRPRatio(r); SD.SetLatticeMBRPRatio(r);
BOOST_FOREACH(size_t const prune_i, prune_grid) BOOST_FOREACH(size_t const prune_i, prune_grid) {
{
SD.SetLatticeMBRPruningFactor(size_t(prune_i)); SD.SetLatticeMBRPruningFactor(size_t(prune_i));
BOOST_FOREACH(float const& scale_i, scale_grid) BOOST_FOREACH(float const& scale_i, scale_grid) {
{
SD.SetMBRScale(scale_i); SD.SetMBRScale(scale_i);
size_t lineCount = source->GetTranslationId(); size_t lineCount = source->GetTranslationId();
cout << lineCount << " ||| " << p << " " cout << lineCount << " ||| " << p << " "

View File

@ -146,9 +146,10 @@ int main(int argc, char** argv)
// main loop over set of input sentences // main loop over set of input sentences
boost::shared_ptr<InputType> source; boost::shared_ptr<InputType> source;
while ((source = ioWrapper->ReadInput()) != NULL) while ((source = ioWrapper->ReadInput()) != NULL) {
{ IFVERBOSE(1) {
IFVERBOSE(1) { ResetUserTime(); } ResetUserTime();
}
InputType* foo = source.get(); InputType* foo = source.get();
FeatureFunction::CallChangeSource(foo); FeatureFunction::CallChangeSource(foo);

View File

@ -17,7 +17,9 @@ BaseManager::BaseManager(ttasksptr const& ttask)
const InputType& const InputType&
BaseManager::GetSource() const BaseManager::GetSource() const
{ return m_source; } {
return m_source;
}

View File

@ -18,16 +18,16 @@
namespace Moses namespace Moses
{ {
class ContextScope class ContextScope
{ {
protected: protected:
typedef std::map<void const*, boost::shared_ptr<void> > scratchpad_t; typedef std::map<void const*, boost::shared_ptr<void> > scratchpad_t;
typedef scratchpad_t::iterator iter_t; typedef scratchpad_t::iterator iter_t;
typedef scratchpad_t::value_type entry_t; typedef scratchpad_t::value_type entry_t;
typedef scratchpad_t::const_iterator const_iter_t; typedef scratchpad_t::const_iterator const_iter_t;
scratchpad_t m_scratchpad; scratchpad_t m_scratchpad;
mutable boost::shared_mutex m_lock; mutable boost::shared_mutex m_lock;
public: public:
// class write_access // class write_access
// { // {
// boost::unique_lock<boost::shared_mutex> m_lock; // boost::unique_lock<boost::shared_mutex> m_lock;
@ -50,26 +50,22 @@ namespace Moses
template<typename T> template<typename T>
boost::shared_ptr<void> const& boost::shared_ptr<void> const&
set(void const* const key, boost::shared_ptr<T> const& val) set(void const* const key, boost::shared_ptr<T> const& val) {
{
boost::unique_lock<boost::shared_mutex> lock(m_lock); boost::unique_lock<boost::shared_mutex> lock(m_lock);
return (m_scratchpad[key] = val); return (m_scratchpad[key] = val);
} }
template<typename T> template<typename T>
boost::shared_ptr<T> const boost::shared_ptr<T> const
get(void const* key, bool CreateNewIfNecessary=false) get(void const* key, bool CreateNewIfNecessary=false) {
{
using boost::shared_mutex; using boost::shared_mutex;
using boost::upgrade_lock; using boost::upgrade_lock;
// T const* key = reinterpret_cast<T const*>(xkey); // T const* key = reinterpret_cast<T const*>(xkey);
upgrade_lock<shared_mutex> lock(m_lock); upgrade_lock<shared_mutex> lock(m_lock);
iter_t m = m_scratchpad.find(key); iter_t m = m_scratchpad.find(key);
boost::shared_ptr< T > ret; boost::shared_ptr< T > ret;
if (m != m_scratchpad.end()) if (m != m_scratchpad.end()) {
{ if (m->second == NULL && CreateNewIfNecessary) {
if (m->second == NULL && CreateNewIfNecessary)
{
boost::upgrade_to_unique_lock<shared_mutex> xlock(lock); boost::upgrade_to_unique_lock<shared_mutex> xlock(lock);
m->second.reset(new T); m->second.reset(new T);
} }
@ -85,13 +81,12 @@ namespace Moses
ContextScope() { } ContextScope() { }
ContextScope(ContextScope const& other) ContextScope(ContextScope const& other) {
{
boost::unique_lock<boost::shared_mutex> lock1(this->m_lock); boost::unique_lock<boost::shared_mutex> lock1(this->m_lock);
boost::unique_lock<boost::shared_mutex> lock2(other.m_lock); boost::unique_lock<boost::shared_mutex> lock2(other.m_lock);
m_scratchpad = other.m_scratchpad; m_scratchpad = other.m_scratchpad;
} }
}; };
}; };

View File

@ -223,8 +223,7 @@ ProcessLEGACY(TranslationOption const& in,
TranslationOptionCollection *toc, TranslationOptionCollection *toc,
bool adhereTableLimit) const bool adhereTableLimit) const
{ {
if (in.GetTargetPhrase().GetSize() == 0) if (in.GetTargetPhrase().GetSize() == 0) {
{
// word deletion // word deletion
out.Add(new TranslationOption(in)); out.Add(new TranslationOption(in));
return; return;
@ -241,16 +240,14 @@ ProcessLEGACY(TranslationOption const& in,
TargetPhraseCollectionWithSourcePhrase const* phraseColl; TargetPhraseCollectionWithSourcePhrase const* phraseColl;
phraseColl = pdict->GetTargetPhraseCollectionLEGACY(toc->GetSource(),srcRange); phraseColl = pdict->GetTargetPhraseCollectionLEGACY(toc->GetSource(),srcRange);
if (phraseColl != NULL) if (phraseColl != NULL) {
{
TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd; TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
iterEnd = ((adhereTableLimit && tableLimit && phraseColl->GetSize() >= tableLimit) iterEnd = ((adhereTableLimit && tableLimit && phraseColl->GetSize() >= tableLimit)
? phraseColl->begin() + tableLimit : phraseColl->end()); ? phraseColl->begin() + tableLimit : phraseColl->end());
for (iterTargetPhrase = phraseColl->begin(); for (iterTargetPhrase = phraseColl->begin();
iterTargetPhrase != iterEnd; iterTargetPhrase != iterEnd;
++iterTargetPhrase) ++iterTargetPhrase) {
{
TargetPhrase const& targetPhrase = **iterTargetPhrase; TargetPhrase const& targetPhrase = **iterTargetPhrase;
if (targetPhrase.GetSize() != currSize || if (targetPhrase.GetSize() != currSize ||
(IsFilteringStep() && !in.IsCompatible(targetPhrase, m_conflictFactors))) (IsFilteringStep() && !in.IsCompatible(targetPhrase, m_conflictFactors)))

View File

@ -114,7 +114,9 @@ string SimpleTranslationInterface::translate(const string &inputString)
boost::shared_ptr<InputType> source = ioWrapper->ReadInput(); boost::shared_ptr<InputType> source = ioWrapper->ReadInput();
if (!source) return "Error: Source==null!!!"; if (!source) return "Error: Source==null!!!";
IFVERBOSE(1) { ResetUserTime(); } IFVERBOSE(1) {
ResetUserTime();
}
FeatureFunction::CallChangeSource(&*source); FeatureFunction::CallChangeSource(&*source);
@ -147,10 +149,14 @@ int
run_as_server() run_as_server()
{ {
#ifdef HAVE_XMLRPC_C #ifdef HAVE_XMLRPC_C
int port; params.SetParameter(port, "server-port", 8080); int port;
bool isSerial; params.SetParameter(isSerial, "serial", false); params.SetParameter(port, "server-port", 8080);
string logfile; params.SetParameter(logfile, "server-log", string("")); bool isSerial;
size_t num_threads; params.SetParameter(num_threads, "threads", size_t(10)); params.SetParameter(isSerial, "serial", false);
string logfile;
params.SetParameter(logfile, "server-log", string(""));
size_t num_threads;
params.SetParameter(num_threads, "threads", size_t(10));
if (isSerial) VERBOSE(1,"Running server in serial mode." << endl); if (isSerial) VERBOSE(1,"Running server in serial mode." << endl);
xmlrpc_c::registry myRegistry; xmlrpc_c::registry myRegistry;
@ -166,8 +172,9 @@ run_as_server()
xmlrpc_c::serverAbyss myAbyssServer(myRegistry, port, logfile); xmlrpc_c::serverAbyss myAbyssServer(myRegistry, port, logfile);
XVERBOSE(1,"Listening on port " << port << endl); XVERBOSE(1,"Listening on port " << port << endl);
if (isSerial) { while(1) myAbyssServer.runOnce(); } if (isSerial) {
else myAbyssServer.run(); while(1) myAbyssServer.runOnce();
} else myAbyssServer.run();
std::cerr << "xmlrpc_c::serverAbyss.run() returned but should not." << std::endl; std::cerr << "xmlrpc_c::serverAbyss.run() returned but should not." << std::endl;
// #pragma message("BUILDING MOSES WITH SERVER SUPPORT") // #pragma message("BUILDING MOSES WITH SERVER SUPPORT")
@ -197,8 +204,7 @@ batch_run()
// check on weights // check on weights
const ScoreComponentCollection& weights = staticData.GetAllWeights(); const ScoreComponentCollection& weights = staticData.GetAllWeights();
IFVERBOSE(2) IFVERBOSE(2) {
{
TRACE_ERR("The global weight vector looks like this: "); TRACE_ERR("The global weight vector looks like this: ");
TRACE_ERR(weights); TRACE_ERR(weights);
TRACE_ERR("\n"); TRACE_ERR("\n");
@ -214,8 +220,7 @@ batch_run()
// main loop over set of input sentences // main loop over set of input sentences
boost::shared_ptr<InputType> source; boost::shared_ptr<InputType> source;
while ((source = ioWrapper->ReadInput()) != NULL) while ((source = ioWrapper->ReadInput()) != NULL) {
{
IFVERBOSE(1) ResetUserTime(); IFVERBOSE(1) ResetUserTime();
FeatureFunction::CallChangeSource(source.get()); FeatureFunction::CallChangeSource(source.get());
@ -236,8 +241,7 @@ batch_run()
// simulated post-editing requires threads (within the dynamic phrase tables) // simulated post-editing requires threads (within the dynamic phrase tables)
// but runs all sentences serially, to allow updating of the bitext. // but runs all sentences serially, to allow updating of the bitext.
bool spe = params.isParamSpecified("spe-src"); bool spe = params.isParamSpecified("spe-src");
if (spe) if (spe) {
{
// simulated post-editing: always run single-threaded! // simulated post-editing: always run single-threaded!
task->Run(); task->Run();
string src,trg,aln; string src,trg,aln;
@ -247,16 +251,14 @@ batch_run()
<< "missing update data for simulated post-editing."); << "missing update data for simulated post-editing.");
UTIL_THROW_IF2(!getline(*ioWrapper->spe_aln,aln), "[" << HERE << "] " UTIL_THROW_IF2(!getline(*ioWrapper->spe_aln,aln), "[" << HERE << "] "
<< "missing update data for simulated post-editing."); << "missing update data for simulated post-editing.");
BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl()) BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl()) {
{
Mmsapt* sapt = dynamic_cast<Mmsapt*>(pd); Mmsapt* sapt = dynamic_cast<Mmsapt*>(pd);
if (sapt) sapt->add(src,trg,aln); if (sapt) sapt->add(src,trg,aln);
VERBOSE(1,"[" << HERE << " added src] " << src << endl); VERBOSE(1,"[" << HERE << " added src] " << src << endl);
VERBOSE(1,"[" << HERE << " added trg] " << trg << endl); VERBOSE(1,"[" << HERE << " added trg] " << trg << endl);
VERBOSE(1,"[" << HERE << " added aln] " << aln << endl); VERBOSE(1,"[" << HERE << " added aln] " << aln << endl);
} }
} } else pool.Submit(task);
else pool.Submit(task);
#else #else
pool.Submit(task); pool.Submit(task);
@ -295,8 +297,7 @@ int decoder_main(int argc, char** argv)
#endif #endif
// echo command line, if verbose // echo command line, if verbose
IFVERBOSE(1) IFVERBOSE(1) {
{
TRACE_ERR("command: "); TRACE_ERR("command: ");
for(int i=0; i<argc; ++i) TRACE_ERR(argv[i]<<" "); for(int i=0; i<argc; ++i) TRACE_ERR(argv[i]<<" ");
TRACE_ERR(endl); TRACE_ERR(endl);
@ -317,8 +318,7 @@ int decoder_main(int argc, char** argv)
exit(1); exit(1);
// setting "-show-weights" -> just dump out weights and exit // setting "-show-weights" -> just dump out weights and exit
if (params.isParamSpecified("show-weights")) if (params.isParamSpecified("show-weights")) {
{
ShowWeights(); ShowWeights();
exit(0); exit(0);
} }
@ -330,8 +330,7 @@ int decoder_main(int argc, char** argv)
} }
#ifdef NDEBUG #ifdef NDEBUG
catch (const std::exception &e) catch (const std::exception &e) {
{
std::cerr << "Exception: " << e.what() << std::endl; std::cerr << "Exception: " << e.what() << std::endl;
return EXIT_FAILURE; return EXIT_FAILURE;
} }

View File

@ -45,7 +45,9 @@ public:
~SimpleTranslationInterface(); ~SimpleTranslationInterface();
std::string translate(const std::string &input); std::string translate(const std::string &input);
Moses::StaticData& getStaticData(); Moses::StaticData& getStaticData();
Moses::Parameter& getParameters(){ return m_params; } Moses::Parameter& getParameters() {
return m_params;
}
private: private:
SimpleTranslationInterface(); SimpleTranslationInterface();
Moses::Parameter m_params; Moses::Parameter m_params;

View File

@ -157,20 +157,15 @@ FeatureFactory
std::vector<float> weights = static_data.GetParameter()->GetWeights(featureName); std::vector<float> weights = static_data.GetParameter()->GetWeights(featureName);
if (feature->GetNumScoreComponents()) if (feature->GetNumScoreComponents()) {
{ if (weights.size() == 0) {
if (weights.size() == 0)
{
weights = feature->DefaultWeights(); weights = feature->DefaultWeights();
if (weights.size() == 0) if (weights.size() == 0) {
{
TRACE_ERR("WARNING: No weights specified in config file for FF " TRACE_ERR("WARNING: No weights specified in config file for FF "
<< featureName << ". This FF does not supply default values.\n" << featureName << ". This FF does not supply default values.\n"
<< "WARNING: Auto-initializing all weights for this FF to 1.0"); << "WARNING: Auto-initializing all weights for this FF to 1.0");
weights.assign(feature->GetNumScoreComponents(),1.0); weights.assign(feature->GetNumScoreComponents(),1.0);
} } else {
else
{
TRACE_ERR("WARNING: No weights specified in config file for FF " TRACE_ERR("WARNING: No weights specified in config file for FF "
<< featureName << ". Using default values supplied by FF."); << featureName << ". Using default values supplied by FF.");
} }
@ -181,8 +176,7 @@ FeatureFactory
<< " (features: " << feature->GetNumScoreComponents() << " (features: " << feature->GetNumScoreComponents()
<< " vs. weights: " << weights.size() << ")"); << " vs. weights: " << weights.size() << ")");
static_data.SetWeights(feature, weights); static_data.SetWeights(feature, weights);
} } else if (feature->IsTuneable())
else if (feature->IsTuneable())
static_data.SetWeights(feature, weights); static_data.SetWeights(feature, weights);
} }

View File

@ -193,17 +193,23 @@ void FeatureFunction::SetTuneableComponents(const std::string& value)
void void
FeatureFunction FeatureFunction
::InitializeForInput(ttasksptr const& ttask) ::InitializeForInput(ttasksptr const& ttask)
{ InitializeForInput(*(ttask->GetSource().get())); } {
InitializeForInput(*(ttask->GetSource().get()));
}
void void
FeatureFunction FeatureFunction
::CleanUpAfterSentenceProcessing(ttasksptr const& ttask) ::CleanUpAfterSentenceProcessing(ttasksptr const& ttask)
{ CleanUpAfterSentenceProcessing(*(ttask->GetSource().get())); } {
CleanUpAfterSentenceProcessing(*(ttask->GetSource().get()));
}
size_t size_t
FeatureFunction FeatureFunction
::GetIndex() const ::GetIndex() const
{ return m_index; } {
return m_index;
}
/// set index /// set index

View File

@ -136,7 +136,9 @@ public:
CleanUpAfterSentenceProcessing(ttasksptr const& ttask); CleanUpAfterSentenceProcessing(ttasksptr const& ttask);
const std::string & const std::string &
GetArgLine() const { return m_argLine; } GetArgLine() const {
return m_argLine;
}
// given a target phrase containing only factors specified in mask // given a target phrase containing only factors specified in mask
// return true if the feature function can be evaluated // return true if the feature function can be evaluated

View File

@ -147,8 +147,7 @@ void InternalTree::GetUnbinarizedChildren(std::vector<TreePointer> &ret) const
const std::string &label = (*itx)->GetLabel(); const std::string &label = (*itx)->GetLabel();
if (!label.empty() && label[0] == '^') { if (!label.empty() && label[0] == '^') {
(*itx)->GetUnbinarizedChildren(ret); (*itx)->GetUnbinarizedChildren(ret);
} } else {
else {
ret.push_back(*itx); ret.push_back(*itx);
} }
} }

View File

@ -96,8 +96,7 @@ public:
bool RecursiveSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const; bool RecursiveSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const;
// Python-like generator that yields next nonterminal leaf on every call // Python-like generator that yields next nonterminal leaf on every call
$generator(leafNT) $generator(leafNT) {
{
std::vector<TreePointer>::iterator it; std::vector<TreePointer>::iterator it;
InternalTree* tree; InternalTree* tree;
leafNT(InternalTree* root = 0): tree(root) {} leafNT(InternalTree* root = 0): tree(root) {}
@ -116,8 +115,7 @@ public:
// Python-like generator that yields the parent of the next nonterminal leaf on every call // Python-like generator that yields the parent of the next nonterminal leaf on every call
$generator(leafNTParent) $generator(leafNTParent) {
{
std::vector<TreePointer>::iterator it; std::vector<TreePointer>::iterator it;
InternalTree* tree; InternalTree* tree;
leafNTParent(InternalTree* root = 0): tree(root) {} leafNTParent(InternalTree* root = 0): tree(root) {}
@ -135,8 +133,7 @@ public:
}; };
// Python-like generator that yields the next nonterminal leaf on every call, and also stores the path from the root of the tree to the nonterminal // Python-like generator that yields the next nonterminal leaf on every call, and also stores the path from the root of the tree to the nonterminal
$generator(leafNTPath) $generator(leafNTPath) {
{
std::vector<TreePointer>::iterator it; std::vector<TreePointer>::iterator it;
InternalTree* tree; InternalTree* tree;
std::vector<InternalTree*> * path; std::vector<InternalTree*> * path;

View File

@ -44,8 +44,7 @@ public:
static const ReorderingType L = 1; // left static const ReorderingType L = 1; // left
static const ReorderingType MAX = 3; // largest possible static const ReorderingType MAX = 3; // largest possible
#else #else
enum ReorderingType enum ReorderingType {
{
M = 0, // monotonic M = 0, // monotonic
NM = 1, // non-monotonic NM = 1, // non-monotonic
S = 1, // swap S = 1, // swap

View File

@ -71,21 +71,18 @@ void Model1Vocabulary::Load(const std::string& fileName)
std::string line; std::string line;
unsigned i = 0; unsigned i = 0;
if ( getline(inFile, line) ) // first line of MGIZA vocabulary files seems to be special : "1 UNK 0" -- skip if it's this if ( getline(inFile, line) ) { // first line of MGIZA vocabulary files seems to be special : "1 UNK 0" -- skip if it's this
{
++i; ++i;
std::vector<std::string> tokens = Tokenize(line); std::vector<std::string> tokens = Tokenize(line);
UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens."); UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens.");
unsigned id = Scan<unsigned>(tokens[0]); unsigned id = Scan<unsigned>(tokens[0]);
if (! ( (id == 1) && (tokens[1] == "UNK") )) if (! ( (id == 1) && (tokens[1] == "UNK") )) {
{
const Factor* factor = factorCollection.AddFactor(tokens[1],false); // TODO: can we assume that the vocabulary is know and filter the model on loading? const Factor* factor = factorCollection.AddFactor(tokens[1],false); // TODO: can we assume that the vocabulary is know and filter the model on loading?
bool stored = Store(factor, id); bool stored = Store(factor, id);
UTIL_THROW_IF2(!stored, "Line " << i << " in " << fileName << " overwrites existing vocabulary entry."); UTIL_THROW_IF2(!stored, "Line " << i << " in " << fileName << " overwrites existing vocabulary entry.");
} }
} }
while ( getline(inFile, line) ) while ( getline(inFile, line) ) {
{
++i; ++i;
std::vector<std::string> tokens = Tokenize(line); std::vector<std::string> tokens = Tokenize(line);
UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens."); UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens.");
@ -104,8 +101,7 @@ void Model1LexicalTable::Load(const std::string &fileName, const Model1Vocabular
std::string line; std::string line;
unsigned i = 0; unsigned i = 0;
while ( getline(inFile, line) ) while ( getline(inFile, line) ) {
{
++i; ++i;
std::vector<std::string> tokens = Tokenize(line); std::vector<std::string> tokens = Tokenize(line);
UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens."); UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens.");
@ -193,25 +189,21 @@ void Model1Feature::EvaluateWithSourceContext(const InputType &input
float score = 0.0; float score = 0.0;
float norm = TransformScore(1+sentence.GetSize()); float norm = TransformScore(1+sentence.GetSize());
for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) {
{
const Word &wordT = targetPhrase.GetWord(posT); const Word &wordT = targetPhrase.GetWord(posT);
if ( !wordT.IsNonTerminal() ) if ( !wordT.IsNonTerminal() ) {
{
float thisWordProb = m_model1.GetProbability(m_emptyWord,wordT[0]); // probability conditioned on empty word float thisWordProb = m_model1.GetProbability(m_emptyWord,wordT[0]); // probability conditioned on empty word
// cache lookup // cache lookup
bool foundInCache = false; bool foundInCache = false;
{ {
#ifdef WITH_THREADS #ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock); boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
#endif #endif
boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> >::const_iterator sentenceCache = m_cache.find(&input); boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> >::const_iterator sentenceCache = m_cache.find(&input);
if (sentenceCache != m_cache.end()) if (sentenceCache != m_cache.end()) {
{
boost::unordered_map<const Factor*, float>::const_iterator cacheHit = sentenceCache->second.find(wordT[0]); boost::unordered_map<const Factor*, float>::const_iterator cacheHit = sentenceCache->second.find(wordT[0]);
if (cacheHit != sentenceCache->second.end()) if (cacheHit != sentenceCache->second.end()) {
{
foundInCache = true; foundInCache = true;
score += cacheHit->second; score += cacheHit->second;
FEATUREVERBOSE(3, "Cached score( " << wordT << " ) = " << cacheHit->second << std::endl); FEATUREVERBOSE(3, "Cached score( " << wordT << " ) = " << cacheHit->second << std::endl);
@ -219,10 +211,8 @@ void Model1Feature::EvaluateWithSourceContext(const InputType &input
} }
} }
if (!foundInCache) if (!foundInCache) {
{ for (size_t posS=1; posS<sentence.GetSize()-1; ++posS) { // ignore <s> and </s>
for (size_t posS=1; posS<sentence.GetSize()-1; ++posS) // ignore <s> and </s>
{
const Word &wordS = sentence.GetWord(posS); const Word &wordS = sentence.GetWord(posS);
float modelProb = m_model1.GetProbability(wordS[0],wordT[0]); float modelProb = m_model1.GetProbability(wordS[0],wordT[0]);
FEATUREVERBOSE(4, "p( " << wordT << " | " << wordS << " ) = " << modelProb << std::endl); FEATUREVERBOSE(4, "p( " << wordT << " | " << wordS << " ) = " << modelProb << std::endl);
@ -231,10 +221,10 @@ void Model1Feature::EvaluateWithSourceContext(const InputType &input
float thisWordScore = TransformScore(thisWordProb) - norm; float thisWordScore = TransformScore(thisWordProb) - norm;
FEATUREVERBOSE(3, "score( " << wordT << " ) = " << thisWordScore << std::endl); FEATUREVERBOSE(3, "score( " << wordT << " ) = " << thisWordScore << std::endl);
{ {
#ifdef WITH_THREADS #ifdef WITH_THREADS
// need to update cache; write lock // need to update cache; write lock
boost::unique_lock<boost::shared_mutex> lock(m_accessLock); boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
#endif #endif
m_cache[&input][wordT[0]] = thisWordScore; m_cache[&input][wordT[0]] = thisWordScore;
} }
score += thisWordScore; score += thisWordScore;
@ -247,14 +237,13 @@ void Model1Feature::EvaluateWithSourceContext(const InputType &input
void Model1Feature::CleanUpAfterSentenceProcessing(const InputType& source) void Model1Feature::CleanUpAfterSentenceProcessing(const InputType& source)
{ {
#ifdef WITH_THREADS #ifdef WITH_THREADS
// need to update cache; write lock // need to update cache; write lock
boost::unique_lock<boost::shared_mutex> lock(m_accessLock); boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
#endif #endif
// clear cache // clear cache
boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> >::iterator sentenceCache = m_cache.find(&source); boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> >::iterator sentenceCache = m_cache.find(&source);
if (sentenceCache != m_cache.end()) if (sentenceCache != m_cache.end()) {
{
sentenceCache->second.clear(); sentenceCache->second.clear();
m_cache.erase(sentenceCache); m_cache.erase(sentenceCache);
} }

View File

@ -17,7 +17,7 @@ class Model1Vocabulary
{ {
public: public:
#define INVALID_ID std::numeric_limits<unsigned>::max() // UINT_MAX #define INVALID_ID std::numeric_limits<unsigned>::max() // UINT_MAX
static const std::string GIZANULL; static const std::string GIZANULL;
Model1Vocabulary(); Model1Vocabulary();
@ -103,10 +103,10 @@ private:
// cache // cache
mutable boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> > m_cache; mutable boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> > m_cache;
#ifdef WITH_THREADS #ifdef WITH_THREADS
// reader-writer lock // reader-writer lock
mutable boost::shared_mutex m_accessLock; mutable boost::shared_mutex m_accessLock;
#endif #endif
}; };

View File

@ -51,8 +51,7 @@ void RulePairUnlexicalizedSource::EvaluateInIsolation(const Phrase &source
return; return;
} }
for (size_t posS=0; posS<source.GetSize(); ++posS) for (size_t posS=0; posS<source.GetSize(); ++posS) {
{
const Word &wordS = source.GetWord(posS); const Word &wordS = source.GetWord(posS);
if ( !wordS.IsNonTerminal() ) { if ( !wordS.IsNonTerminal() ) {
return; return;
@ -61,8 +60,7 @@ void RulePairUnlexicalizedSource::EvaluateInIsolation(const Phrase &source
ostringstream namestr; ostringstream namestr;
for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) {
{
const Word &wordT = targetPhrase.GetWord(posT); const Word &wordT = targetPhrase.GetWord(posT);
const Factor* factorT = wordT[0]; const Factor* factorT = wordT[0];
if ( wordT.IsNonTerminal() ) { if ( wordT.IsNonTerminal() ) {
@ -78,8 +76,7 @@ void RulePairUnlexicalizedSource::EvaluateInIsolation(const Phrase &source
namestr << targetPhraseLHS->GetString() << "|"; namestr << targetPhraseLHS->GetString() << "|";
for (AlignmentInfo::const_iterator it=targetPhrase.GetAlignNonTerm().begin(); for (AlignmentInfo::const_iterator it=targetPhrase.GetAlignNonTerm().begin();
it!=targetPhrase.GetAlignNonTerm().end(); ++it) it!=targetPhrase.GetAlignNonTerm().end(); ++it) {
{
namestr << "|" << it->first << "-" << it->second; namestr << "|" << it->first << "-" << it->second;
} }

View File

@ -68,15 +68,12 @@ void RuleScope::EvaluateInIsolation(const Phrase &source
if (m_futureCostOnly) { if (m_futureCostOnly) {
estimatedFutureScore.PlusEquals(this, scores); estimatedFutureScore.PlusEquals(this, scores);
} } else {
else {
scoreBreakdown.PlusEquals(this, scores); scoreBreakdown.PlusEquals(this, scores);
} }
} } else if (m_futureCostOnly) {
else if (m_futureCostOnly) {
estimatedFutureScore.PlusEquals(this, score); estimatedFutureScore.PlusEquals(this, score);
} } else {
else {
scoreBreakdown.PlusEquals(this, score); scoreBreakdown.PlusEquals(this, score);
} }
} }
@ -85,14 +82,11 @@ void RuleScope::SetParameter(const std::string& key, const std::string& value)
{ {
if (key == "source-syntax") { if (key == "source-syntax") {
m_sourceSyntax = Scan<bool>(value); m_sourceSyntax = Scan<bool>(value);
} } else if (key == "per-scope") {
else if (key == "per-scope") {
m_perScope = Scan<bool>(value); m_perScope = Scan<bool>(value);
} } else if ("future-cost-only") {
else if ("future-cost-only") {
m_futureCostOnly = Scan<bool>(value); m_futureCostOnly = Scan<bool>(value);
} } else {
else {
StatelessFeatureFunction::SetParameter(key, value); StatelessFeatureFunction::SetParameter(key, value);
} }
} }

View File

@ -45,11 +45,11 @@ namespace Moses
{ {
#ifdef USE_HYPO_POOL #ifdef USE_HYPO_POOL
ObjectPool<Hypothesis> Hypothesis::s_objectPool("Hypothesis", 300000); ObjectPool<Hypothesis> Hypothesis::s_objectPool("Hypothesis", 300000);
#endif #endif
Hypothesis:: Hypothesis::
Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt) Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt)
: m_prevHypo(NULL) : m_prevHypo(NULL)
, m_sourceCompleted(source.GetSize(), manager.GetSource().m_sourceCompleted) , m_sourceCompleted(source.GetSize(), manager.GetSource().m_sourceCompleted)
, m_sourceInput(source) , m_sourceInput(source)
@ -65,7 +65,7 @@ namespace Moses
, m_transOpt(initialTransOpt) , m_transOpt(initialTransOpt)
, m_manager(manager) , m_manager(manager)
, m_id(m_manager.GetNextHypoId()) , m_id(m_manager.GetNextHypoId())
{ {
// used for initial seeding of trans process // used for initial seeding of trans process
// initialize scores // initialize scores
//_hash_computed = false; //_hash_computed = false;
@ -74,13 +74,13 @@ namespace Moses
for (unsigned i = 0; i < ffs.size(); ++i) for (unsigned i = 0; i < ffs.size(); ++i)
m_ffStates[i] = ffs[i]->EmptyHypothesisState(source); m_ffStates[i] = ffs[i]->EmptyHypothesisState(source);
m_manager.GetSentenceStats().AddCreated(); m_manager.GetSentenceStats().AddCreated();
} }
/*** /***
* continue prevHypo by appending the phrases in transOpt * continue prevHypo by appending the phrases in transOpt
*/ */
Hypothesis:: Hypothesis::
Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt) Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt)
: m_prevHypo(&prevHypo) : m_prevHypo(&prevHypo)
, m_sourceCompleted(prevHypo.m_sourceCompleted ) , m_sourceCompleted(prevHypo.m_sourceCompleted )
, m_sourceInput(prevHypo.m_sourceInput) , m_sourceInput(prevHypo.m_sourceInput)
@ -96,7 +96,7 @@ namespace Moses
, m_transOpt(transOpt) , m_transOpt(transOpt)
, m_manager(prevHypo.GetManager()) , m_manager(prevHypo.GetManager())
, m_id(m_manager.GetNextHypoId()) , m_id(m_manager.GetNextHypoId())
{ {
m_currScoreBreakdown.PlusEquals(transOpt.GetScoreBreakdown()); m_currScoreBreakdown.PlusEquals(transOpt.GetScoreBreakdown());
// assert that we are not extending our hypothesis by retranslating something // assert that we are not extending our hypothesis by retranslating something
@ -107,11 +107,11 @@ namespace Moses
m_sourceCompleted.SetValue(m_currSourceWordsRange.GetStartPos(), m_currSourceWordsRange.GetEndPos(), true); m_sourceCompleted.SetValue(m_currSourceWordsRange.GetStartPos(), m_currSourceWordsRange.GetEndPos(), true);
m_wordDeleted = transOpt.IsDeletionOption(); m_wordDeleted = transOpt.IsDeletionOption();
m_manager.GetSentenceStats().AddCreated(); m_manager.GetSentenceStats().AddCreated();
} }
Hypothesis:: Hypothesis::
~Hypothesis() ~Hypothesis()
{ {
for (unsigned i = 0; i < m_ffStates.size(); ++i) for (unsigned i = 0; i < m_ffStates.size(); ++i)
delete m_ffStates[i]; delete m_ffStates[i];
@ -125,12 +125,12 @@ namespace Moses
delete m_arcList; delete m_arcList;
m_arcList = NULL; m_arcList = NULL;
} }
} }
void void
Hypothesis:: Hypothesis::
AddArc(Hypothesis *loserHypo) AddArc(Hypothesis *loserHypo)
{ {
if (!m_arcList) { if (!m_arcList) {
if (loserHypo->m_arcList) { // we don't have an arcList, but loser does if (loserHypo->m_arcList) { // we don't have an arcList, but loser does
this->m_arcList = loserHypo->m_arcList; // take ownership, we'll delete this->m_arcList = loserHypo->m_arcList; // take ownership, we'll delete
@ -151,25 +151,25 @@ namespace Moses
} }
} }
m_arcList->push_back(loserHypo); m_arcList->push_back(loserHypo);
} }
/*** /***
* return the subclass of Hypothesis most appropriate to the given translation option * return the subclass of Hypothesis most appropriate to the given translation option
*/ */
Hypothesis* Hypothesis*
Hypothesis:: Hypothesis::
CreateNext(const TranslationOption &transOpt) const CreateNext(const TranslationOption &transOpt) const
{ {
return Create(*this, transOpt); return Create(*this, transOpt);
} }
/*** /***
* return the subclass of Hypothesis most appropriate to the given translation option * return the subclass of Hypothesis most appropriate to the given translation option
*/ */
Hypothesis* Hypothesis*
Hypothesis:: Hypothesis::
Create(const Hypothesis &prevHypo, const TranslationOption &transOpt) Create(const Hypothesis &prevHypo, const TranslationOption &transOpt)
{ {
#ifdef USE_HYPO_POOL #ifdef USE_HYPO_POOL
Hypothesis *ptr = s_objectPool.getPtr(); Hypothesis *ptr = s_objectPool.getPtr();
@ -177,33 +177,33 @@ namespace Moses
#else #else
return new Hypothesis(prevHypo, transOpt); return new Hypothesis(prevHypo, transOpt);
#endif #endif
} }
/*** /***
* return the subclass of Hypothesis most appropriate to the given target phrase * return the subclass of Hypothesis most appropriate to the given target phrase
*/ */
Hypothesis* Hypothesis*
Hypothesis:: Hypothesis::
Create(Manager& manager, InputType const& m_source, Create(Manager& manager, InputType const& m_source,
const TranslationOption &initialTransOpt) const TranslationOption &initialTransOpt)
{ {
#ifdef USE_HYPO_POOL #ifdef USE_HYPO_POOL
Hypothesis *ptr = s_objectPool.getPtr(); Hypothesis *ptr = s_objectPool.getPtr();
return new(ptr) Hypothesis(manager, m_source, initialTransOpt); return new(ptr) Hypothesis(manager, m_source, initialTransOpt);
#else #else
return new Hypothesis(manager, m_source, initialTransOpt); return new Hypothesis(manager, m_source, initialTransOpt);
#endif #endif
} }
/** check, if two hypothesis can be recombined. /** check, if two hypothesis can be recombined.
this is actually a sorting function that allows us to this is actually a sorting function that allows us to
keep an ordered list of hypotheses. This makes recombination keep an ordered list of hypotheses. This makes recombination
much quicker. much quicker.
*/ */
int int
Hypothesis:: Hypothesis::
RecombineCompare(const Hypothesis &compare) const RecombineCompare(const Hypothesis &compare) const
{ {
// -1 = this < compare // -1 = this < compare
// +1 = this > compare // +1 = this > compare
// 0 = this ==compare // 0 = this ==compare
@ -221,40 +221,39 @@ namespace Moses
} }
return 0; return 0;
} }
void void
Hypothesis:: Hypothesis::
EvaluateWhenApplied(StatefulFeatureFunction const& sfff, EvaluateWhenApplied(StatefulFeatureFunction const& sfff,
int state_idx) int state_idx)
{ {
const StaticData &staticData = StaticData::Instance(); const StaticData &staticData = StaticData::Instance();
if (! staticData.IsFeatureFunctionIgnored( sfff )) if (! staticData.IsFeatureFunctionIgnored( sfff )) {
{
m_ffStates[state_idx] m_ffStates[state_idx]
= sfff.EvaluateWhenApplied = sfff.EvaluateWhenApplied
(*this, m_prevHypo ? m_prevHypo->m_ffStates[state_idx] : NULL, (*this, m_prevHypo ? m_prevHypo->m_ffStates[state_idx] : NULL,
&m_currScoreBreakdown); &m_currScoreBreakdown);
} }
} }
void void
Hypothesis:: Hypothesis::
EvaluateWhenApplied(const StatelessFeatureFunction& slff) EvaluateWhenApplied(const StatelessFeatureFunction& slff)
{ {
const StaticData &staticData = StaticData::Instance(); const StaticData &staticData = StaticData::Instance();
if (! staticData.IsFeatureFunctionIgnored( slff )) { if (! staticData.IsFeatureFunctionIgnored( slff )) {
slff.EvaluateWhenApplied(*this, &m_currScoreBreakdown); slff.EvaluateWhenApplied(*this, &m_currScoreBreakdown);
} }
} }
/*** /***
* calculate the logarithm of our total translation score (sum up components) * calculate the logarithm of our total translation score (sum up components)
*/ */
void void
Hypothesis:: Hypothesis::
EvaluateWhenApplied(const SquareMatrix &futureScore) EvaluateWhenApplied(const SquareMatrix &futureScore)
{ {
IFVERBOSE(2) { IFVERBOSE(2) {
m_manager.GetSentenceStats().StartTimeOtherScore(); m_manager.GetSentenceStats().StartTimeOtherScore();
} }
@ -299,20 +298,20 @@ namespace Moses
IFVERBOSE(2) { IFVERBOSE(2) {
m_manager.GetSentenceStats().StopTimeEstimateScore(); m_manager.GetSentenceStats().StopTimeEstimateScore();
} }
} }
const Hypothesis* Hypothesis::GetPrevHypo()const const Hypothesis* Hypothesis::GetPrevHypo()const
{ {
return m_prevHypo; return m_prevHypo;
} }
/** /**
* print hypothesis information for pharaoh-style logging * print hypothesis information for pharaoh-style logging
*/ */
void void
Hypothesis:: Hypothesis::
PrintHypothesis() const PrintHypothesis() const
{ {
if (!m_prevHypo) { if (!m_prevHypo) {
TRACE_ERR(endl << "NULL hypo" << endl); TRACE_ERR(endl << "NULL hypo" << endl);
return; return;
@ -344,12 +343,12 @@ namespace Moses
TRACE_ERR( "\tscore "<<m_totalScore - m_futureScore<<" + future cost "<<m_futureScore<<" = "<<m_totalScore<<endl); TRACE_ERR( "\tscore "<<m_totalScore - m_futureScore<<" + future cost "<<m_futureScore<<" = "<<m_totalScore<<endl);
TRACE_ERR( "\tunweighted feature scores: " << m_currScoreBreakdown << endl); TRACE_ERR( "\tunweighted feature scores: " << m_currScoreBreakdown << endl);
//PrintLMScores(); //PrintLMScores();
} }
void void
Hypothesis:: Hypothesis::
CleanupArcList() CleanupArcList()
{ {
// point this hypo's main hypo to itself // point this hypo's main hypo to itself
SetWinningHypo(this); SetWinningHypo(this);
@ -369,8 +368,7 @@ namespace Moses
staticData.GetOutputSearchGraphHypergraph() || staticData.GetOutputSearchGraphHypergraph() ||
staticData.UseLatticeMBR()); staticData.UseLatticeMBR());
if (!distinctNBest && m_arcList->size() > nBestSize * 5) if (!distinctNBest && m_arcList->size() > nBestSize * 5) {
{
// prune arc list only if there too many arcs // prune arc list only if there too many arcs
NTH_ELEMENT4(m_arcList->begin(), m_arcList->begin() + nBestSize - 1, NTH_ELEMENT4(m_arcList->begin(), m_arcList->begin() + nBestSize - 1,
m_arcList->end(), CompareHypothesisTotalScore()); m_arcList->end(), CompareHypothesisTotalScore());
@ -388,27 +386,29 @@ namespace Moses
Hypothesis *arc = *iter; Hypothesis *arc = *iter;
arc->SetWinningHypo(this); arc->SetWinningHypo(this);
} }
} }
TargetPhrase const& TargetPhrase const&
Hypothesis:: Hypothesis::
GetCurrTargetPhrase() const GetCurrTargetPhrase() const
{ return m_transOpt.GetTargetPhrase(); } {
return m_transOpt.GetTargetPhrase();
}
void void
Hypothesis:: Hypothesis::
GetOutputPhrase(Phrase &out) const GetOutputPhrase(Phrase &out) const
{ {
if (m_prevHypo != NULL) if (m_prevHypo != NULL)
m_prevHypo->GetOutputPhrase(out); m_prevHypo->GetOutputPhrase(out);
out.Append(GetCurrTargetPhrase()); out.Append(GetCurrTargetPhrase());
} }
TO_STRING_BODY(Hypothesis) TO_STRING_BODY(Hypothesis)
// friend // friend
ostream& operator<<(ostream& out, const Hypothesis& hypo) ostream& operator<<(ostream& out, const Hypothesis& hypo)
{ {
hypo.ToStream(out); hypo.ToStream(out);
// words bitmap // words bitmap
out << "[" << hypo.m_sourceCompleted << "] "; out << "[" << hypo.m_sourceCompleted << "] ";
@ -421,43 +421,47 @@ namespace Moses
out << " " << hypo.GetCurrTargetPhrase().GetAlignNonTerm(); out << " " << hypo.GetCurrTargetPhrase().GetAlignNonTerm();
return out; return out;
} }
std::string std::string
Hypothesis:: Hypothesis::
GetSourcePhraseStringRep(const vector<FactorType> factorsToPrint) const GetSourcePhraseStringRep(const vector<FactorType> factorsToPrint) const
{ return m_transOpt.GetInputPath().GetPhrase().GetStringRep(factorsToPrint); } {
return m_transOpt.GetInputPath().GetPhrase().GetStringRep(factorsToPrint);
}
std::string std::string
Hypothesis:: Hypothesis::
GetTargetPhraseStringRep(const vector<FactorType> factorsToPrint) const GetTargetPhraseStringRep(const vector<FactorType> factorsToPrint) const
{ return (m_prevHypo ? GetCurrTargetPhrase().GetStringRep(factorsToPrint) : ""); } {
return (m_prevHypo ? GetCurrTargetPhrase().GetStringRep(factorsToPrint) : "");
}
std::string std::string
Hypothesis:: Hypothesis::
GetSourcePhraseStringRep() const GetSourcePhraseStringRep() const
{ {
vector<FactorType> allFactors(MAX_NUM_FACTORS); vector<FactorType> allFactors(MAX_NUM_FACTORS);
for(size_t i=0; i < MAX_NUM_FACTORS; i++) for(size_t i=0; i < MAX_NUM_FACTORS; i++)
allFactors[i] = i; allFactors[i] = i;
return GetSourcePhraseStringRep(allFactors); return GetSourcePhraseStringRep(allFactors);
} }
std::string std::string
Hypothesis:: Hypothesis::
GetTargetPhraseStringRep() const GetTargetPhraseStringRep() const
{ {
vector<FactorType> allFactors(MAX_NUM_FACTORS); vector<FactorType> allFactors(MAX_NUM_FACTORS);
for(size_t i=0; i < MAX_NUM_FACTORS; i++) for(size_t i=0; i < MAX_NUM_FACTORS; i++)
allFactors[i] = i; allFactors[i] = i;
return GetTargetPhraseStringRep(allFactors); return GetTargetPhraseStringRep(allFactors);
} }
void void
Hypothesis:: Hypothesis::
OutputAlignment(std::ostream &out) const OutputAlignment(std::ostream &out) const
{ {
std::vector<const Hypothesis *> edges; std::vector<const Hypothesis *> edges;
const Hypothesis *currentHypo = this; const Hypothesis *currentHypo = this;
while (currentHypo) { while (currentHypo) {
@ -467,12 +471,12 @@ namespace Moses
OutputAlignment(out, edges); OutputAlignment(out, edges);
} }
void void
Hypothesis:: Hypothesis::
OutputAlignment(ostream &out, const vector<const Hypothesis *> &edges) OutputAlignment(ostream &out, const vector<const Hypothesis *> &edges)
{ {
size_t targetOffset = 0; size_t targetOffset = 0;
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) { for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
@ -485,13 +489,13 @@ namespace Moses
targetOffset += tp.GetSize(); targetOffset += tp.GetSize();
} }
// Used by --print-alignment-info, so no endl // Used by --print-alignment-info, so no endl
} }
void void
Hypothesis:: Hypothesis::
OutputAlignment(ostream &out, const AlignmentInfo &ai, OutputAlignment(ostream &out, const AlignmentInfo &ai,
size_t sourceOffset, size_t targetOffset) size_t sourceOffset, size_t targetOffset)
{ {
typedef std::vector< const std::pair<size_t,size_t>* > AlignVec; typedef std::vector< const std::pair<size_t,size_t>* > AlignVec;
AlignVec alignments = ai.GetSortedAlignments(); AlignVec alignments = ai.GetSortedAlignments();
@ -501,51 +505,51 @@ namespace Moses
out << alignment.first + sourceOffset << "-" << alignment.second + targetOffset << " "; out << alignment.first + sourceOffset << "-" << alignment.second + targetOffset << " ";
} }
} }
void void
Hypothesis:: Hypothesis::
OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo) OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo)
{ {
if (!hypo->GetPrevHypo()) return; if (!hypo->GetPrevHypo()) return;
OutputInput(map, hypo->GetPrevHypo()); OutputInput(map, hypo->GetPrevHypo());
map[hypo->GetCurrSourceWordsRange().GetStartPos()] map[hypo->GetCurrSourceWordsRange().GetStartPos()]
= &hypo->GetTranslationOption().GetInputPath().GetPhrase(); = &hypo->GetTranslationOption().GetInputPath().GetPhrase();
} }
void void
Hypothesis:: Hypothesis::
OutputInput(std::ostream& os) const OutputInput(std::ostream& os) const
{ {
size_t len = this->GetInput().GetSize(); size_t len = this->GetInput().GetSize();
std::vector<const Phrase*> inp_phrases(len, 0); std::vector<const Phrase*> inp_phrases(len, 0);
OutputInput(inp_phrases, this); OutputInput(inp_phrases, this);
for (size_t i=0; i<len; ++i) for (size_t i=0; i<len; ++i)
if (inp_phrases[i]) os << *inp_phrases[i]; if (inp_phrases[i]) os << *inp_phrases[i];
} }
void void
Hypothesis:: Hypothesis::
OutputBestSurface(std::ostream &out, const std::vector<FactorType> &outputFactorOrder, OutputBestSurface(std::ostream &out, const std::vector<FactorType> &outputFactorOrder,
char reportSegmentation, bool reportAllFactors) const char reportSegmentation, bool reportAllFactors) const
{ {
if (m_prevHypo) if (m_prevHypo) {
{ // recursively retrace this best path through the lattice, starting from the end of the hypothesis sentence // recursively retrace this best path through the lattice, starting from the end of the hypothesis sentence
m_prevHypo->OutputBestSurface(out, outputFactorOrder, reportSegmentation, reportAllFactors); m_prevHypo->OutputBestSurface(out, outputFactorOrder, reportSegmentation, reportAllFactors);
} }
OutputSurface(out, *this, outputFactorOrder, reportSegmentation, reportAllFactors); OutputSurface(out, *this, outputFactorOrder, reportSegmentation, reportAllFactors);
} }
////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////
/*** /***
* print surface factor only for the given phrase * print surface factor only for the given phrase
*/ */
void void
Hypothesis:: Hypothesis::
OutputSurface(std::ostream &out, const Hypothesis &edge, OutputSurface(std::ostream &out, const Hypothesis &edge,
const std::vector<FactorType> &outputFactorOrder, const std::vector<FactorType> &outputFactorOrder,
char reportSegmentation, bool reportAllFactors) const char reportSegmentation, bool reportAllFactors) const
{ {
UTIL_THROW_IF2(outputFactorOrder.size() == 0, UTIL_THROW_IF2(outputFactorOrder.size() == 0,
"Must specific at least 1 output factor"); "Must specific at least 1 output factor");
const TargetPhrase& phrase = edge.GetCurrTargetPhrase(); const TargetPhrase& phrase = edge.GetCurrTargetPhrase();
@ -614,12 +618,12 @@ namespace Moses
} }
out << "| "; out << "| ";
} }
} }
std::map<size_t, const Factor*> std::map<size_t, const Factor*>
Hypothesis:: Hypothesis::
GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor) const GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor) const
{ {
const InputPath &inputPath = hypo.GetTranslationOption().GetInputPath(); const InputPath &inputPath = hypo.GetTranslationOption().GetInputPath();
const Phrase &inputPhrase = inputPath.GetPhrase(); const Phrase &inputPhrase = inputPath.GetPhrase();
@ -636,13 +640,13 @@ namespace Moses
} }
return ret; return ret;
} }
#ifdef HAVE_XMLRPC_C #ifdef HAVE_XMLRPC_C
void void
Hypothesis:: Hypothesis::
OutputLocalWordAlignment(vector<xmlrpc_c::value>& dest) const OutputLocalWordAlignment(vector<xmlrpc_c::value>& dest) const
{ {
using namespace std; using namespace std;
WordsRange const& src = this->GetCurrSourceWordsRange(); WordsRange const& src = this->GetCurrSourceWordsRange();
WordsRange const& trg = this->GetCurrTargetWordsRange(); WordsRange const& trg = this->GetCurrTargetWordsRange();
@ -651,24 +655,23 @@ namespace Moses
= this->GetCurrTargetPhrase().GetAlignTerm().GetSortedAlignments(); = this->GetCurrTargetPhrase().GetAlignTerm().GetSortedAlignments();
typedef pair<size_t,size_t> item; typedef pair<size_t,size_t> item;
map<string, xmlrpc_c::value> M; map<string, xmlrpc_c::value> M;
BOOST_FOREACH(item const* p, a) BOOST_FOREACH(item const* p, a) {
{
M["source-word"] = xmlrpc_c::value_int(src.GetStartPos() + p->first); M["source-word"] = xmlrpc_c::value_int(src.GetStartPos() + p->first);
M["target-word"] = xmlrpc_c::value_int(trg.GetStartPos() + p->second); M["target-word"] = xmlrpc_c::value_int(trg.GetStartPos() + p->second);
dest.push_back(xmlrpc_c::value_struct(M)); dest.push_back(xmlrpc_c::value_struct(M));
} }
} }
void void
Hypothesis:: Hypothesis::
OutputWordAlignment(vector<xmlrpc_c::value>& out) const OutputWordAlignment(vector<xmlrpc_c::value>& out) const
{ {
vector<Hypothesis const*> tmp; vector<Hypothesis const*> tmp;
for (Hypothesis const* h = this; h; h = h->GetPrevHypo()) for (Hypothesis const* h = this; h; h = h->GetPrevHypo())
tmp.push_back(h); tmp.push_back(h);
for (size_t i = tmp.size(); i-- > 0;) for (size_t i = tmp.size(); i-- > 0;)
tmp[i]->OutputLocalWordAlignment(out); tmp[i]->OutputLocalWordAlignment(out);
} }
#endif #endif

View File

@ -161,7 +161,7 @@ public:
return m_detailTreeFragmentsOutputCollector.get(); return m_detailTreeFragmentsOutputCollector.get();
} }
void SetInputStreamFromString(std::istringstream &input){ void SetInputStreamFromString(std::istringstream &input) {
m_inputStream = &input; m_inputStream = &input;
} }

View File

@ -13,12 +13,14 @@ namespace Moses
typedef Eigen::Map<Eigen::Matrix<int,Eigen::Dynamic,1> > EigenMap; typedef Eigen::Map<Eigen::Matrix<int,Eigen::Dynamic,1> > EigenMap;
RDLM::~RDLM() { RDLM::~RDLM()
{
delete lm_head_base_instance_; delete lm_head_base_instance_;
delete lm_label_base_instance_; delete lm_label_base_instance_;
} }
void RDLM::Load() { void RDLM::Load()
{
lm_head_base_instance_ = new nplm::neuralTM(); lm_head_base_instance_ = new nplm::neuralTM();
lm_head_base_instance_->read(m_path_head_lm); lm_head_base_instance_->read(m_path_head_lm);
@ -202,8 +204,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
// ignore glue rules // ignore glue rules
if (root->GetLabel() == m_glueSymbol) { if (root->GetLabel() == m_glueSymbol) {
// recursion // recursion
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it) for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it) {
{
Score(it->get(), back_pointers, score, ancestor_heads, ancestor_labels, boundary_hash, num_virtual, rescoring_levels); Score(it->get(), back_pointers, score, ancestor_heads, ancestor_labels, boundary_hash, num_virtual, rescoring_levels);
} }
return; return;
@ -260,8 +261,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
} }
if (ancestor_labels.size() >= m_context_up && !num_virtual) { if (ancestor_labels.size() >= m_context_up && !num_virtual) {
score[0] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size()))); score[0] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size())));
} } else {
else {
boost::hash_combine(boundary_hash, ngram_head_null.back()); boost::hash_combine(boundary_hash, ngram_head_null.back());
score[1] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size()))); score[1] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size())));
} }
@ -276,8 +276,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
return; return;
} }
rescoring_levels = m_context_up-1; rescoring_levels = m_context_up-1;
} } else {
else {
return; return;
} }
} }
@ -305,16 +304,14 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
virtual_head = true; virtual_head = true;
if (m_binarized == 1 || (m_binarized == 3 && head_label[2] == 'l')) { if (m_binarized == 1 || (m_binarized == 3 && head_label[2] == 'l')) {
reached_end = 1; //indicate that we've seen the first symbol of the RHS reached_end = 1; //indicate that we've seen the first symbol of the RHS
} } else if (m_binarized == 2 || (m_binarized == 3 && head_label[2] == 'r')) {
else if (m_binarized == 2 || (m_binarized == 3 && head_label[2] == 'r')) {
reached_end = 2; // indicate that we've seen the last symbol of the RHS reached_end = 2; // indicate that we've seen the last symbol of the RHS
} }
// with 'full' binarization, direction is encoded in 2nd char // with 'full' binarization, direction is encoded in 2nd char
std::string clipped_label = (m_binarized == 3) ? head_label.substr(2,head_label.size()-2) : head_label.substr(1,head_label.size()-1); std::string clipped_label = (m_binarized == 3) ? head_label.substr(2,head_label.size()-2) : head_label.substr(1,head_label.size()-1);
label_idx = lm_label->lookup_input_word(clipped_label); label_idx = lm_label->lookup_input_word(clipped_label);
label_idx_out = lm_label->lookup_output_word(clipped_label); label_idx_out = lm_label->lookup_output_word(clipped_label);
} } else {
else {
reached_end = 3; // indicate that we've seen first and last symbol of the RHS reached_end = 3; // indicate that we've seen first and last symbol of the RHS
label_idx = lm_label->lookup_input_word(head_label); label_idx = lm_label->lookup_input_word(head_label);
label_idx_out = lm_label->lookup_output_word(head_label); label_idx_out = lm_label->lookup_output_word(head_label);
@ -341,8 +338,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
it = std::copy(ancestor_heads.end()-context_up_nonempty, ancestor_heads.end(), it); it = std::copy(ancestor_heads.end()-context_up_nonempty, ancestor_heads.end(), it);
it = std::copy(ancestor_labels.end()-context_up_nonempty, ancestor_labels.end(), it); it = std::copy(ancestor_labels.end()-context_up_nonempty, ancestor_labels.end(), it);
score[0] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size()))); score[0] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size())));
} } else {
else {
boost::hash_combine(boundary_hash, ngram_head_null.back()); boost::hash_combine(boundary_hash, ngram_head_null.back());
score[1] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size()))); score[1] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size())));
} }
@ -362,8 +358,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
it = std::copy(ancestor_heads.end()-context_up_nonempty, ancestor_heads.end(), it); it = std::copy(ancestor_heads.end()-context_up_nonempty, ancestor_heads.end(), it);
it = std::copy(ancestor_labels.end()-context_up_nonempty, ancestor_labels.end(), it); it = std::copy(ancestor_labels.end()-context_up_nonempty, ancestor_labels.end(), it);
score[2] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram_label_null.data(), ngram_label_null.size()))); score[2] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram_label_null.data(), ngram_label_null.size())));
} } else {
else {
boost::hash_combine(boundary_hash, ngram_label_null.back()); boost::hash_combine(boundary_hash, ngram_label_null.back());
score[3] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram_label_null.data(), ngram_label_null.size()))); score[3] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram_label_null.data(), ngram_label_null.size())));
} }
@ -374,8 +369,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
if (virtual_head) { if (virtual_head) {
num_virtual = m_context_up; num_virtual = m_context_up;
} } else if (num_virtual) {
else if (num_virtual) {
--num_virtual; --num_virtual;
} }
@ -456,8 +450,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
if (reached_end == 2 || reached_end == 3) { if (reached_end == 2 || reached_end == 3) {
std::fill_n(it, right_padding, static_stop_head); std::fill_n(it, right_padding, static_stop_head);
it += right_padding; it += right_padding;
} } else {
else {
std::copy(static_label_null.begin()+offset_up_head-m_context_right-right_padding, static_label_null.begin()-m_context_right+offset_up_head, it); std::copy(static_label_null.begin()+offset_up_head-m_context_right-right_padding, static_label_null.begin()-m_context_right+offset_up_head, it);
} }
} }
@ -468,8 +461,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
if (reached_end == 2 || reached_end == 3) { if (reached_end == 2 || reached_end == 3) {
std::fill_n(it, right_padding, static_stop_label); std::fill_n(it, right_padding, static_stop_label);
it += right_padding; it += right_padding;
} } else {
else {
std::copy(static_label_null.begin()+offset_up_head-right_padding, static_label_null.begin()+offset_up_head, it); std::copy(static_label_null.begin()+offset_up_head-right_padding, static_label_null.begin()+offset_up_head, it);
} }
} }
@ -478,8 +470,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
if (ancestor_labels.size() >= m_context_up && !num_virtual) { if (ancestor_labels.size() >= m_context_up && !num_virtual) {
score[2] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram.data(), ngram.size()))); score[2] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram.data(), ngram.size())));
} } else {
else {
boost::hash_combine(boundary_hash, ngram.back()); boost::hash_combine(boundary_hash, ngram.back());
score[3] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram.data(), ngram.size()))); score[3] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram.data(), ngram.size())));
} }
@ -492,8 +483,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
if (ancestor_labels.size() >= m_context_up && !num_virtual) { if (ancestor_labels.size() >= m_context_up && !num_virtual) {
score[0] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram.data(), ngram.size()))); score[0] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram.data(), ngram.size())));
} } else {
else {
boost::hash_combine(boundary_hash, ngram.back()); boost::hash_combine(boundary_hash, ngram.back());
score[1] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram.data(), ngram.size()))); score[1] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram.data(), ngram.size())));
} }
@ -519,8 +509,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
return; return;
} }
// recursion // recursion
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it) for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it) {
{
Score(it->get(), back_pointers, score, ancestor_heads, ancestor_labels, boundary_hash, num_virtual, rescoring_levels - 1); Score(it->get(), back_pointers, score, ancestor_heads, ancestor_labels, boundary_hash, num_virtual, rescoring_levels - 1);
} }
ancestor_heads.pop_back(); ancestor_heads.pop_back();
@ -531,12 +520,10 @@ InternalTree* RDLM::GetHead(InternalTree* root, const TreePointerMap & back_poin
{ {
InternalTree *tree; InternalTree *tree;
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it) for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it) {
{
if ((*it)->IsLeafNT()) { if ((*it)->IsLeafNT()) {
tree = back_pointers.find(it->get())->second.get(); tree = back_pointers.find(it->get())->second.get();
} } else {
else {
tree = it->get(); tree = it->get();
} }
@ -563,8 +550,7 @@ InternalTree* RDLM::GetHead(InternalTree* root, const TreePointerMap & back_poin
for (std::vector<TreePointer>::const_iterator it2 = tree->GetChildren().begin(); it2 != tree->GetChildren().end(); ++it2) { for (std::vector<TreePointer>::const_iterator it2 = tree->GetChildren().begin(); it2 != tree->GetChildren().end(); ++it2) {
if ((*it2)->IsLeafNT()) { if ((*it2)->IsLeafNT()) {
tree2 = back_pointers.find(it2->get())->second.get(); tree2 = back_pointers.find(it2->get())->second.get();
} } else {
else {
tree2 = it2->get(); tree2 = it2->get();
} }
if (tree2->GetLabel() == "PTKVZ" && tree2->GetLength() == 1 && tree2->GetChildren()[0]->IsTerminal()) { if (tree2->GetLabel() == "PTKVZ" && tree2->GetLength() == 1 && tree2->GetChildren()[0]->IsTerminal()) {
@ -659,8 +645,7 @@ void RDLM::GetIDs(const std::string & head, const std::string & preterminal, std
} }
if (m_sharedVocab) { if (m_sharedVocab) {
IDs.second = IDs.first; IDs.second = IDs.first;
} } else {
else {
IDs.second = lm_head_base_instance_->lookup_output_word(head); IDs.second = lm_head_base_instance_->lookup_output_word(head);
if (m_isPretermBackoff && IDs.second == 0) { if (m_isPretermBackoff && IDs.second == 0) {
IDs.second = lm_head_base_instance_->lookup_output_word(preterminal); IDs.second = lm_head_base_instance_->lookup_output_word(preterminal);
@ -694,8 +679,7 @@ RDLM::TreePointerMap RDLM::AssociateLeafNTs(InternalTree* root, const std::vecto
found = next_leafNT(it); found = next_leafNT(it);
if (found) { if (found) {
ret[it->get()] = *it_prev; ret[it->get()] = *it_prev;
} } else {
else {
std::cerr << "Warning: leaf nonterminal not found in rule; why did this happen?\n"; std::cerr << "Warning: leaf nonterminal not found in rule; why did this happen?\n";
} }
} }
@ -822,8 +806,7 @@ FFState* RDLM::EvaluateWhenApplied(const ChartHypothesis& cur_hypo
} }
return new RDLMState(mytree, score[1], score[3], boundary_hash); return new RDLMState(mytree, score[1], score[3], boundary_hash);
} } else {
else {
UTIL_THROW2("Error: RDLM active, but no internal tree structure found"); UTIL_THROW2("Error: RDLM active, but no internal tree structure found");
} }

View File

@ -11,8 +11,9 @@
// Sennrich, Rico (2015). Modelling and Optimizing on Syntactic N-Grams for Statistical Machine Translation. Transactions of the Association for Computational Linguistics. // Sennrich, Rico (2015). Modelling and Optimizing on Syntactic N-Grams for Statistical Machine Translation. Transactions of the Association for Computational Linguistics.
// see 'scripts/training/rdlm' for training scripts // see 'scripts/training/rdlm' for training scripts
namespace nplm { namespace nplm
class neuralTM; {
class neuralTM;
} }
namespace Moses namespace Moses
@ -121,8 +122,7 @@ public:
, m_normalizeLabelLM(false) , m_normalizeLabelLM(false)
, m_sharedVocab(false) , m_sharedVocab(false)
, m_binarized(0) , m_binarized(0)
, m_cacheSize(1000000) , m_cacheSize(1000000) {
{
ReadParameters(); ReadParameters();
} }
@ -161,7 +161,9 @@ public:
FFState* EvaluateWhenApplied( FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo, const Hypothesis& cur_hypo,
const FFState* prev_state, const FFState* prev_state,
ScoreComponentCollection* accumulator) const {UTIL_THROW(util::Exception, "Not implemented");}; ScoreComponentCollection* accumulator) const {
UTIL_THROW(util::Exception, "Not implemented");
};
FFState* EvaluateWhenApplied( FFState* EvaluateWhenApplied(
const ChartHypothesis& /* cur_hypo */, const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */, int /* featureID - used to index the state in the previous hypotheses */,
@ -185,8 +187,7 @@ public:
UnbinarizedChildren(InternalTree* root, const TreePointerMap & pointers, bool binary): UnbinarizedChildren(InternalTree* root, const TreePointerMap & pointers, bool binary):
current(root), current(root),
back_pointers(pointers), back_pointers(pointers),
binarized(binary) binarized(binary) {
{
stack.reserve(10); stack.reserve(10);
_end = current->GetChildren().end(); _end = current->GetChildren().end();
iter = current->GetChildren().begin(); iter = current->GetChildren().begin();
@ -196,8 +197,7 @@ public:
// also go through trees or previous hypotheses to rescore nodes for which more context has become available // also go through trees or previous hypotheses to rescore nodes for which more context has become available
if ((*iter)->IsLeafNT()) { if ((*iter)->IsLeafNT()) {
current = back_pointers.find(iter->get())->second.get(); current = back_pointers.find(iter->get())->second.get();
} } else {
else {
current = iter->get(); current = iter->get();
} }
iter = current->GetChildren().begin(); iter = current->GetChildren().begin();
@ -205,8 +205,12 @@ public:
_begin = iter; _begin = iter;
} }
std::vector<TreePointer>::const_iterator begin() const { return _begin; } std::vector<TreePointer>::const_iterator begin() const {
std::vector<TreePointer>::const_iterator end() const { return _end; } return _begin;
}
std::vector<TreePointer>::const_iterator end() const {
return _end;
}
std::vector<TreePointer>::const_iterator operator++() { std::vector<TreePointer>::const_iterator operator++() {
iter++; iter++;
@ -230,8 +234,7 @@ public:
// also go through trees or previous hypotheses to rescore nodes for which more context has become available // also go through trees or previous hypotheses to rescore nodes for which more context has become available
if ((*iter)->IsLeafNT()) { if ((*iter)->IsLeafNT()) {
current = back_pointers.find(iter->get())->second.get(); current = back_pointers.find(iter->get())->second.get();
} } else {
else {
current = iter->get(); current = iter->get();
} }
iter = current->GetChildren().begin(); iter = current->GetChildren().begin();

View File

@ -87,7 +87,9 @@ Manager::~Manager()
const InputType& const InputType&
Manager::GetSource() const Manager::GetSource() const
{ return m_source ; } {
return m_source ;
}
/** /**
* Main decoder loop that translates a sentence by expanding * Main decoder loop that translates a sentence by expanding

View File

@ -110,7 +110,7 @@ private:
#endif #endif
public: public:
void SetOutputStream(std::ostream* outStream){ void SetOutputStream(std::ostream* outStream) {
m_outStream = outStream; m_outStream = outStream;
} }

View File

@ -368,8 +368,7 @@ AddParam(po::options_description& optgroup,
m_fullname[abbrevName] = paramName; m_fullname[abbrevName] = paramName;
m_description[paramName] = description; m_description[paramName] = description;
string optname = paramName; string optname = paramName;
if (abbrevName.size() == 1) if (abbrevName.size() == 1) {
{
optname += string(",")+abbrevName; optname += string(",")+abbrevName;
// m_confusable[abbrevName[0]].insert(paramName); // m_confusable[abbrevName[0]].insert(paramName);
} }
@ -429,8 +428,7 @@ LoadParam(int argc, char* xargv[])
// legacy parameter handling: all parameters are expected // legacy parameter handling: all parameters are expected
// to start with a single dash // to start with a single dash
char* argv[argc+1]; char* argv[argc+1];
for (int i = 0; i < argc; ++i) for (int i = 0; i < argc; ++i) {
{
argv[i] = xargv[i]; argv[i] = xargv[i];
if (strlen(argv[i]) > 2 && argv[i][0] == '-' && argv[i][1] == '-') if (strlen(argv[i]) > 2 && argv[i][0] == '-' && argv[i][1] == '-')
++argv[i]; ++argv[i];

View File

@ -195,14 +195,12 @@ void ScoreComponentCollection::Save(ostream& out, bool multiline) const
std::vector<FeatureFunction*> const& all_ff std::vector<FeatureFunction*> const& all_ff
= FeatureFunction::GetFeatureFunctions(); = FeatureFunction::GetFeatureFunctions();
BOOST_FOREACH(FeatureFunction const* ff, all_ff) BOOST_FOREACH(FeatureFunction const* ff, all_ff) {
{
string name = ff->GetScoreProducerDescription(); string name = ff->GetScoreProducerDescription();
size_t i = ff->GetIndex(); size_t i = ff->GetIndex();
if (ff->GetNumScoreComponents() == 1) if (ff->GetNumScoreComponents() == 1)
out << name << sep << m_scores[i] << linesep; out << name << sep << m_scores[i] << linesep;
else else {
{
size_t stop = i + ff->GetNumScoreComponents(); size_t stop = i + ff->GetNumScoreComponents();
boost::format fmt("%s_%d"); boost::format fmt("%s_%d");
for (size_t k = 1; i < stop; ++i, ++k) for (size_t k = 1; i < stop; ++i, ++k)

View File

@ -234,7 +234,7 @@ public:
const ScoreComponentCollection& scores) { const ScoreComponentCollection& scores) {
size_t i = sp->GetIndex(); size_t i = sp->GetIndex();
size_t stop = i + sp->GetNumScoreComponents(); size_t stop = i + sp->GetNumScoreComponents();
for (;i < stop; ++i) m_scores[i] += scores.m_scores[i]; for (; i < stop; ++i) m_scores[i] += scores.m_scores[i];
} }
//! Add scores from a single FeatureFunction only //! Add scores from a single FeatureFunction only

View File

@ -60,23 +60,18 @@ aux_init_partial_translation(string& line)
string sourceCompletedStr; string sourceCompletedStr;
int loc1 = line.find( "|||", 0 ); int loc1 = line.find( "|||", 0 );
int loc2 = line.find( "|||", loc1 + 3 ); int loc2 = line.find( "|||", loc1 + 3 );
if (loc1 > -1 && loc2 > -1) if (loc1 > -1 && loc2 > -1) {
{
m_initialTargetPhrase = Trim(line.substr(0, loc1)); m_initialTargetPhrase = Trim(line.substr(0, loc1));
string scov = Trim(line.substr(loc1 + 3, loc2 - loc1 - 3)); string scov = Trim(line.substr(loc1 + 3, loc2 - loc1 - 3));
line = line.substr(loc2 + 3); line = line.substr(loc2 + 3);
m_sourceCompleted.resize(scov.size()); m_sourceCompleted.resize(scov.size());
int contiguous = 1; int contiguous = 1;
for (size_t i = 0; i < scov.size(); ++i) for (size_t i = 0; i < scov.size(); ++i) {
{ if (sourceCompletedStr.at(i) == '1') {
if (sourceCompletedStr.at(i) == '1')
{
m_sourceCompleted[i] = true; m_sourceCompleted[i] = true;
if (contiguous) m_frontSpanCoveredLength++; if (contiguous) m_frontSpanCoveredLength++;
} } else {
else
{
m_sourceCompleted[i] = false; m_sourceCompleted[i] = false;
contiguous = 0; contiguous = 0;
} }
@ -94,38 +89,31 @@ aux_interpret_sgml_markup(string& line)
metamap::const_iterator i; metamap::const_iterator i;
if ((i = meta.find("id")) != meta.end()) if ((i = meta.find("id")) != meta.end())
this->SetTranslationId(atol(i->second.c_str())); this->SetTranslationId(atol(i->second.c_str()));
if ((i = meta.find("docid")) != meta.end()) if ((i = meta.find("docid")) != meta.end()) {
{
this->SetDocumentId(atol(i->second.c_str())); this->SetDocumentId(atol(i->second.c_str()));
this->SetUseTopicId(false); this->SetUseTopicId(false);
this->SetUseTopicIdAndProb(false); this->SetUseTopicIdAndProb(false);
} }
if ((i = meta.find("topic")) != meta.end()) if ((i = meta.find("topic")) != meta.end()) {
{
vector<string> topic_params; vector<string> topic_params;
boost::split(topic_params, i->second, boost::is_any_of("\t ")); boost::split(topic_params, i->second, boost::is_any_of("\t "));
if (topic_params.size() == 1) if (topic_params.size() == 1) {
{
this->SetTopicId(atol(topic_params[0].c_str())); this->SetTopicId(atol(topic_params[0].c_str()));
this->SetUseTopicId(true); this->SetUseTopicId(true);
this->SetUseTopicIdAndProb(false); this->SetUseTopicIdAndProb(false);
} } else {
else
{
this->SetTopicIdAndProb(topic_params); this->SetTopicIdAndProb(topic_params);
this->SetUseTopicId(false); this->SetUseTopicId(false);
this->SetUseTopicIdAndProb(true); this->SetUseTopicIdAndProb(true);
} }
} }
if ((i = meta.find("weight-setting")) != meta.end()) if ((i = meta.find("weight-setting")) != meta.end()) {
{
this->SetWeightSetting(i->second); this->SetWeightSetting(i->second);
this->SetSpecifiesWeightSetting(true); this->SetSpecifiesWeightSetting(true);
StaticData::Instance().SetWeightSetting(i->second); StaticData::Instance().SetWeightSetting(i->second);
// oh this is so horrible! Why does this have to be propagated globally? // oh this is so horrible! Why does this have to be propagated globally?
// --- UG // --- UG
} } else this->SetSpecifiesWeightSetting(false);
else this->SetSpecifiesWeightSetting(false);
} }
void void
@ -135,21 +123,17 @@ aux_interpret_dlt(string& line) // whatever DLT means ... --- UG
using namespace std; using namespace std;
typedef map<string, string> str2str_map; typedef map<string, string> str2str_map;
vector<str2str_map> meta = ProcessAndStripDLT(line); vector<str2str_map> meta = ProcessAndStripDLT(line);
BOOST_FOREACH(str2str_map const& M, meta) BOOST_FOREACH(str2str_map const& M, meta) {
{
str2str_map::const_iterator i,j; str2str_map::const_iterator i,j;
if ((i = M.find("type")) != M.end()) if ((i = M.find("type")) != M.end()) {
{
j = M.find("id"); j = M.find("id");
string id = j == M.end() ? "default" : j->second; string id = j == M.end() ? "default" : j->second;
if (i->second == "cbtm") if (i->second == "cbtm") {
{
PhraseDictionaryDynamicCacheBased* cbtm; PhraseDictionaryDynamicCacheBased* cbtm;
cbtm = PhraseDictionaryDynamicCacheBased::InstanceNonConst(id); cbtm = PhraseDictionaryDynamicCacheBased::InstanceNonConst(id);
if (cbtm) cbtm->ExecuteDlt(M); if (cbtm) cbtm->ExecuteDlt(M);
} }
if (i->second == "cblm") if (i->second == "cblm") {
{
DynamicCacheBasedLanguageModel* cblm; DynamicCacheBasedLanguageModel* cblm;
cblm = DynamicCacheBasedLanguageModel::InstanceNonConst(id); cblm = DynamicCacheBasedLanguageModel::InstanceNonConst(id);
if (cblm) cblm->ExecuteDlt(M); if (cblm) cblm->ExecuteDlt(M);
@ -162,13 +146,13 @@ void
Sentence:: Sentence::
aux_interpret_xml(std::string& line, std::vector<size_t> & xmlWalls, aux_interpret_xml(std::string& line, std::vector<size_t> & xmlWalls,
std::vector<std::pair<size_t, std::string> >& placeholders) std::vector<std::pair<size_t, std::string> >& placeholders)
{ // parse XML markup in translation line {
// parse XML markup in translation line
const StaticData &SD = StaticData::Instance(); const StaticData &SD = StaticData::Instance();
using namespace std; using namespace std;
if (SD.GetXmlInputType() != XmlPassThrough) if (SD.GetXmlInputType() != XmlPassThrough) {
{
int offset = SD.IsSyntax() ? 1 : 0; int offset = SD.IsSyntax() ? 1 : 0;
bool OK = ProcessAndStripXMLTags(line, m_xmlOptions, bool OK = ProcessAndStripXMLTags(line, m_xmlOptions,
m_reorderingConstraint, m_reorderingConstraint,
@ -197,8 +181,7 @@ init(string line, std::vector<FactorType> const& factorOrder)
aux_interpret_dlt(line); // some poorly documented cache-based stuff aux_interpret_dlt(line); // some poorly documented cache-based stuff
// if sentences is specified as "<passthrough tag1=""/>" // if sentences is specified as "<passthrough tag1=""/>"
if (SD.IsPassthroughEnabled() || SD.IsPassthroughInNBestEnabled()) if (SD.IsPassthroughEnabled() || SD.IsPassthroughInNBestEnabled()) {
{
string pthru = PassthroughSGML(line,"passthrough"); string pthru = PassthroughSGML(line,"passthrough");
this->SetPassthroughInformation(pthru); this->SetPassthroughInformation(pthru);
} }
@ -218,11 +201,9 @@ init(string line, std::vector<FactorType> const& factorOrder)
// our XmlOptions and create TranslationOptions // our XmlOptions and create TranslationOptions
// only fill the vector if we are parsing XML // only fill the vector if we are parsing XML
if (SD.GetXmlInputType() != XmlPassThrough) if (SD.GetXmlInputType() != XmlPassThrough) {
{
m_xmlCoverageMap.assign(GetSize(), false); m_xmlCoverageMap.assign(GetSize(), false);
BOOST_FOREACH(XmlOption* o, m_xmlOptions) BOOST_FOREACH(XmlOption* o, m_xmlOptions) {
{
WordsRange const& r = o->range; WordsRange const& r = o->range;
for(size_t j = r.GetStartPos(); j <= r.GetEndPos(); ++j) for(size_t j = r.GetStartPos(); j <= r.GetEndPos(); ++j)
m_xmlCoverageMap[j]=true; m_xmlCoverageMap[j]=true;
@ -233,8 +214,7 @@ init(string line, std::vector<FactorType> const& factorOrder)
m_reorderingConstraint.InitializeWalls(GetSize()); m_reorderingConstraint.InitializeWalls(GetSize());
// set reordering walls, if "-monotone-at-punction" is set // set reordering walls, if "-monotone-at-punction" is set
if (SD.UseReorderingConstraint() && GetSize()) if (SD.UseReorderingConstraint() && GetSize()) {
{
WordsRange r(0, GetSize()-1); WordsRange r(0, GetSize()-1);
m_reorderingConstraint.SetMonotoneAtPunctuation(GetSubString(r)); m_reorderingConstraint.SetMonotoneAtPunctuation(GetSubString(r));
} }

View File

@ -32,22 +32,22 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
namespace Moses namespace Moses
{ {
class WordsRange; class WordsRange;
class PhraseDictionary; class PhraseDictionary;
class TranslationOption; class TranslationOption;
class TranslationOptionCollection; class TranslationOptionCollection;
class ChartTranslationOptions; class ChartTranslationOptions;
class TranslationTask; class TranslationTask;
struct XmlOption; struct XmlOption;
/** /**
* A Phrase class with an ID. Used specifically as source input so contains functionality to read * A Phrase class with an ID. Used specifically as source input so contains functionality to read
* from IODevice and create trans opt * from IODevice and create trans opt
*/ */
class Sentence : public Phrase, public InputType class Sentence : public Phrase, public InputType
{ {
protected: protected:
/** /**
* Utility method that takes in a string representing an XML tag and the name of the attribute, * Utility method that takes in a string representing an XML tag and the name of the attribute,
@ -61,7 +61,7 @@ namespace Moses
void ProcessPlaceholders(const std::vector< std::pair<size_t, std::string> > &placeholders); void ProcessPlaceholders(const std::vector< std::pair<size_t, std::string> > &placeholders);
public: public:
Sentence(); Sentence();
Sentence(size_t const transId, std::string const& stext, Sentence(size_t const transId, std::string const& stext,
std::vector<FactorType> const* IFO = NULL); std::vector<FactorType> const* IFO = NULL);
@ -106,14 +106,15 @@ namespace Moses
std::string const& phraseString); std::string const& phraseString);
const NonTerminalSet& const NonTerminalSet&
GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const {
{ return m_defaultLabelSet; } return m_defaultLabelSet;
}
void void
init(std::string line, std::vector<FactorType> const& factorOrder); init(std::string line, std::vector<FactorType> const& factorOrder);
private: private:
// auxliliary functions for Sentence initialization // auxliliary functions for Sentence initialization
// void aux_interpret_sgml_markup(std::string& line); // void aux_interpret_sgml_markup(std::string& line);
// void aux_interpret_dlt(std::string& line); // void aux_interpret_dlt(std::string& line);
@ -134,7 +135,7 @@ namespace Moses
void void
aux_init_partial_translation(std::string& line); aux_init_partial_translation(std::string& line);
}; };
} }

View File

@ -476,8 +476,7 @@ public:
// m_searchAlgorithm == SyntaxF2S; // m_searchAlgorithm == SyntaxF2S;
// } // }
bool IsSyntax(SearchAlgorithm algo = DefaultSearchAlgorithm) const bool IsSyntax(SearchAlgorithm algo = DefaultSearchAlgorithm) const {
{
if (algo == DefaultSearchAlgorithm) if (algo == DefaultSearchAlgorithm)
algo = m_searchAlgorithm; algo = m_searchAlgorithm;
return (algo == CYKPlus || algo == ChartIncremental || return (algo == CYKPlus || algo == ChartIncremental ||
@ -486,8 +485,9 @@ public:
} }
const ScoreComponentCollection& const ScoreComponentCollection&
GetAllWeights() const GetAllWeights() const {
{ return m_allWeights; } return m_allWeights;
}
void SetAllWeights(const ScoreComponentCollection& weights) { void SetAllWeights(const ScoreComponentCollection& weights) {
m_allWeights = weights; m_allWeights = weights;

View File

@ -232,11 +232,9 @@ mergescores(boost::shared_ptr<Scores> const& a,
if (!b) return a; if (!b) return a;
if (a->size() != b->size()) return ret; if (a->size() != b->size()) return ret;
ret.reset(new Scores(*a)); ret.reset(new Scores(*a));
for (size_t i = 0; i < a->size(); ++i) for (size_t i = 0; i < a->size(); ++i) {
{
if ((*a)[i] == 0) (*a)[i] = (*b)[i]; if ((*a)[i] == 0) (*a)[i] = (*b)[i];
else if ((*b)[i]) else if ((*b)[i]) {
{
UTIL_THROW_IF2((*a)[i] != (*b)[i], "can't merge feature vectors"); UTIL_THROW_IF2((*a)[i] != (*b)[i], "can't merge feature vectors");
} }
} }
@ -253,8 +251,7 @@ Merge(const TargetPhrase &copy, const std::vector<FactorType>& factorVec)
m_fullScore += copy.m_fullScore; m_fullScore += copy.m_fullScore;
typedef ScoreCache_t::iterator iter; typedef ScoreCache_t::iterator iter;
typedef ScoreCache_t::value_type item; typedef ScoreCache_t::value_type item;
BOOST_FOREACH(item const& s, copy.m_cached_scores) BOOST_FOREACH(item const& s, copy.m_cached_scores) {
{
pair<iter,bool> foo = m_cached_scores.insert(s); pair<iter,bool> foo = m_cached_scores.insert(s);
if (foo.second == false) if (foo.second == false)
foo.first->second = mergescores(foo.first->second, s.second); foo.first->second = mergescores(foo.first->second, s.second);
@ -280,7 +277,9 @@ void
TargetPhrase:: TargetPhrase::
SetExtraScores(FeatureFunction const* ff, SetExtraScores(FeatureFunction const* ff,
boost::shared_ptr<Scores> const& s) boost::shared_ptr<Scores> const& s)
{ m_cached_scores[ff] = s; } {
m_cached_scores[ff] = s;
}
void TargetPhrase::SetProperties(const StringPiece &str) void TargetPhrase::SetProperties(const StringPiece &str)

View File

@ -51,7 +51,7 @@ class PhraseDictionary;
*/ */
class TargetPhrase: public Phrase class TargetPhrase: public Phrase
{ {
public: public:
typedef std::map<FeatureFunction const*, boost::shared_ptr<Scores> > typedef std::map<FeatureFunction const*, boost::shared_ptr<Scores> >
ScoreCache_t; ScoreCache_t;
ScoreCache_t const& GetExtraScores() const; ScoreCache_t const& GetExtraScores() const;
@ -59,7 +59,7 @@ class TargetPhrase: public Phrase
void SetExtraScores(FeatureFunction const* ff, void SetExtraScores(FeatureFunction const* ff,
boost::shared_ptr<Scores> const& scores); boost::shared_ptr<Scores> const& scores);
private: private:
ScoreCache_t m_cached_scores; ScoreCache_t m_cached_scores;
private: private:

View File

@ -26,8 +26,7 @@ public:
// factory function // factory function
static boost::shared_ptr<TrainingTask> static boost::shared_ptr<TrainingTask>
create(boost::shared_ptr<InputType> const& source) create(boost::shared_ptr<InputType> const& source) {
{
boost::shared_ptr<IOWrapper> nix; boost::shared_ptr<IOWrapper> nix;
boost::shared_ptr<TrainingTask> ret(new TrainingTask(source, nix)); boost::shared_ptr<TrainingTask> ret(new TrainingTask(source, nix));
ret->m_self = ret; ret->m_self = ret;
@ -37,8 +36,7 @@ public:
// factory function // factory function
static boost::shared_ptr<TrainingTask> static boost::shared_ptr<TrainingTask>
create(boost::shared_ptr<InputType> const& source, create(boost::shared_ptr<InputType> const& source,
boost::shared_ptr<IOWrapper> const& ioWrapper) boost::shared_ptr<IOWrapper> const& ioWrapper) {
{
boost::shared_ptr<TrainingTask> ret(new TrainingTask(source, ioWrapper)); boost::shared_ptr<TrainingTask> ret(new TrainingTask(source, ioWrapper));
ret->m_self = ret; ret->m_self = ret;
return ret; return ret;

View File

@ -117,8 +117,7 @@ public:
virtual virtual
TargetPhraseCollection const * TargetPhraseCollection const *
GetTargetPhraseCollectionLEGACY(ttasksptr const& ttask, const Phrase& src) GetTargetPhraseCollectionLEGACY(ttasksptr const& ttask, const Phrase& src) {
{
return GetTargetPhraseCollectionLEGACY(src); return GetTargetPhraseCollectionLEGACY(src);
} }
@ -129,8 +128,7 @@ public:
virtual virtual
void void
GetTargetPhraseCollectionBatch(ttasksptr const& ttask, GetTargetPhraseCollectionBatch(ttasksptr const& ttask,
const InputPathList &inputPathQueue) const const InputPathList &inputPathQueue) const {
{
GetTargetPhraseCollectionBatch(inputPathQueue); GetTargetPhraseCollectionBatch(inputPathQueue);
} }

View File

@ -109,7 +109,7 @@ ostream& operator<<(ostream& out, const TranslationOption& possibleTranslation)
return out; return out;
} }
/** returns cached scores */ /** returns cached scores */
const Scores* const Scores*
TranslationOption:: TranslationOption::
GetLexReorderingScores(LexicalReordering const* scoreProducer) const GetLexReorderingScores(LexicalReordering const* scoreProducer) const

View File

@ -626,8 +626,7 @@ CacheLexReordering()
{ {
size_t const stop = m_source.GetSize(); size_t const stop = m_source.GetSize();
typedef StatefulFeatureFunction sfFF; typedef StatefulFeatureFunction sfFF;
BOOST_FOREACH(sfFF const* ff, sfFF::GetStatefulFeatureFunctions()) BOOST_FOREACH(sfFF const* ff, sfFF::GetStatefulFeatureFunctions()) {
{
if (typeid(*ff) != typeid(LexicalReordering)) continue; if (typeid(*ff) != typeid(LexicalReordering)) continue;
LexicalReordering const& lr = static_cast<const LexicalReordering&>(*ff); LexicalReordering const& lr = static_cast<const LexicalReordering&>(*ff);
for (size_t s = 0 ; s < stop ; s++) for (size_t s = 0 ; s < stop ; s++)

View File

@ -177,8 +177,7 @@ public:
return m_inputPathQueue; return m_inputPathQueue;
} }
ttasksptr GetTranslationTask() const ttasksptr GetTranslationTask() const {
{
return m_ttask.lock(); return m_ttask.lock();
} }
TO_STRING(); TO_STRING();

View File

@ -82,33 +82,29 @@ TranslationTask
if (!staticData.IsSyntax(algo)) if (!staticData.IsSyntax(algo))
manager.reset(new Manager(this->self())); // phrase-based manager.reset(new Manager(this->self())); // phrase-based
else if (algo == SyntaxF2S || algo == SyntaxT2S) else if (algo == SyntaxF2S || algo == SyntaxT2S) {
{ // STSG-based tree-to-string / forest-to-string decoding (ask Phil Williams) // STSG-based tree-to-string / forest-to-string decoding (ask Phil Williams)
typedef Syntax::F2S::RuleMatcherCallback Callback; typedef Syntax::F2S::RuleMatcherCallback Callback;
typedef Syntax::F2S::RuleMatcherHyperTree<Callback> RuleMatcher; typedef Syntax::F2S::RuleMatcherHyperTree<Callback> RuleMatcher;
manager.reset(new Syntax::F2S::Manager<RuleMatcher>(this->self())); manager.reset(new Syntax::F2S::Manager<RuleMatcher>(this->self()));
} }
else if (algo == SyntaxS2T) else if (algo == SyntaxS2T) {
{ // new-style string-to-tree decoding (ask Phil Williams) // new-style string-to-tree decoding (ask Phil Williams)
S2TParsingAlgorithm algorithm = staticData.GetS2TParsingAlgorithm(); S2TParsingAlgorithm algorithm = staticData.GetS2TParsingAlgorithm();
if (algorithm == RecursiveCYKPlus) if (algorithm == RecursiveCYKPlus) {
{
typedef Syntax::S2T::EagerParserCallback Callback; typedef Syntax::S2T::EagerParserCallback Callback;
typedef Syntax::S2T::RecursiveCYKPlusParser<Callback> Parser; typedef Syntax::S2T::RecursiveCYKPlusParser<Callback> Parser;
manager.reset(new Syntax::S2T::Manager<Parser>(this->self())); manager.reset(new Syntax::S2T::Manager<Parser>(this->self()));
} } else if (algorithm == Scope3) {
else if (algorithm == Scope3)
{
typedef Syntax::S2T::StandardParserCallback Callback; typedef Syntax::S2T::StandardParserCallback Callback;
typedef Syntax::S2T::Scope3Parser<Callback> Parser; typedef Syntax::S2T::Scope3Parser<Callback> Parser;
manager.reset(new Syntax::S2T::Manager<Parser>(this->self())); manager.reset(new Syntax::S2T::Manager<Parser>(this->self()));
} } else UTIL_THROW2("ERROR: unhandled S2T parsing algorithm");
else UTIL_THROW2("ERROR: unhandled S2T parsing algorithm");
} }
else if (algo == SyntaxT2S_SCFG) else if (algo == SyntaxT2S_SCFG) {
{ // SCFG-based tree-to-string decoding (ask Phil Williams) // SCFG-based tree-to-string decoding (ask Phil Williams)
typedef Syntax::F2S::RuleMatcherCallback Callback; typedef Syntax::F2S::RuleMatcherCallback Callback;
typedef Syntax::T2S::RuleMatcherSCFG<Callback> RuleMatcher; typedef Syntax::T2S::RuleMatcherSCFG<Callback> RuleMatcher;
manager.reset(new Syntax::T2S::Manager<RuleMatcher>(this->self())); manager.reset(new Syntax::T2S::Manager<RuleMatcher>(this->self()));

View File

@ -40,7 +40,9 @@ class TranslationTask : public Moses::Task
TranslationTask(TranslationTask const& other) { } TranslationTask(TranslationTask const& other) { }
TranslationTask const& TranslationTask const&
operator=(TranslationTask const& other) { return *this; } operator=(TranslationTask const& other) {
return *this;
}
protected: protected:
boost::weak_ptr<TranslationTask> m_self; // weak ptr to myself boost::weak_ptr<TranslationTask> m_self; // weak ptr to myself
@ -68,11 +70,15 @@ protected:
public: public:
boost::shared_ptr<TranslationTask> boost::shared_ptr<TranslationTask>
self() { return m_self.lock(); } self() {
return m_self.lock();
}
virtual virtual
boost::shared_ptr<TranslationTask const> boost::shared_ptr<TranslationTask const>
self() const { return m_self.lock(); } self() const {
return m_self.lock();
}
// creator functions // creator functions
static boost::shared_ptr<TranslationTask> create(); static boost::shared_ptr<TranslationTask> create();
@ -92,15 +98,16 @@ public:
virtual void Run(); virtual void Run();
boost::shared_ptr<Moses::InputType> boost::shared_ptr<Moses::InputType>
GetSource() const { return m_source; } GetSource() const {
return m_source;
}
boost::shared_ptr<BaseManager> boost::shared_ptr<BaseManager>
SetupManager(SearchAlgorithm algo = DefaultSearchAlgorithm); SetupManager(SearchAlgorithm algo = DefaultSearchAlgorithm);
boost::shared_ptr<ContextScope> const& boost::shared_ptr<ContextScope> const&
GetScope() const GetScope() const {
{
UTIL_THROW_IF2(m_scope == NULL, "No context scope!"); UTIL_THROW_IF2(m_scope == NULL, "No context scope!");
return m_scope; return m_scope;
} }

View File

@ -8,7 +8,7 @@
namespace Moses namespace Moses
{ {
class TranslationTask; class TranslationTask;
//! @todo what is this? //! @todo what is this?
class XMLParseOutput class XMLParseOutput
{ {

View File

@ -3,36 +3,34 @@
namespace MosesServer namespace MosesServer
{ {
using namespace std; using namespace std;
Optimizer:: Optimizer::
Optimizer() Optimizer()
{ {
// signature and help strings are documentation -- the client // signature and help strings are documentation -- the client
// can query this information with a system.methodSignature and // can query this information with a system.methodSignature and
// system.methodHelp RPC. // system.methodHelp RPC.
this->_signature = "S:S"; this->_signature = "S:S";
this->_help = "Optimizes multi-model translation model"; this->_help = "Optimizes multi-model translation model";
} }
void void
Optimizer:: Optimizer::
execute(xmlrpc_c::paramList const& paramList, execute(xmlrpc_c::paramList const& paramList,
xmlrpc_c::value * const retvalP) xmlrpc_c::value * const retvalP)
{ {
#ifdef WITH_DLIB #ifdef WITH_DLIB
const params_t params = paramList.getStruct(0); const params_t params = paramList.getStruct(0);
params_t::const_iterator si; params_t::const_iterator si;
if ((si = params.find("model_name")) == params.end()) if ((si = params.find("model_name")) == params.end()) {
{
string msg = "Missing name of model to be optimized"; string msg = "Missing name of model to be optimized";
msg += " (e.g. PhraseDictionaryMultiModelCounts0)"; msg += " (e.g. PhraseDictionaryMultiModelCounts0)";
throw xmlrpc_c::fault(msg, xmlrpc_c::fault::CODE_PARSE); throw xmlrpc_c::fault(msg, xmlrpc_c::fault::CODE_PARSE);
} }
const string model_name = xmlrpc_c::value_string(si->second); const string model_name = xmlrpc_c::value_string(si->second);
if ((si = params.find("phrase_pairs")) == params.end()) if ((si = params.find("phrase_pairs")) == params.end()) {
{
throw xmlrpc_c::fault("Missing list of phrase pairs", throw xmlrpc_c::fault("Missing list of phrase pairs",
xmlrpc_c::fault::CODE_PARSE); xmlrpc_c::fault::CODE_PARSE);
} }
@ -42,8 +40,7 @@ namespace MosesServer
xmlrpc_c::value_array pp_array = xmlrpc_c::value_array(si->second); xmlrpc_c::value_array pp_array = xmlrpc_c::value_array(si->second);
vector<xmlrpc_c::value> ppValVec(pp_array.vectorValueValue()); vector<xmlrpc_c::value> ppValVec(pp_array.vectorValueValue());
for (size_t i = 0; i < ppValVec.size(); ++i) for (size_t i = 0; i < ppValVec.size(); ++i) {
{
xmlrpc_c::value_array pp_array xmlrpc_c::value_array pp_array
= xmlrpc_c::value_array(ppValVec[i]); = xmlrpc_c::value_array(ppValVec[i]);
vector<xmlrpc_c::value> pp(pp_array.vectorValueValue()); vector<xmlrpc_c::value> pp(pp_array.vectorValueValue());
@ -58,7 +55,7 @@ namespace MosesServer
vector<float> weight_vector = pdmm->MinimizePerplexity(phrase_pairs); vector<float> weight_vector = pdmm->MinimizePerplexity(phrase_pairs);
vector<xmlrpc_c::value> weight_vector_ret; vector<xmlrpc_c::value> weight_vector_ret;
for (size_t i=0;i < weight_vector.size();i++) for (size_t i=0; i < weight_vector.size(); i++)
weight_vector_ret.push_back(xmlrpc_c::value_double(weight_vector[i])); weight_vector_ret.push_back(xmlrpc_c::value_double(weight_vector[i]));
*retvalP = xmlrpc_c::value_array(weight_vector_ret); *retvalP = xmlrpc_c::value_array(weight_vector_ret);
@ -68,5 +65,5 @@ namespace MosesServer
std::cerr << errmsg << std::endl; std::cerr << errmsg << std::endl;
*retvalP = xmlrpc_c::value_string(errmsg); *retvalP = xmlrpc_c::value_string(errmsg);
#endif #endif
} }
} }

View File

@ -6,12 +6,12 @@
namespace MosesServer namespace MosesServer
{ {
class class
Optimizer : public xmlrpc_c::method Optimizer : public xmlrpc_c::method
{ {
public: public:
Optimizer(); Optimizer();
void execute(xmlrpc_c::paramList const& paramList, void execute(xmlrpc_c::paramList const& paramList,
xmlrpc_c::value * const retvalP); xmlrpc_c::value * const retvalP);
}; };
} }

View File

@ -3,40 +3,40 @@
namespace MosesServer namespace MosesServer
{ {
using namespace std; using namespace std;
using Moses::Hypothesis; using Moses::Hypothesis;
using Moses::StaticData; using Moses::StaticData;
using Moses::WordsRange; using Moses::WordsRange;
using Moses::ChartHypothesis; using Moses::ChartHypothesis;
using Moses::Phrase; using Moses::Phrase;
using Moses::Manager; using Moses::Manager;
using Moses::SearchGraphNode; using Moses::SearchGraphNode;
using Moses::TrellisPathList; using Moses::TrellisPathList;
using Moses::TranslationOptionCollection; using Moses::TranslationOptionCollection;
using Moses::TranslationOptionList; using Moses::TranslationOptionList;
using Moses::TranslationOption; using Moses::TranslationOption;
using Moses::TargetPhrase; using Moses::TargetPhrase;
using Moses::FValue; using Moses::FValue;
using Moses::PhraseDictionaryMultiModel; using Moses::PhraseDictionaryMultiModel;
using Moses::FindPhraseDictionary; using Moses::FindPhraseDictionary;
using Moses::Sentence; using Moses::Sentence;
boost::shared_ptr<TranslationRequest> boost::shared_ptr<TranslationRequest>
TranslationRequest:: TranslationRequest::
create(xmlrpc_c::paramList const& paramList, create(xmlrpc_c::paramList const& paramList,
boost::condition_variable& cond, boost::condition_variable& cond,
boost::mutex& mut) boost::mutex& mut)
{ {
boost::shared_ptr<TranslationRequest> ret; boost::shared_ptr<TranslationRequest> ret;
ret.reset(new TranslationRequest(paramList,cond, mut)); ret.reset(new TranslationRequest(paramList,cond, mut));
ret->m_self = ret; ret->m_self = ret;
return ret; return ret;
} }
void void
TranslationRequest:: TranslationRequest::
Run() Run()
{ {
parse_request(m_paramList.getStruct(0)); parse_request(m_paramList.getStruct(0));
Moses::StaticData const& SD = Moses::StaticData::Instance(); Moses::StaticData const& SD = Moses::StaticData::Instance();
@ -60,13 +60,13 @@ namespace MosesServer
} }
m_cond.notify_one(); m_cond.notify_one();
} }
/// add phrase alignment information from a Hypothesis /// add phrase alignment information from a Hypothesis
void void
TranslationRequest:: TranslationRequest::
add_phrase_aln_info(Hypothesis const& h, vector<xmlrpc_c::value>& aInfo) const add_phrase_aln_info(Hypothesis const& h, vector<xmlrpc_c::value>& aInfo) const
{ {
if (!m_withAlignInfo) return; if (!m_withAlignInfo) return;
WordsRange const& trg = h.GetCurrTargetWordsRange(); WordsRange const& trg = h.GetCurrTargetWordsRange();
WordsRange const& src = h.GetCurrSourceWordsRange(); WordsRange const& src = h.GetCurrSourceWordsRange();
@ -76,12 +76,12 @@ namespace MosesServer
pAlnInfo["src-start"] = xmlrpc_c::value_int(src.GetStartPos()); pAlnInfo["src-start"] = xmlrpc_c::value_int(src.GetStartPos());
pAlnInfo["src-end"] = xmlrpc_c::value_int(src.GetEndPos()); pAlnInfo["src-end"] = xmlrpc_c::value_int(src.GetEndPos());
aInfo.push_back(xmlrpc_c::value_struct(pAlnInfo)); aInfo.push_back(xmlrpc_c::value_struct(pAlnInfo));
} }
void void
TranslationRequest:: TranslationRequest::
outputChartHypo(ostream& out, const ChartHypothesis* hypo) outputChartHypo(ostream& out, const ChartHypothesis* hypo)
{ {
Phrase outPhrase(20); Phrase outPhrase(20);
hypo->GetOutputPhrase(outPhrase); hypo->GetOutputPhrase(outPhrase);
@ -91,18 +91,20 @@ namespace MosesServer
outPhrase.RemoveWord(outPhrase.GetSize() - 1); outPhrase.RemoveWord(outPhrase.GetSize() - 1);
for (size_t pos = 0 ; pos < outPhrase.GetSize() ; pos++) for (size_t pos = 0 ; pos < outPhrase.GetSize() ; pos++)
out << *outPhrase.GetFactor(pos, 0) << " "; out << *outPhrase.GetFactor(pos, 0) << " ";
} }
bool bool
TranslationRequest:: TranslationRequest::
compareSearchGraphNode(const Moses::SearchGraphNode& a, compareSearchGraphNode(const Moses::SearchGraphNode& a,
const Moses::SearchGraphNode& b) const Moses::SearchGraphNode& b)
{ return a.hypo->GetId() < b.hypo->GetId(); } {
return a.hypo->GetId() < b.hypo->GetId();
}
void void
TranslationRequest:: TranslationRequest::
insertGraphInfo(Manager& manager, map<string, xmlrpc_c::value>& retData) insertGraphInfo(Manager& manager, map<string, xmlrpc_c::value>& retData)
{ {
using xmlrpc_c::value_int; using xmlrpc_c::value_int;
using xmlrpc_c::value_double; using xmlrpc_c::value_double;
using xmlrpc_c::value_struct; using xmlrpc_c::value_struct;
@ -111,16 +113,14 @@ namespace MosesServer
vector<SearchGraphNode> searchGraph; vector<SearchGraphNode> searchGraph;
manager.GetSearchGraph(searchGraph); manager.GetSearchGraph(searchGraph);
std::sort(searchGraph.begin(), searchGraph.end()); std::sort(searchGraph.begin(), searchGraph.end());
BOOST_FOREACH(Moses::SearchGraphNode const& n, searchGraph) BOOST_FOREACH(Moses::SearchGraphNode const& n, searchGraph) {
{
map<string, xmlrpc_c::value> x; // search graph xml node map<string, xmlrpc_c::value> x; // search graph xml node
x["forward"] = value_double(n.forward); x["forward"] = value_double(n.forward);
x["fscore"] = value_double(n.fscore); x["fscore"] = value_double(n.fscore);
const Hypothesis* hypo = n.hypo; const Hypothesis* hypo = n.hypo;
x["hyp"] = value_int(hypo->GetId()); x["hyp"] = value_int(hypo->GetId());
x["stack"] = value_int(hypo->GetWordsBitmap().GetNumWordsCovered()); x["stack"] = value_int(hypo->GetWordsBitmap().GetNumWordsCovered());
if (hypo->GetId() != 0) if (hypo->GetId() != 0) {
{
const Hypothesis *prevHypo = hypo->GetPrevHypo(); const Hypothesis *prevHypo = hypo->GetPrevHypo();
x["back"] = value_int(prevHypo->GetId()); x["back"] = value_int(prevHypo->GetId());
x["score"] = value_double(hypo->GetScore()); x["score"] = value_double(hypo->GetScore());
@ -134,36 +134,32 @@ namespace MosesServer
searchGraphXml.push_back(value_struct(x)); searchGraphXml.push_back(value_struct(x));
} }
retData["sg"] = xmlrpc_c::value_array(searchGraphXml); retData["sg"] = xmlrpc_c::value_array(searchGraphXml);
} }
void void
TranslationRequest:: TranslationRequest::
output_phrase(ostream& out, Phrase const& phrase) const output_phrase(ostream& out, Phrase const& phrase) const
{ {
if (!m_reportAllFactors) if (!m_reportAllFactors) {
{
for (size_t i = 0 ; i < phrase.GetSize(); ++i) for (size_t i = 0 ; i < phrase.GetSize(); ++i)
out << *phrase.GetFactor(i, 0) << " "; out << *phrase.GetFactor(i, 0) << " ";
} } else out << phrase;
else out << phrase; }
}
void void
TranslationRequest:: TranslationRequest::
outputNBest(const Manager& manager, map<string, xmlrpc_c::value>& retData) outputNBest(const Manager& manager, map<string, xmlrpc_c::value>& retData)
{ {
TrellisPathList nBestList; TrellisPathList nBestList;
vector<xmlrpc_c::value> nBestXml; vector<xmlrpc_c::value> nBestXml;
manager.CalcNBest(m_nbestSize, nBestList, m_nbestDistinct); manager.CalcNBest(m_nbestSize, nBestList, m_nbestDistinct);
BOOST_FOREACH(Moses::TrellisPath const* path, nBestList) BOOST_FOREACH(Moses::TrellisPath const* path, nBestList) {
{
vector<const Hypothesis *> const& E = path->GetEdges(); vector<const Hypothesis *> const& E = path->GetEdges();
if (!E.size()) continue; if (!E.size()) continue;
std::map<std::string, xmlrpc_c::value> nBestXmlItem; std::map<std::string, xmlrpc_c::value> nBestXmlItem;
pack_hypothesis(E, "hyp", nBestXmlItem); pack_hypothesis(E, "hyp", nBestXmlItem);
if (m_withScoreBreakdown) if (m_withScoreBreakdown) {
{
// should the score breakdown be reported in a more structured manner? // should the score breakdown be reported in a more structured manner?
ostringstream buf; ostringstream buf;
path->GetScoreBreakdown()->OutputAllFeatureScores(buf); path->GetScoreBreakdown()->OutputAllFeatureScores(buf);
@ -175,26 +171,23 @@ namespace MosesServer
nBestXml.push_back(xmlrpc_c::value_struct(nBestXmlItem)); nBestXml.push_back(xmlrpc_c::value_struct(nBestXmlItem));
} }
retData["nbest"] = xmlrpc_c::value_array(nBestXml); retData["nbest"] = xmlrpc_c::value_array(nBestXml);
} }
void void
TranslationRequest:: TranslationRequest::
insertTranslationOptions(Moses::Manager& manager, insertTranslationOptions(Moses::Manager& manager,
std::map<std::string, xmlrpc_c::value>& retData) std::map<std::string, xmlrpc_c::value>& retData)
{ {
const TranslationOptionCollection* toptsColl const TranslationOptionCollection* toptsColl
= manager.getSntTranslationOptions(); = manager.getSntTranslationOptions();
vector<xmlrpc_c::value> toptsXml; vector<xmlrpc_c::value> toptsXml;
size_t const stop = toptsColl->GetSource().GetSize(); size_t const stop = toptsColl->GetSource().GetSize();
TranslationOptionList const* tol; TranslationOptionList const* tol;
for (size_t s = 0 ; s < stop ; ++s) for (size_t s = 0 ; s < stop ; ++s) {
{
for (size_t e = s; for (size_t e = s;
(tol = toptsColl->GetTranslationOptionList(s,e)) != NULL; (tol = toptsColl->GetTranslationOptionList(s,e)) != NULL;
++e) ++e) {
{ BOOST_FOREACH(TranslationOption const* topt, *tol) {
BOOST_FOREACH(TranslationOption const* topt, *tol)
{
std::map<std::string, xmlrpc_c::value> toptXml; std::map<std::string, xmlrpc_c::value> toptXml;
TargetPhrase const& tp = topt->GetTargetPhrase(); TargetPhrase const& tp = topt->GetTargetPhrase();
StaticData const& GLOBAL = StaticData::Instance(); StaticData const& GLOBAL = StaticData::Instance();
@ -215,25 +208,26 @@ namespace MosesServer
} }
} }
retData["topt"] = xmlrpc_c::value_array(toptsXml); retData["topt"] = xmlrpc_c::value_array(toptsXml);
} }
bool bool
check(std::map<std::string, xmlrpc_c::value> const& params, std::string const key) check(std::map<std::string, xmlrpc_c::value> const& params, std::string const key)
{ {
std::map<std::string, xmlrpc_c::value>::const_iterator m; std::map<std::string, xmlrpc_c::value>::const_iterator m;
return (params.find(key) != params.end()); return (params.find(key) != params.end());
} }
TranslationRequest:: TranslationRequest::
TranslationRequest(xmlrpc_c::paramList const& paramList, TranslationRequest(xmlrpc_c::paramList const& paramList,
boost::condition_variable& cond, boost::mutex& mut) boost::condition_variable& cond, boost::mutex& mut)
: m_cond(cond), m_mutex(mut), m_done(false), m_paramList(paramList) : m_cond(cond), m_mutex(mut), m_done(false), m_paramList(paramList)
{ } { }
void void
TranslationRequest:: TranslationRequest::
parse_request(std::map<std::string, xmlrpc_c::value> const& params) parse_request(std::map<std::string, xmlrpc_c::value> const& params)
{ // parse XMLRPC request {
// parse XMLRPC request
// params_t const params = m_paramList.getStruct(0); // params_t const params = m_paramList.getStruct(0);
m_paramList.verifyEnd(1); // ??? UG m_paramList.verifyEnd(1); // ??? UG
@ -254,16 +248,14 @@ namespace MosesServer
m_withScoreBreakdown = check(params, "add-score-breakdown"); m_withScoreBreakdown = check(params, "add-score-breakdown");
m_source.reset(new Sentence(0,m_source_string)); m_source.reset(new Sentence(0,m_source_string));
si = params.find("lambda"); si = params.find("lambda");
if (si != params.end()) if (si != params.end()) {
{
// muMo = multiModel // muMo = multiModel
xmlrpc_c::value_array muMoArray = xmlrpc_c::value_array(si->second); xmlrpc_c::value_array muMoArray = xmlrpc_c::value_array(si->second);
vector<xmlrpc_c::value> muMoValVec(muMoArray.vectorValueValue()); vector<xmlrpc_c::value> muMoValVec(muMoArray.vectorValueValue());
vector<float> w(muMoValVec.size()); vector<float> w(muMoValVec.size());
for (size_t i = 0; i < muMoValVec.size(); ++i) for (size_t i = 0; i < muMoValVec.size(); ++i)
w[i] = xmlrpc_c::value_double(muMoValVec[i]); w[i] = xmlrpc_c::value_double(muMoValVec[i]);
if (w.size() && (si = params.find("model_name")) != params.end()) if (w.size() && (si = params.find("model_name")) != params.end()) {
{
string const model_name = xmlrpc_c::value_string(si->second); string const model_name = xmlrpc_c::value_string(si->second);
PhraseDictionaryMultiModel* pdmm PhraseDictionaryMultiModel* pdmm
= (PhraseDictionaryMultiModel*) FindPhraseDictionary(model_name); = (PhraseDictionaryMultiModel*) FindPhraseDictionary(model_name);
@ -281,13 +273,13 @@ namespace MosesServer
// for (size_t i = 1; i < tmp.size(); i += 2) // for (size_t i = 1; i < tmp.size(); i += 2)
// m_bias[xmlrpc_c::value_int(tmp[i-1])] = xmlrpc_c::value_double(tmp[i]); // m_bias[xmlrpc_c::value_int(tmp[i-1])] = xmlrpc_c::value_double(tmp[i]);
// } // }
} // end of Translationtask::parse_request() } // end of Translationtask::parse_request()
void void
TranslationRequest:: TranslationRequest::
run_chart_decoder() run_chart_decoder()
{ {
Moses::TreeInput tinput; Moses::TreeInput tinput;
istringstream buf(m_source_string + "\n"); istringstream buf(m_source_string + "\n");
tinput.Read(buf, StaticData::Instance().GetInputFactorOrder()); tinput.Read(buf, StaticData::Instance().GetInputFactorOrder());
@ -302,27 +294,26 @@ namespace MosesServer
m_target_string = out.str(); m_target_string = out.str();
m_retData["text"] = xmlrpc_c::value_string(m_target_string); m_retData["text"] = xmlrpc_c::value_string(m_target_string);
if (m_withGraphInfo) if (m_withGraphInfo) {
{
std::ostringstream sgstream; std::ostringstream sgstream;
manager.OutputSearchGraphMoses(sgstream); manager.OutputSearchGraphMoses(sgstream);
m_retData["sg"] = xmlrpc_c::value_string(sgstream.str()); m_retData["sg"] = xmlrpc_c::value_string(sgstream.str());
} }
} // end of TranslationRequest::run_chart_decoder() } // end of TranslationRequest::run_chart_decoder()
void void
TranslationRequest:: TranslationRequest::
pack_hypothesis(vector<Hypothesis const* > const& edges, string const& key, pack_hypothesis(vector<Hypothesis const* > const& edges, string const& key,
map<string, xmlrpc_c::value> & dest) const map<string, xmlrpc_c::value> & dest) const
{ {
// target string // target string
ostringstream target; ostringstream target;
BOOST_REVERSE_FOREACH(Hypothesis const* e, edges) BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
output_phrase(target, e->GetCurrTargetPhrase()); output_phrase(target, e->GetCurrTargetPhrase());
dest[key] = xmlrpc_c::value_string(target.str()); dest[key] = xmlrpc_c::value_string(target.str());
if (m_withAlignInfo) if (m_withAlignInfo) {
{ // phrase alignment, if requested // phrase alignment, if requested
vector<xmlrpc_c::value> p_aln; vector<xmlrpc_c::value> p_aln;
BOOST_REVERSE_FOREACH(Hypothesis const* e, edges) BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
@ -330,32 +321,32 @@ namespace MosesServer
dest["align"] = xmlrpc_c::value_array(p_aln); dest["align"] = xmlrpc_c::value_array(p_aln);
} }
if (m_withWordAlignInfo) if (m_withWordAlignInfo) {
{ // word alignment, if requested // word alignment, if requested
vector<xmlrpc_c::value> w_aln; vector<xmlrpc_c::value> w_aln;
BOOST_FOREACH(Hypothesis const* e, edges) BOOST_FOREACH(Hypothesis const* e, edges)
e->OutputLocalWordAlignment(w_aln); e->OutputLocalWordAlignment(w_aln);
dest["word-align"] = xmlrpc_c::value_array(w_aln); dest["word-align"] = xmlrpc_c::value_array(w_aln);
} }
} }
void void
TranslationRequest:: TranslationRequest::
pack_hypothesis(Hypothesis const* h, string const& key, pack_hypothesis(Hypothesis const* h, string const& key,
map<string, xmlrpc_c::value>& dest) const map<string, xmlrpc_c::value>& dest) const
{ {
using namespace std; using namespace std;
vector<Hypothesis const*> edges; vector<Hypothesis const*> edges;
for (;h; h = h->GetPrevHypo()) for (; h; h = h->GetPrevHypo())
edges.push_back(h); edges.push_back(h);
pack_hypothesis(edges, key, dest); pack_hypothesis(edges, key, dest);
} }
void void
TranslationRequest:: TranslationRequest::
run_phrase_decoder() run_phrase_decoder()
{ {
Manager manager(this->self()); Manager manager(this->self());
// if (m_bias.size()) manager.SetBias(&m_bias); // if (m_bias.size()) manager.SetBias(&m_bias);
manager.Decode(); manager.Decode();
@ -370,5 +361,5 @@ namespace MosesServer
.SetOutputSearchGraph(false); .SetOutputSearchGraph(false);
// WTF? one more reason not to have this as global variable! --- UG // WTF? one more reason not to have this as global variable! --- UG
} }
} }

View File

@ -23,9 +23,9 @@
#include <xmlrpc-c/base.hpp> #include <xmlrpc-c/base.hpp>
namespace MosesServer namespace MosesServer
{ {
class class
TranslationRequest : public virtual Moses::TranslationTask TranslationRequest : public virtual Moses::TranslationTask
{ {
boost::condition_variable& m_cond; boost::condition_variable& m_cond;
boost::mutex& m_mutex; boost::mutex& m_mutex;
bool m_done; bool m_done;
@ -90,12 +90,12 @@ namespace MosesServer
void void
insertTranslationOptions(Moses::Manager& manager, insertTranslationOptions(Moses::Manager& manager,
std::map<std::string, xmlrpc_c::value>& retData); std::map<std::string, xmlrpc_c::value>& retData);
protected: protected:
TranslationRequest(xmlrpc_c::paramList const& paramList, TranslationRequest(xmlrpc_c::paramList const& paramList,
boost::condition_variable& cond, boost::condition_variable& cond,
boost::mutex& mut); boost::mutex& mut);
public: public:
static static
boost::shared_ptr<TranslationRequest> boost::shared_ptr<TranslationRequest>
@ -105,18 +105,24 @@ namespace MosesServer
virtual bool virtual bool
DeleteAfterExecution() { return false; } DeleteAfterExecution() {
return false;
}
bool bool
IsDone() const { return m_done; } IsDone() const {
return m_done;
}
std::map<std::string, xmlrpc_c::value> const& std::map<std::string, xmlrpc_c::value> const&
GetRetData() { return m_retData; } GetRetData() {
return m_retData;
}
void void
Run(); Run();
}; };
} }

View File

@ -4,25 +4,25 @@
namespace MosesServer namespace MosesServer
{ {
using namespace std; using namespace std;
using namespace Moses; using namespace Moses;
Translator:: Translator::
Translator(size_t numThreads) Translator(size_t numThreads)
: m_threadPool(numThreads) : m_threadPool(numThreads)
{ {
// signature and help strings are documentation -- the client // signature and help strings are documentation -- the client
// can query this information with a system.methodSignature and // can query this information with a system.methodSignature and
// system.methodHelp RPC. // system.methodHelp RPC.
this->_signature = "S:S"; this->_signature = "S:S";
this->_help = "Does translation"; this->_help = "Does translation";
} }
void void
Translator:: Translator::
execute(xmlrpc_c::paramList const& paramList, execute(xmlrpc_c::paramList const& paramList,
xmlrpc_c::value * const retvalP) xmlrpc_c::value * const retvalP)
{ {
boost::condition_variable cond; boost::condition_variable cond;
boost::mutex mut; boost::mutex mut;
boost::shared_ptr<TranslationRequest> task boost::shared_ptr<TranslationRequest> task
@ -32,6 +32,6 @@ namespace MosesServer
while (!task->IsDone()) while (!task->IsDone())
cond.wait(lock); cond.wait(lock);
*retvalP = xmlrpc_c::value_struct(task->GetRetData()); *retvalP = xmlrpc_c::value_struct(task->GetRetData());
} }
} }

View File

@ -10,17 +10,17 @@
#endif #endif
namespace MosesServer namespace MosesServer
{ {
class class
// MosesServer:: // MosesServer::
Translator : public xmlrpc_c::method Translator : public xmlrpc_c::method
{ {
public: public:
Translator(size_t numThreads = 10); Translator(size_t numThreads = 10);
void execute(xmlrpc_c::paramList const& paramList, void execute(xmlrpc_c::paramList const& paramList,
xmlrpc_c::value * const retvalP); xmlrpc_c::value * const retvalP);
private: private:
Moses::ThreadPool m_threadPool; Moses::ThreadPool m_threadPool;
}; };
} }

View File

@ -2,24 +2,24 @@
namespace MosesServer namespace MosesServer
{ {
using namespace Moses; using namespace Moses;
using namespace std; using namespace std;
Updater:: Updater::
Updater() Updater()
{ {
// signature and help strings are documentation -- the client // signature and help strings are documentation -- the client
// can query this information with a system.methodSignature and // can query this information with a system.methodSignature and
// system.methodHelp RPC. // system.methodHelp RPC.
this->_signature = "S:S"; this->_signature = "S:S";
this->_help = "Updates stuff"; this->_help = "Updates stuff";
} }
void void
Updater:: Updater::
execute(xmlrpc_c::paramList const& paramList, execute(xmlrpc_c::paramList const& paramList,
xmlrpc_c::value * const retvalP) xmlrpc_c::value * const retvalP)
{ {
#if PT_UG #if PT_UG
const params_t params = paramList.getStruct(0); const params_t params = paramList.getStruct(0);
breakOutParams(params); breakOutParams(params);
@ -28,12 +28,12 @@ namespace MosesServer
XVERBOSE(1,"Done inserting\n"); XVERBOSE(1,"Done inserting\n");
*retvalP = xmlrpc_c::value_string("Phrase table updated"); *retvalP = xmlrpc_c::value_string("Phrase table updated");
#endif #endif
}; };
void void
Updater:: Updater::
breakOutParams(const params_t& params) breakOutParams(const params_t& params)
{ {
params_t::const_iterator si = params.find("source"); params_t::const_iterator si = params.find("source");
if(si == params.end()) if(si == params.end())
throw xmlrpc_c::fault("Missing source sentence", throw xmlrpc_c::fault("Missing source sentence",
@ -52,6 +52,6 @@ namespace MosesServer
XVERBOSE(1,"alignment = " << m_aln << endl); XVERBOSE(1,"alignment = " << m_aln << endl);
m_bounded = ((si = params.find("bounded")) != params.end()); m_bounded = ((si = params.find("bounded")) != params.end());
m_add2ORLM = ((si = params.find("updateORLM")) != params.end()); m_add2ORLM = ((si = params.find("updateORLM")) != params.end());
}; };
} }

View File

@ -19,9 +19,9 @@
namespace MosesServer namespace MosesServer
{ {
class class
Updater: public xmlrpc_c::method Updater: public xmlrpc_c::method
{ {
typedef std::map<std::string, xmlrpc_c::value> params_t; typedef std::map<std::string, xmlrpc_c::value> params_t;
@ -29,7 +29,7 @@ namespace MosesServer
std::string m_src, m_trg, m_aln; std::string m_src, m_trg, m_aln;
bool m_bounded, m_add2ORLM; bool m_bounded, m_add2ORLM;
public: public:
Updater(); Updater();
void void
@ -39,6 +39,6 @@ namespace MosesServer
void void
breakOutParams(const params_t& params); breakOutParams(const params_t& params);
}; };
} }

View File

@ -18,20 +18,20 @@
namespace Moses namespace Moses
{ {
// todo: replace this with thread lock-free containers, if a stable library can // todo: replace this with thread lock-free containers, if a stable library can
// be found somewhere // be found somewhere
template<typename KEY, typename VAL, class CONTAINER = std::map<KEY,VAL> > template<typename KEY, typename VAL, class CONTAINER = std::map<KEY,VAL> >
class class
ThreadSafeContainer ThreadSafeContainer
{ {
protected: protected:
mutable boost::shared_mutex m_lock; mutable boost::shared_mutex m_lock;
CONTAINER m_container; CONTAINER m_container;
typedef typename CONTAINER::iterator iter_t; typedef typename CONTAINER::iterator iter_t;
typedef typename CONTAINER::const_iterator const_iter_t; typedef typename CONTAINER::const_iterator const_iter_t;
typedef typename CONTAINER::value_type entry_t; typedef typename CONTAINER::value_type entry_t;
public: public:
class locking_iterator class locking_iterator
{ {
@ -49,8 +49,7 @@ namespace Moses
: m_lock(lock), m_container(container), m_iter(iter) : m_lock(lock), m_container(container), m_iter(iter)
{ } { }
entry_t const& operator->() entry_t const& operator->() {
{
UTIL_THROW_IF2(m_container == NULL, "This locking iterator is invalid " UTIL_THROW_IF2(m_container == NULL, "This locking iterator is invalid "
<< "or has not been assigned."); << "or has not been assigned.");
return m_iter.operator->(); return m_iter.operator->();
@ -58,21 +57,22 @@ namespace Moses
// locking operators transfer the lock upon assignment and become invalid // locking operators transfer the lock upon assignment and become invalid
locking_iterator const& locking_iterator const&
operator=(locking_iterator& other) operator=(locking_iterator& other) {
{
m_lock.swap(other.m_lock); m_lock.swap(other.m_lock);
m_iter = other.m_iter; m_iter = other.m_iter;
other.m_iter = other.m_container.end(); other.m_iter = other.m_container.end();
} }
bool bool
operator==(const_iter_t const& other) operator==(const_iter_t const& other) {
{
return m_iter == other; return m_iter == other;
} }
locking_iterator const& locking_iterator const&
operator++() { ++m_iter; return *this; } operator++() {
++m_iter;
return *this;
}
// DO NOT DEFINE THE POST-INCREMENT OPERATOR! // DO NOT DEFINE THE POST-INCREMENT OPERATOR!
// locking_operators are non-copyable, // locking_operators are non-copyable,
@ -82,16 +82,15 @@ namespace Moses
operator++(int); operator++(int);
}; };
const_iter_t const& end() const const_iter_t const& end() const {
{ return m_container.end(); } return m_container.end();
}
locking_iterator begin() const locking_iterator begin() const {
{
return locking_iterator(m_lock, this, m_container.begin()); return locking_iterator(m_lock, this, m_container.begin());
} }
VAL const& set(KEY const& key, VAL const& val) VAL const& set(KEY const& key, VAL const& val) {
{
boost::unique_lock< boost::shared_mutex > lock(m_lock); boost::unique_lock< boost::shared_mutex > lock(m_lock);
entry_t entry(key,val); entry_t entry(key,val);
iter_t foo = m_container.insert(entry).first; iter_t foo = m_container.insert(entry).first;
@ -99,27 +98,24 @@ namespace Moses
return foo->second; return foo->second;
} }
VAL const* get(KEY const& key, VAL const& default_val) VAL const* get(KEY const& key, VAL const& default_val) {
{
boost::shared_lock< boost::shared_mutex > lock(m_lock); boost::shared_lock< boost::shared_mutex > lock(m_lock);
entry_t entry(key, default_val); entry_t entry(key, default_val);
iter_t foo = m_container.insert(entry).first; iter_t foo = m_container.insert(entry).first;
return &(foo->second); return &(foo->second);
} }
VAL const* get(KEY const& key) const VAL const* get(KEY const& key) const {
{
boost::shared_lock< boost::shared_mutex > lock(m_lock); boost::shared_lock< boost::shared_mutex > lock(m_lock);
const_iter_t m = m_container.find(key); const_iter_t m = m_container.find(key);
if (m == m_container.end()) return NULL; if (m == m_container.end()) return NULL;
return &m->second; return &m->second;
} }
size_t erase(KEY const& key) size_t erase(KEY const& key) {
{
boost::unique_lock< boost::shared_mutex > lock(m_lock); boost::unique_lock< boost::shared_mutex > lock(m_lock);
return m_container.erase(key); return m_container.erase(key);
} }
}; };
} }
#endif #endif

View File

@ -116,7 +116,7 @@ void PropertiesConsolidator::ProcessPropertiesString(const std::string &properti
} else if ( !keyValue[0].compare("POS") ) { } else if ( !keyValue[0].compare("POS") ) {
/* DO NOTHING (property is not registered in the decoder at the moment) /* DO NOTHING (property is not registered in the decoder at the moment)
if ( m_partsOfSpeechFlag ) { if ( m_partsOfSpeechFlag ) {
// POS property: replace strings with vocabulary indices // POS property: replace strings with vocabulary indices
@ -127,7 +127,7 @@ void PropertiesConsolidator::ProcessPropertiesString(const std::string &properti
} else { // don't process POS property } else { // don't process POS property
out << " {{" << keyValue[0] << " " << keyValue[1] << "}}"; out << " {{" << keyValue[0] << " " << keyValue[1] << "}}";
} }
*/ */
} else { } else {

View File

@ -699,12 +699,12 @@ void ExtractGHKM::WriteGlueGrammar(
// const size_t partOfSpeechSentenceStart = 0; // const size_t partOfSpeechSentenceStart = 0;
// const size_t partOfSpeechSentenceEnd = 1; // const size_t partOfSpeechSentenceEnd = 1;
#ifndef BOS_ #ifndef BOS_
#define BOS_ "<s>" //Beginning of sentence symbol #define BOS_ "<s>" //Beginning of sentence symbol
#endif #endif
#ifndef EOS_ #ifndef EOS_
#define EOS_ "</s>" //End of sentence symbol #define EOS_ "</s>" //End of sentence symbol
#endif #endif
std::string sentenceStartSource = BOS_; std::string sentenceStartSource = BOS_;
std::string sentenceEndSource = EOS_; std::string sentenceEndSource = EOS_;

View File

@ -25,8 +25,9 @@ namespace FilterRuleTable
// Filters a rule table, discarding rules that cannot be applied to a given // Filters a rule table, discarding rules that cannot be applied to a given
// test set. The rule table must have a TSG source-side and the test sentences // test set. The rule table must have a TSG source-side and the test sentences
// must be parse trees. // must be parse trees.
class TreeCfgFilter : public CfgFilter { class TreeCfgFilter : public CfgFilter
public: {
public:
// Initialize the filter for a given set of test sentences. // Initialize the filter for a given set of test sentences.
TreeCfgFilter(const std::vector<boost::shared_ptr<StringTree> > &); TreeCfgFilter(const std::vector<boost::shared_ptr<StringTree> > &);

View File

@ -15,7 +15,7 @@ namespace PostprocessEgretForests
class Forest class Forest
{ {
public: public:
struct Vertex; struct Vertex;
struct Hyperedge { struct Hyperedge {
@ -35,7 +35,7 @@ class Forest
std::vector<boost::shared_ptr<Vertex> > vertices; std::vector<boost::shared_ptr<Vertex> > vertices;
private: private:
// Copying is not allowed. // Copying is not allowed.
Forest(const Forest &); Forest(const Forest &);
Forest &operator=(const Forest &); Forest &operator=(const Forest &);

View File

@ -17,15 +17,18 @@ namespace PostprocessEgretForests
{ {
ForestParser::ForestParser() ForestParser::ForestParser()
: m_input(0) { : m_input(0)
{
} }
ForestParser::ForestParser(std::istream &input) ForestParser::ForestParser(std::istream &input)
: m_input(&input) { : m_input(&input)
{
++(*this); ++(*this);
} }
ForestParser &ForestParser::operator++() { ForestParser &ForestParser::operator++()
{
if (!m_input) { if (!m_input) {
return *this; return *this;
} }
@ -132,12 +135,14 @@ boost::shared_ptr<Forest::Vertex> ForestParser::ParseVertex(
return v; return v;
} }
bool operator==(const ForestParser &lhs, const ForestParser &rhs) { bool operator==(const ForestParser &lhs, const ForestParser &rhs)
{
// TODO Is this right? Compare values of istreams if non-zero? // TODO Is this right? Compare values of istreams if non-zero?
return lhs.m_input == rhs.m_input; return lhs.m_input == rhs.m_input;
} }
bool operator!=(const ForestParser &lhs, const ForestParser &rhs) { bool operator!=(const ForestParser &lhs, const ForestParser &rhs)
{
return !(lhs == rhs); return !(lhs == rhs);
} }

View File

@ -20,8 +20,9 @@ namespace Syntax
namespace PostprocessEgretForests namespace PostprocessEgretForests
{ {
class ForestParser { class ForestParser
public: {
public:
struct Entry { struct Entry {
std::size_t sentNum; std::size_t sentNum;
std::string sentence; std::string sentence;
@ -31,15 +32,19 @@ class ForestParser {
ForestParser(); ForestParser();
ForestParser(std::istream &); ForestParser(std::istream &);
Entry &operator*() { return m_entry; } Entry &operator*() {
Entry *operator->() { return &m_entry; } return m_entry;
}
Entry *operator->() {
return &m_entry;
}
ForestParser &operator++(); ForestParser &operator++();
friend bool operator==(const ForestParser &, const ForestParser &); friend bool operator==(const ForestParser &, const ForestParser &);
friend bool operator!=(const ForestParser &, const ForestParser &); friend bool operator!=(const ForestParser &, const ForestParser &);
private: private:
typedef boost::shared_ptr<Forest::Vertex> VertexSP; typedef boost::shared_ptr<Forest::Vertex> VertexSP;
typedef boost::shared_ptr<Forest::Hyperedge> HyperedgeSP; typedef boost::shared_ptr<Forest::Hyperedge> HyperedgeSP;

View File

@ -61,7 +61,8 @@ void ForestWriter::WriteVertex(const Forest::Vertex &v)
} }
} }
std::string ForestWriter::PossiblyEscape(const std::string &s) const { std::string ForestWriter::PossiblyEscape(const std::string &s) const
{
if (m_options.escape) { if (m_options.escape) {
return Escape(s); return Escape(s);
} else { } else {
@ -70,7 +71,8 @@ std::string ForestWriter::PossiblyEscape(const std::string &s) const {
} }
// Escapes XML special characters. // Escapes XML special characters.
std::string ForestWriter::Escape(const std::string &s) const { std::string ForestWriter::Escape(const std::string &s) const
{
std::string t; std::string t;
std::size_t len = s.size(); std::size_t len = s.size();
t.reserve(len); t.reserve(len);

View File

@ -15,13 +15,13 @@ namespace PostprocessEgretForests
class ForestWriter class ForestWriter
{ {
public: public:
ForestWriter(const Options &options, std::ostream &out) ForestWriter(const Options &options, std::ostream &out)
: m_options(options), m_out(out) {} : m_options(options), m_out(out) {}
void Write(const std::string &, const Forest &, std::size_t); void Write(const std::string &, const Forest &, std::size_t);
private: private:
std::string Escape(const std::string &) const; std::string Escape(const std::string &) const;
std::string PossiblyEscape(const std::string &) const; std::string PossiblyEscape(const std::string &) const;
void WriteHyperedgeLine(const Forest::Hyperedge &); void WriteHyperedgeLine(const Forest::Hyperedge &);

View File

@ -16,15 +16,18 @@ namespace PostprocessEgretForests
{ {
SplitPointFileParser::SplitPointFileParser() SplitPointFileParser::SplitPointFileParser()
: m_input(0) { : m_input(0)
{
} }
SplitPointFileParser::SplitPointFileParser(std::istream &input) SplitPointFileParser::SplitPointFileParser(std::istream &input)
: m_input(&input) { : m_input(&input)
{
++(*this); ++(*this);
} }
SplitPointFileParser &SplitPointFileParser::operator++() { SplitPointFileParser &SplitPointFileParser::operator++()
{
if (!m_input) { if (!m_input) {
return *this; return *this;
} }
@ -66,13 +69,15 @@ void SplitPointFileParser::ParseLine(const std::string &line,
} }
bool operator==(const SplitPointFileParser &lhs, bool operator==(const SplitPointFileParser &lhs,
const SplitPointFileParser &rhs) { const SplitPointFileParser &rhs)
{
// TODO Is this right? Compare values of istreams if non-zero? // TODO Is this right? Compare values of istreams if non-zero?
return lhs.m_input == rhs.m_input; return lhs.m_input == rhs.m_input;
} }
bool operator!=(const SplitPointFileParser &lhs, bool operator!=(const SplitPointFileParser &lhs,
const SplitPointFileParser &rhs) { const SplitPointFileParser &rhs)
{
return !(lhs == rhs); return !(lhs == rhs);
} }

View File

@ -13,8 +13,9 @@ namespace Syntax
namespace PostprocessEgretForests namespace PostprocessEgretForests
{ {
class SplitPointFileParser { class SplitPointFileParser
public: {
public:
struct Entry { struct Entry {
std::vector<SplitPoint> splitPoints; std::vector<SplitPoint> splitPoints;
}; };
@ -22,8 +23,12 @@ class SplitPointFileParser {
SplitPointFileParser(); SplitPointFileParser();
SplitPointFileParser(std::istream &); SplitPointFileParser(std::istream &);
const Entry &operator*() const { return m_entry; } const Entry &operator*() const {
const Entry *operator->() const { return &m_entry; } return m_entry;
}
const Entry *operator->() const {
return &m_entry;
}
SplitPointFileParser &operator++(); SplitPointFileParser &operator++();
@ -33,7 +38,7 @@ class SplitPointFileParser {
friend bool operator!=(const SplitPointFileParser &, friend bool operator!=(const SplitPointFileParser &,
const SplitPointFileParser &); const SplitPointFileParser &);
private: private:
void ParseLine(const std::string &, std::vector<SplitPoint> &); void ParseLine(const std::string &, std::vector<SplitPoint> &);
Entry m_entry; Entry m_entry;

View File

@ -30,14 +30,14 @@ inline bool operator==(const Symbol &s, const Symbol &t)
} }
struct SymbolHasher { struct SymbolHasher {
public: public:
std::size_t operator()(const Symbol &s) const { std::size_t operator()(const Symbol &s) const {
return hash_value(s); return hash_value(s);
} }
}; };
struct SymbolEqualityPred { struct SymbolEqualityPred {
public: public:
bool operator()(const Symbol &s, const Symbol &t) const { bool operator()(const Symbol &s, const Symbol &t) const {
return s.value == t.value && s.isNonTerminal == t.isNonTerminal; return s.value == t.value && s.isNonTerminal == t.isNonTerminal;
} }

View File

@ -16,10 +16,10 @@ namespace PostprocessEgretForests
class TopologicalSorter class TopologicalSorter
{ {
public: public:
void Sort(const Forest &, std::vector<const Forest::Vertex *> &); void Sort(const Forest &, std::vector<const Forest::Vertex *> &);
private: private:
typedef boost::unordered_set<const Forest::Vertex *> VertexSet; typedef boost::unordered_set<const Forest::Vertex *> VertexSet;
void BuildPredSets(const Forest &); void BuildPredSets(const Forest &);