mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-08-17 07:20:48 +03:00
beautify
This commit is contained in:
parent
a4a7c14593
commit
cc8c6b7b10
@ -94,8 +94,7 @@ void BleuScorer::setReferenceFiles(const vector<string>& referenceFiles)
|
||||
mert::VocabularyFactory::GetVocabulary()->clear();
|
||||
|
||||
//load reference data
|
||||
for (size_t i = 0; i < referenceFiles.size(); ++i)
|
||||
{
|
||||
for (size_t i = 0; i < referenceFiles.size(); ++i) {
|
||||
TRACE_ERR("Loading reference from " << referenceFiles[i] << endl);
|
||||
|
||||
ifstream ifs(referenceFiles[i].c_str());
|
||||
@ -153,8 +152,7 @@ void BleuScorer::ProcessReferenceLine(const std::string& line, Reference* ref) c
|
||||
|
||||
bool BleuScorer::GetNextReferenceFromStreams(std::vector<boost::shared_ptr<std::ifstream> >& referenceStreams, Reference& ref) const
|
||||
{
|
||||
for (vector<boost::shared_ptr<ifstream> >::iterator ifs=referenceStreams.begin(); ifs!=referenceStreams.end(); ++ifs)
|
||||
{
|
||||
for (vector<boost::shared_ptr<ifstream> >::iterator ifs=referenceStreams.begin(); ifs!=referenceStreams.end(); ++ifs) {
|
||||
if (!(*ifs)) return false;
|
||||
string line;
|
||||
if (!getline(**ifs, line)) return false;
|
||||
@ -309,8 +307,7 @@ vector<float> BleuScorer::ScoreNbestList(const string& scoreFile, const string&
|
||||
|
||||
vector<FeatureDataIterator> featureDataIters;
|
||||
vector<ScoreDataIterator> scoreDataIters;
|
||||
for (size_t i = 0; i < featureFiles.size(); ++i)
|
||||
{
|
||||
for (size_t i = 0; i < featureFiles.size(); ++i) {
|
||||
featureDataIters.push_back(FeatureDataIterator(featureFiles[i]));
|
||||
scoreDataIters.push_back(ScoreDataIterator(scoreFiles[i]));
|
||||
}
|
||||
@ -318,8 +315,7 @@ vector<float> BleuScorer::ScoreNbestList(const string& scoreFile, const string&
|
||||
vector<pair<size_t,size_t> > hypotheses;
|
||||
UTIL_THROW_IF2(featureDataIters[0] == FeatureDataIterator::end(),
|
||||
"At the end of feature data iterator");
|
||||
for (size_t i = 0; i < featureFiles.size(); ++i)
|
||||
{
|
||||
for (size_t i = 0; i < featureFiles.size(); ++i) {
|
||||
UTIL_THROW_IF2(featureDataIters[i] == FeatureDataIterator::end(),
|
||||
"Feature file " << i << " ended prematurely");
|
||||
UTIL_THROW_IF2(scoreDataIters[i] == ScoreDataIterator::end(),
|
||||
|
@ -13,7 +13,8 @@
|
||||
using namespace std;
|
||||
using namespace MosesTuning;
|
||||
|
||||
BOOST_AUTO_TEST_CASE(viterbi_simple_lattice) {
|
||||
BOOST_AUTO_TEST_CASE(viterbi_simple_lattice)
|
||||
{
|
||||
Vocab vocab;
|
||||
WordVec words;
|
||||
string wordStrings[] =
|
||||
@ -244,7 +245,8 @@ BOOST_AUTO_TEST_CASE(viterbi_3branch_lattice)
|
||||
BOOST_CHECK_EQUAL(6, hopeHypo.bleuStats[8]);
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_CASE(viterbi_full_hypergraph) {
|
||||
BOOST_AUTO_TEST_CASE(viterbi_full_hypergraph)
|
||||
{
|
||||
Vocab vocab;
|
||||
//References
|
||||
ReferenceSet references;
|
||||
|
@ -11,7 +11,8 @@ how many of the features are really "dense". This is because in hg mira
|
||||
all features (sparse and dense) are to get rolled in to SparseVector
|
||||
*/
|
||||
|
||||
BOOST_AUTO_TEST_CASE(from_sparse) {
|
||||
BOOST_AUTO_TEST_CASE(from_sparse)
|
||||
{
|
||||
SparseVector sp;
|
||||
sp.set("dense0", 0.2);
|
||||
sp.set("dense1", 0.3);
|
||||
|
@ -32,8 +32,7 @@ int main(int argc, char **argv)
|
||||
|
||||
// initialize reference streams
|
||||
std::vector<boost::shared_ptr<std::ifstream> > refStreams;
|
||||
for (std::vector<std::string>::const_iterator refFile=refFiles.begin(); refFile!=refFiles.end(); ++refFile)
|
||||
{
|
||||
for (std::vector<std::string>::const_iterator refFile=refFiles.begin(); refFile!=refFiles.end(); ++refFile) {
|
||||
TRACE_ERR("Loading reference from " << *refFile << std::endl);
|
||||
boost::shared_ptr<std::ifstream> ifs(new std::ifstream(refFile->c_str()));
|
||||
UTIL_THROW_IF2(!ifs, "Cannot open " << *refFile);
|
||||
@ -44,8 +43,7 @@ int main(int argc, char **argv)
|
||||
std::string nbestLine;
|
||||
int sid = -1;
|
||||
Reference ref;
|
||||
while ( getline(std::cin, nbestLine) )
|
||||
{
|
||||
while ( getline(std::cin, nbestLine) ) {
|
||||
std::vector<std::string> items;
|
||||
Moses::TokenizeMultiCharSeparator(items, nbestLine, " ||| ");
|
||||
int sidCurrent = Moses::Scan<int>(items[0]);
|
||||
|
@ -34,8 +34,7 @@ int main(int argc, char **argv)
|
||||
|
||||
// initialize reference streams
|
||||
vector<boost::shared_ptr<ifstream> > refStreams;
|
||||
for (vector<string>::const_iterator refFile=refFiles.begin(); refFile!=refFiles.end(); ++refFile)
|
||||
{
|
||||
for (vector<string>::const_iterator refFile=refFiles.begin(); refFile!=refFiles.end(); ++refFile) {
|
||||
TRACE_ERR("Loading reference from " << *refFile << endl);
|
||||
boost::shared_ptr<ifstream> ifs(new ifstream(refFile->c_str()));
|
||||
UTIL_THROW_IF2(!ifs, "Cannot open " << *refFile);
|
||||
@ -45,8 +44,7 @@ int main(int argc, char **argv)
|
||||
// load sentences, preparing statistics, score
|
||||
string hypothesisLine;
|
||||
size_t sid = 0;
|
||||
while (getline(std::cin, hypothesisLine))
|
||||
{
|
||||
while (getline(std::cin, hypothesisLine)) {
|
||||
Reference ref;
|
||||
if (!scorer.GetNextReferenceFromStreams(refStreams, ref)) {
|
||||
UTIL_THROW2("Missing references");
|
||||
|
@ -177,8 +177,7 @@ int main(int argc, char* argv[])
|
||||
const vector<float>& scale_grid = grid.getGrid(lmbr_scale);
|
||||
|
||||
boost::shared_ptr<InputType> source;
|
||||
while((source = ioWrapper->ReadInput()) != NULL)
|
||||
{
|
||||
while((source = ioWrapper->ReadInput()) != NULL) {
|
||||
// set up task of translating one sentence
|
||||
boost::shared_ptr<TranslationTask> ttask;
|
||||
ttask = TranslationTask::create(source, ioWrapper);
|
||||
@ -187,17 +186,13 @@ int main(int argc, char* argv[])
|
||||
TrellisPathList nBestList;
|
||||
manager.CalcNBest(nBestSize, nBestList,true);
|
||||
//grid search
|
||||
BOOST_FOREACH(float const& p, pgrid)
|
||||
{
|
||||
BOOST_FOREACH(float const& p, pgrid) {
|
||||
SD.SetLatticeMBRPrecision(p);
|
||||
BOOST_FOREACH(float const& r, rgrid)
|
||||
{
|
||||
BOOST_FOREACH(float const& r, rgrid) {
|
||||
SD.SetLatticeMBRPRatio(r);
|
||||
BOOST_FOREACH(size_t const prune_i, prune_grid)
|
||||
{
|
||||
BOOST_FOREACH(size_t const prune_i, prune_grid) {
|
||||
SD.SetLatticeMBRPruningFactor(size_t(prune_i));
|
||||
BOOST_FOREACH(float const& scale_i, scale_grid)
|
||||
{
|
||||
BOOST_FOREACH(float const& scale_i, scale_grid) {
|
||||
SD.SetMBRScale(scale_i);
|
||||
size_t lineCount = source->GetTranslationId();
|
||||
cout << lineCount << " ||| " << p << " "
|
||||
|
@ -146,9 +146,10 @@ int main(int argc, char** argv)
|
||||
// main loop over set of input sentences
|
||||
|
||||
boost::shared_ptr<InputType> source;
|
||||
while ((source = ioWrapper->ReadInput()) != NULL)
|
||||
{
|
||||
IFVERBOSE(1) { ResetUserTime(); }
|
||||
while ((source = ioWrapper->ReadInput()) != NULL) {
|
||||
IFVERBOSE(1) {
|
||||
ResetUserTime();
|
||||
}
|
||||
|
||||
InputType* foo = source.get();
|
||||
FeatureFunction::CallChangeSource(foo);
|
||||
|
@ -17,7 +17,9 @@ BaseManager::BaseManager(ttasksptr const& ttask)
|
||||
|
||||
const InputType&
|
||||
BaseManager::GetSource() const
|
||||
{ return m_source; }
|
||||
{
|
||||
return m_source;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
@ -18,16 +18,16 @@
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
class ContextScope
|
||||
{
|
||||
protected:
|
||||
class ContextScope
|
||||
{
|
||||
protected:
|
||||
typedef std::map<void const*, boost::shared_ptr<void> > scratchpad_t;
|
||||
typedef scratchpad_t::iterator iter_t;
|
||||
typedef scratchpad_t::value_type entry_t;
|
||||
typedef scratchpad_t::const_iterator const_iter_t;
|
||||
scratchpad_t m_scratchpad;
|
||||
mutable boost::shared_mutex m_lock;
|
||||
public:
|
||||
public:
|
||||
// class write_access
|
||||
// {
|
||||
// boost::unique_lock<boost::shared_mutex> m_lock;
|
||||
@ -50,26 +50,22 @@ namespace Moses
|
||||
|
||||
template<typename T>
|
||||
boost::shared_ptr<void> const&
|
||||
set(void const* const key, boost::shared_ptr<T> const& val)
|
||||
{
|
||||
set(void const* const key, boost::shared_ptr<T> const& val) {
|
||||
boost::unique_lock<boost::shared_mutex> lock(m_lock);
|
||||
return (m_scratchpad[key] = val);
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
boost::shared_ptr<T> const
|
||||
get(void const* key, bool CreateNewIfNecessary=false)
|
||||
{
|
||||
get(void const* key, bool CreateNewIfNecessary=false) {
|
||||
using boost::shared_mutex;
|
||||
using boost::upgrade_lock;
|
||||
// T const* key = reinterpret_cast<T const*>(xkey);
|
||||
upgrade_lock<shared_mutex> lock(m_lock);
|
||||
iter_t m = m_scratchpad.find(key);
|
||||
boost::shared_ptr< T > ret;
|
||||
if (m != m_scratchpad.end())
|
||||
{
|
||||
if (m->second == NULL && CreateNewIfNecessary)
|
||||
{
|
||||
if (m != m_scratchpad.end()) {
|
||||
if (m->second == NULL && CreateNewIfNecessary) {
|
||||
boost::upgrade_to_unique_lock<shared_mutex> xlock(lock);
|
||||
m->second.reset(new T);
|
||||
}
|
||||
@ -85,13 +81,12 @@ namespace Moses
|
||||
|
||||
ContextScope() { }
|
||||
|
||||
ContextScope(ContextScope const& other)
|
||||
{
|
||||
ContextScope(ContextScope const& other) {
|
||||
boost::unique_lock<boost::shared_mutex> lock1(this->m_lock);
|
||||
boost::unique_lock<boost::shared_mutex> lock2(other.m_lock);
|
||||
m_scratchpad = other.m_scratchpad;
|
||||
}
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
};
|
||||
|
@ -223,8 +223,7 @@ ProcessLEGACY(TranslationOption const& in,
|
||||
TranslationOptionCollection *toc,
|
||||
bool adhereTableLimit) const
|
||||
{
|
||||
if (in.GetTargetPhrase().GetSize() == 0)
|
||||
{
|
||||
if (in.GetTargetPhrase().GetSize() == 0) {
|
||||
// word deletion
|
||||
out.Add(new TranslationOption(in));
|
||||
return;
|
||||
@ -241,16 +240,14 @@ ProcessLEGACY(TranslationOption const& in,
|
||||
TargetPhraseCollectionWithSourcePhrase const* phraseColl;
|
||||
phraseColl = pdict->GetTargetPhraseCollectionLEGACY(toc->GetSource(),srcRange);
|
||||
|
||||
if (phraseColl != NULL)
|
||||
{
|
||||
if (phraseColl != NULL) {
|
||||
TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
|
||||
iterEnd = ((adhereTableLimit && tableLimit && phraseColl->GetSize() >= tableLimit)
|
||||
? phraseColl->begin() + tableLimit : phraseColl->end());
|
||||
|
||||
for (iterTargetPhrase = phraseColl->begin();
|
||||
iterTargetPhrase != iterEnd;
|
||||
++iterTargetPhrase)
|
||||
{
|
||||
++iterTargetPhrase) {
|
||||
TargetPhrase const& targetPhrase = **iterTargetPhrase;
|
||||
if (targetPhrase.GetSize() != currSize ||
|
||||
(IsFilteringStep() && !in.IsCompatible(targetPhrase, m_conflictFactors)))
|
||||
|
@ -114,7 +114,9 @@ string SimpleTranslationInterface::translate(const string &inputString)
|
||||
|
||||
boost::shared_ptr<InputType> source = ioWrapper->ReadInput();
|
||||
if (!source) return "Error: Source==null!!!";
|
||||
IFVERBOSE(1) { ResetUserTime(); }
|
||||
IFVERBOSE(1) {
|
||||
ResetUserTime();
|
||||
}
|
||||
|
||||
FeatureFunction::CallChangeSource(&*source);
|
||||
|
||||
@ -147,10 +149,14 @@ int
|
||||
run_as_server()
|
||||
{
|
||||
#ifdef HAVE_XMLRPC_C
|
||||
int port; params.SetParameter(port, "server-port", 8080);
|
||||
bool isSerial; params.SetParameter(isSerial, "serial", false);
|
||||
string logfile; params.SetParameter(logfile, "server-log", string(""));
|
||||
size_t num_threads; params.SetParameter(num_threads, "threads", size_t(10));
|
||||
int port;
|
||||
params.SetParameter(port, "server-port", 8080);
|
||||
bool isSerial;
|
||||
params.SetParameter(isSerial, "serial", false);
|
||||
string logfile;
|
||||
params.SetParameter(logfile, "server-log", string(""));
|
||||
size_t num_threads;
|
||||
params.SetParameter(num_threads, "threads", size_t(10));
|
||||
if (isSerial) VERBOSE(1,"Running server in serial mode." << endl);
|
||||
|
||||
xmlrpc_c::registry myRegistry;
|
||||
@ -166,8 +172,9 @@ run_as_server()
|
||||
xmlrpc_c::serverAbyss myAbyssServer(myRegistry, port, logfile);
|
||||
|
||||
XVERBOSE(1,"Listening on port " << port << endl);
|
||||
if (isSerial) { while(1) myAbyssServer.runOnce(); }
|
||||
else myAbyssServer.run();
|
||||
if (isSerial) {
|
||||
while(1) myAbyssServer.runOnce();
|
||||
} else myAbyssServer.run();
|
||||
|
||||
std::cerr << "xmlrpc_c::serverAbyss.run() returned but should not." << std::endl;
|
||||
// #pragma message("BUILDING MOSES WITH SERVER SUPPORT")
|
||||
@ -197,8 +204,7 @@ batch_run()
|
||||
|
||||
// check on weights
|
||||
const ScoreComponentCollection& weights = staticData.GetAllWeights();
|
||||
IFVERBOSE(2)
|
||||
{
|
||||
IFVERBOSE(2) {
|
||||
TRACE_ERR("The global weight vector looks like this: ");
|
||||
TRACE_ERR(weights);
|
||||
TRACE_ERR("\n");
|
||||
@ -214,8 +220,7 @@ batch_run()
|
||||
// main loop over set of input sentences
|
||||
|
||||
boost::shared_ptr<InputType> source;
|
||||
while ((source = ioWrapper->ReadInput()) != NULL)
|
||||
{
|
||||
while ((source = ioWrapper->ReadInput()) != NULL) {
|
||||
IFVERBOSE(1) ResetUserTime();
|
||||
|
||||
FeatureFunction::CallChangeSource(source.get());
|
||||
@ -236,8 +241,7 @@ batch_run()
|
||||
// simulated post-editing requires threads (within the dynamic phrase tables)
|
||||
// but runs all sentences serially, to allow updating of the bitext.
|
||||
bool spe = params.isParamSpecified("spe-src");
|
||||
if (spe)
|
||||
{
|
||||
if (spe) {
|
||||
// simulated post-editing: always run single-threaded!
|
||||
task->Run();
|
||||
string src,trg,aln;
|
||||
@ -247,16 +251,14 @@ batch_run()
|
||||
<< "missing update data for simulated post-editing.");
|
||||
UTIL_THROW_IF2(!getline(*ioWrapper->spe_aln,aln), "[" << HERE << "] "
|
||||
<< "missing update data for simulated post-editing.");
|
||||
BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl())
|
||||
{
|
||||
BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl()) {
|
||||
Mmsapt* sapt = dynamic_cast<Mmsapt*>(pd);
|
||||
if (sapt) sapt->add(src,trg,aln);
|
||||
VERBOSE(1,"[" << HERE << " added src] " << src << endl);
|
||||
VERBOSE(1,"[" << HERE << " added trg] " << trg << endl);
|
||||
VERBOSE(1,"[" << HERE << " added aln] " << aln << endl);
|
||||
}
|
||||
}
|
||||
else pool.Submit(task);
|
||||
} else pool.Submit(task);
|
||||
#else
|
||||
pool.Submit(task);
|
||||
|
||||
@ -295,8 +297,7 @@ int decoder_main(int argc, char** argv)
|
||||
#endif
|
||||
|
||||
// echo command line, if verbose
|
||||
IFVERBOSE(1)
|
||||
{
|
||||
IFVERBOSE(1) {
|
||||
TRACE_ERR("command: ");
|
||||
for(int i=0; i<argc; ++i) TRACE_ERR(argv[i]<<" ");
|
||||
TRACE_ERR(endl);
|
||||
@ -317,8 +318,7 @@ int decoder_main(int argc, char** argv)
|
||||
exit(1);
|
||||
|
||||
// setting "-show-weights" -> just dump out weights and exit
|
||||
if (params.isParamSpecified("show-weights"))
|
||||
{
|
||||
if (params.isParamSpecified("show-weights")) {
|
||||
ShowWeights();
|
||||
exit(0);
|
||||
}
|
||||
@ -330,8 +330,7 @@ int decoder_main(int argc, char** argv)
|
||||
|
||||
}
|
||||
#ifdef NDEBUG
|
||||
catch (const std::exception &e)
|
||||
{
|
||||
catch (const std::exception &e) {
|
||||
std::cerr << "Exception: " << e.what() << std::endl;
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
@ -45,7 +45,9 @@ public:
|
||||
~SimpleTranslationInterface();
|
||||
std::string translate(const std::string &input);
|
||||
Moses::StaticData& getStaticData();
|
||||
Moses::Parameter& getParameters(){ return m_params; }
|
||||
Moses::Parameter& getParameters() {
|
||||
return m_params;
|
||||
}
|
||||
private:
|
||||
SimpleTranslationInterface();
|
||||
Moses::Parameter m_params;
|
||||
|
@ -157,20 +157,15 @@ FeatureFactory
|
||||
std::vector<float> weights = static_data.GetParameter()->GetWeights(featureName);
|
||||
|
||||
|
||||
if (feature->GetNumScoreComponents())
|
||||
{
|
||||
if (weights.size() == 0)
|
||||
{
|
||||
if (feature->GetNumScoreComponents()) {
|
||||
if (weights.size() == 0) {
|
||||
weights = feature->DefaultWeights();
|
||||
if (weights.size() == 0)
|
||||
{
|
||||
if (weights.size() == 0) {
|
||||
TRACE_ERR("WARNING: No weights specified in config file for FF "
|
||||
<< featureName << ". This FF does not supply default values.\n"
|
||||
<< "WARNING: Auto-initializing all weights for this FF to 1.0");
|
||||
weights.assign(feature->GetNumScoreComponents(),1.0);
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
TRACE_ERR("WARNING: No weights specified in config file for FF "
|
||||
<< featureName << ". Using default values supplied by FF.");
|
||||
}
|
||||
@ -181,8 +176,7 @@ FeatureFactory
|
||||
<< " (features: " << feature->GetNumScoreComponents()
|
||||
<< " vs. weights: " << weights.size() << ")");
|
||||
static_data.SetWeights(feature, weights);
|
||||
}
|
||||
else if (feature->IsTuneable())
|
||||
} else if (feature->IsTuneable())
|
||||
static_data.SetWeights(feature, weights);
|
||||
}
|
||||
|
||||
|
@ -193,17 +193,23 @@ void FeatureFunction::SetTuneableComponents(const std::string& value)
|
||||
void
|
||||
FeatureFunction
|
||||
::InitializeForInput(ttasksptr const& ttask)
|
||||
{ InitializeForInput(*(ttask->GetSource().get())); }
|
||||
{
|
||||
InitializeForInput(*(ttask->GetSource().get()));
|
||||
}
|
||||
|
||||
void
|
||||
FeatureFunction
|
||||
::CleanUpAfterSentenceProcessing(ttasksptr const& ttask)
|
||||
{ CleanUpAfterSentenceProcessing(*(ttask->GetSource().get())); }
|
||||
{
|
||||
CleanUpAfterSentenceProcessing(*(ttask->GetSource().get()));
|
||||
}
|
||||
|
||||
size_t
|
||||
FeatureFunction
|
||||
::GetIndex() const
|
||||
{ return m_index; }
|
||||
{
|
||||
return m_index;
|
||||
}
|
||||
|
||||
|
||||
/// set index
|
||||
|
@ -136,7 +136,9 @@ public:
|
||||
CleanUpAfterSentenceProcessing(ttasksptr const& ttask);
|
||||
|
||||
const std::string &
|
||||
GetArgLine() const { return m_argLine; }
|
||||
GetArgLine() const {
|
||||
return m_argLine;
|
||||
}
|
||||
|
||||
// given a target phrase containing only factors specified in mask
|
||||
// return true if the feature function can be evaluated
|
||||
|
@ -147,8 +147,7 @@ void InternalTree::GetUnbinarizedChildren(std::vector<TreePointer> &ret) const
|
||||
const std::string &label = (*itx)->GetLabel();
|
||||
if (!label.empty() && label[0] == '^') {
|
||||
(*itx)->GetUnbinarizedChildren(ret);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
ret.push_back(*itx);
|
||||
}
|
||||
}
|
||||
|
@ -96,8 +96,7 @@ public:
|
||||
bool RecursiveSearch(const std::vector<NTLabel> & labels, std::vector<TreePointer>::const_iterator & it, InternalTree const* &parent) const;
|
||||
|
||||
// Python-like generator that yields next nonterminal leaf on every call
|
||||
$generator(leafNT)
|
||||
{
|
||||
$generator(leafNT) {
|
||||
std::vector<TreePointer>::iterator it;
|
||||
InternalTree* tree;
|
||||
leafNT(InternalTree* root = 0): tree(root) {}
|
||||
@ -116,8 +115,7 @@ public:
|
||||
|
||||
|
||||
// Python-like generator that yields the parent of the next nonterminal leaf on every call
|
||||
$generator(leafNTParent)
|
||||
{
|
||||
$generator(leafNTParent) {
|
||||
std::vector<TreePointer>::iterator it;
|
||||
InternalTree* tree;
|
||||
leafNTParent(InternalTree* root = 0): tree(root) {}
|
||||
@ -135,8 +133,7 @@ public:
|
||||
};
|
||||
|
||||
// Python-like generator that yields the next nonterminal leaf on every call, and also stores the path from the root of the tree to the nonterminal
|
||||
$generator(leafNTPath)
|
||||
{
|
||||
$generator(leafNTPath) {
|
||||
std::vector<TreePointer>::iterator it;
|
||||
InternalTree* tree;
|
||||
std::vector<InternalTree*> * path;
|
||||
|
@ -44,8 +44,7 @@ public:
|
||||
static const ReorderingType L = 1; // left
|
||||
static const ReorderingType MAX = 3; // largest possible
|
||||
#else
|
||||
enum ReorderingType
|
||||
{
|
||||
enum ReorderingType {
|
||||
M = 0, // monotonic
|
||||
NM = 1, // non-monotonic
|
||||
S = 1, // swap
|
||||
|
@ -71,21 +71,18 @@ void Model1Vocabulary::Load(const std::string& fileName)
|
||||
std::string line;
|
||||
|
||||
unsigned i = 0;
|
||||
if ( getline(inFile, line) ) // first line of MGIZA vocabulary files seems to be special : "1 UNK 0" -- skip if it's this
|
||||
{
|
||||
if ( getline(inFile, line) ) { // first line of MGIZA vocabulary files seems to be special : "1 UNK 0" -- skip if it's this
|
||||
++i;
|
||||
std::vector<std::string> tokens = Tokenize(line);
|
||||
UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens.");
|
||||
unsigned id = Scan<unsigned>(tokens[0]);
|
||||
if (! ( (id == 1) && (tokens[1] == "UNK") ))
|
||||
{
|
||||
if (! ( (id == 1) && (tokens[1] == "UNK") )) {
|
||||
const Factor* factor = factorCollection.AddFactor(tokens[1],false); // TODO: can we assume that the vocabulary is know and filter the model on loading?
|
||||
bool stored = Store(factor, id);
|
||||
UTIL_THROW_IF2(!stored, "Line " << i << " in " << fileName << " overwrites existing vocabulary entry.");
|
||||
}
|
||||
}
|
||||
while ( getline(inFile, line) )
|
||||
{
|
||||
while ( getline(inFile, line) ) {
|
||||
++i;
|
||||
std::vector<std::string> tokens = Tokenize(line);
|
||||
UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens.");
|
||||
@ -104,8 +101,7 @@ void Model1LexicalTable::Load(const std::string &fileName, const Model1Vocabular
|
||||
std::string line;
|
||||
|
||||
unsigned i = 0;
|
||||
while ( getline(inFile, line) )
|
||||
{
|
||||
while ( getline(inFile, line) ) {
|
||||
++i;
|
||||
std::vector<std::string> tokens = Tokenize(line);
|
||||
UTIL_THROW_IF2(tokens.size()!=3, "Line " << i << " in " << fileName << " has wrong number of tokens.");
|
||||
@ -193,25 +189,21 @@ void Model1Feature::EvaluateWithSourceContext(const InputType &input
|
||||
float score = 0.0;
|
||||
float norm = TransformScore(1+sentence.GetSize());
|
||||
|
||||
for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT)
|
||||
{
|
||||
for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) {
|
||||
const Word &wordT = targetPhrase.GetWord(posT);
|
||||
if ( !wordT.IsNonTerminal() )
|
||||
{
|
||||
if ( !wordT.IsNonTerminal() ) {
|
||||
float thisWordProb = m_model1.GetProbability(m_emptyWord,wordT[0]); // probability conditioned on empty word
|
||||
|
||||
// cache lookup
|
||||
bool foundInCache = false;
|
||||
{
|
||||
#ifdef WITH_THREADS
|
||||
#ifdef WITH_THREADS
|
||||
boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
|
||||
#endif
|
||||
#endif
|
||||
boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> >::const_iterator sentenceCache = m_cache.find(&input);
|
||||
if (sentenceCache != m_cache.end())
|
||||
{
|
||||
if (sentenceCache != m_cache.end()) {
|
||||
boost::unordered_map<const Factor*, float>::const_iterator cacheHit = sentenceCache->second.find(wordT[0]);
|
||||
if (cacheHit != sentenceCache->second.end())
|
||||
{
|
||||
if (cacheHit != sentenceCache->second.end()) {
|
||||
foundInCache = true;
|
||||
score += cacheHit->second;
|
||||
FEATUREVERBOSE(3, "Cached score( " << wordT << " ) = " << cacheHit->second << std::endl);
|
||||
@ -219,10 +211,8 @@ void Model1Feature::EvaluateWithSourceContext(const InputType &input
|
||||
}
|
||||
}
|
||||
|
||||
if (!foundInCache)
|
||||
{
|
||||
for (size_t posS=1; posS<sentence.GetSize()-1; ++posS) // ignore <s> and </s>
|
||||
{
|
||||
if (!foundInCache) {
|
||||
for (size_t posS=1; posS<sentence.GetSize()-1; ++posS) { // ignore <s> and </s>
|
||||
const Word &wordS = sentence.GetWord(posS);
|
||||
float modelProb = m_model1.GetProbability(wordS[0],wordT[0]);
|
||||
FEATUREVERBOSE(4, "p( " << wordT << " | " << wordS << " ) = " << modelProb << std::endl);
|
||||
@ -231,10 +221,10 @@ void Model1Feature::EvaluateWithSourceContext(const InputType &input
|
||||
float thisWordScore = TransformScore(thisWordProb) - norm;
|
||||
FEATUREVERBOSE(3, "score( " << wordT << " ) = " << thisWordScore << std::endl);
|
||||
{
|
||||
#ifdef WITH_THREADS
|
||||
#ifdef WITH_THREADS
|
||||
// need to update cache; write lock
|
||||
boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
|
||||
#endif
|
||||
#endif
|
||||
m_cache[&input][wordT[0]] = thisWordScore;
|
||||
}
|
||||
score += thisWordScore;
|
||||
@ -247,14 +237,13 @@ void Model1Feature::EvaluateWithSourceContext(const InputType &input
|
||||
|
||||
void Model1Feature::CleanUpAfterSentenceProcessing(const InputType& source)
|
||||
{
|
||||
#ifdef WITH_THREADS
|
||||
#ifdef WITH_THREADS
|
||||
// need to update cache; write lock
|
||||
boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
|
||||
#endif
|
||||
#endif
|
||||
// clear cache
|
||||
boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> >::iterator sentenceCache = m_cache.find(&source);
|
||||
if (sentenceCache != m_cache.end())
|
||||
{
|
||||
if (sentenceCache != m_cache.end()) {
|
||||
sentenceCache->second.clear();
|
||||
m_cache.erase(sentenceCache);
|
||||
}
|
||||
|
@ -17,7 +17,7 @@ class Model1Vocabulary
|
||||
{
|
||||
public:
|
||||
|
||||
#define INVALID_ID std::numeric_limits<unsigned>::max() // UINT_MAX
|
||||
#define INVALID_ID std::numeric_limits<unsigned>::max() // UINT_MAX
|
||||
static const std::string GIZANULL;
|
||||
|
||||
Model1Vocabulary();
|
||||
@ -103,10 +103,10 @@ private:
|
||||
|
||||
// cache
|
||||
mutable boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> > m_cache;
|
||||
#ifdef WITH_THREADS
|
||||
#ifdef WITH_THREADS
|
||||
// reader-writer lock
|
||||
mutable boost::shared_mutex m_accessLock;
|
||||
#endif
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
|
@ -51,8 +51,7 @@ void RulePairUnlexicalizedSource::EvaluateInIsolation(const Phrase &source
|
||||
return;
|
||||
}
|
||||
|
||||
for (size_t posS=0; posS<source.GetSize(); ++posS)
|
||||
{
|
||||
for (size_t posS=0; posS<source.GetSize(); ++posS) {
|
||||
const Word &wordS = source.GetWord(posS);
|
||||
if ( !wordS.IsNonTerminal() ) {
|
||||
return;
|
||||
@ -61,8 +60,7 @@ void RulePairUnlexicalizedSource::EvaluateInIsolation(const Phrase &source
|
||||
|
||||
ostringstream namestr;
|
||||
|
||||
for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT)
|
||||
{
|
||||
for (size_t posT=0; posT<targetPhrase.GetSize(); ++posT) {
|
||||
const Word &wordT = targetPhrase.GetWord(posT);
|
||||
const Factor* factorT = wordT[0];
|
||||
if ( wordT.IsNonTerminal() ) {
|
||||
@ -78,8 +76,7 @@ void RulePairUnlexicalizedSource::EvaluateInIsolation(const Phrase &source
|
||||
namestr << targetPhraseLHS->GetString() << "|";
|
||||
|
||||
for (AlignmentInfo::const_iterator it=targetPhrase.GetAlignNonTerm().begin();
|
||||
it!=targetPhrase.GetAlignNonTerm().end(); ++it)
|
||||
{
|
||||
it!=targetPhrase.GetAlignNonTerm().end(); ++it) {
|
||||
namestr << "|" << it->first << "-" << it->second;
|
||||
}
|
||||
|
||||
|
@ -68,15 +68,12 @@ void RuleScope::EvaluateInIsolation(const Phrase &source
|
||||
|
||||
if (m_futureCostOnly) {
|
||||
estimatedFutureScore.PlusEquals(this, scores);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
scoreBreakdown.PlusEquals(this, scores);
|
||||
}
|
||||
}
|
||||
else if (m_futureCostOnly) {
|
||||
} else if (m_futureCostOnly) {
|
||||
estimatedFutureScore.PlusEquals(this, score);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
scoreBreakdown.PlusEquals(this, score);
|
||||
}
|
||||
}
|
||||
@ -85,14 +82,11 @@ void RuleScope::SetParameter(const std::string& key, const std::string& value)
|
||||
{
|
||||
if (key == "source-syntax") {
|
||||
m_sourceSyntax = Scan<bool>(value);
|
||||
}
|
||||
else if (key == "per-scope") {
|
||||
} else if (key == "per-scope") {
|
||||
m_perScope = Scan<bool>(value);
|
||||
}
|
||||
else if ("future-cost-only") {
|
||||
} else if ("future-cost-only") {
|
||||
m_futureCostOnly = Scan<bool>(value);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
StatelessFeatureFunction::SetParameter(key, value);
|
||||
}
|
||||
}
|
||||
|
@ -45,11 +45,11 @@ namespace Moses
|
||||
{
|
||||
|
||||
#ifdef USE_HYPO_POOL
|
||||
ObjectPool<Hypothesis> Hypothesis::s_objectPool("Hypothesis", 300000);
|
||||
ObjectPool<Hypothesis> Hypothesis::s_objectPool("Hypothesis", 300000);
|
||||
#endif
|
||||
|
||||
Hypothesis::
|
||||
Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt)
|
||||
Hypothesis::
|
||||
Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt)
|
||||
: m_prevHypo(NULL)
|
||||
, m_sourceCompleted(source.GetSize(), manager.GetSource().m_sourceCompleted)
|
||||
, m_sourceInput(source)
|
||||
@ -65,7 +65,7 @@ namespace Moses
|
||||
, m_transOpt(initialTransOpt)
|
||||
, m_manager(manager)
|
||||
, m_id(m_manager.GetNextHypoId())
|
||||
{
|
||||
{
|
||||
// used for initial seeding of trans process
|
||||
// initialize scores
|
||||
//_hash_computed = false;
|
||||
@ -74,13 +74,13 @@ namespace Moses
|
||||
for (unsigned i = 0; i < ffs.size(); ++i)
|
||||
m_ffStates[i] = ffs[i]->EmptyHypothesisState(source);
|
||||
m_manager.GetSentenceStats().AddCreated();
|
||||
}
|
||||
}
|
||||
|
||||
/***
|
||||
/***
|
||||
* continue prevHypo by appending the phrases in transOpt
|
||||
*/
|
||||
Hypothesis::
|
||||
Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt)
|
||||
Hypothesis::
|
||||
Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt)
|
||||
: m_prevHypo(&prevHypo)
|
||||
, m_sourceCompleted(prevHypo.m_sourceCompleted )
|
||||
, m_sourceInput(prevHypo.m_sourceInput)
|
||||
@ -96,7 +96,7 @@ namespace Moses
|
||||
, m_transOpt(transOpt)
|
||||
, m_manager(prevHypo.GetManager())
|
||||
, m_id(m_manager.GetNextHypoId())
|
||||
{
|
||||
{
|
||||
m_currScoreBreakdown.PlusEquals(transOpt.GetScoreBreakdown());
|
||||
|
||||
// assert that we are not extending our hypothesis by retranslating something
|
||||
@ -107,11 +107,11 @@ namespace Moses
|
||||
m_sourceCompleted.SetValue(m_currSourceWordsRange.GetStartPos(), m_currSourceWordsRange.GetEndPos(), true);
|
||||
m_wordDeleted = transOpt.IsDeletionOption();
|
||||
m_manager.GetSentenceStats().AddCreated();
|
||||
}
|
||||
}
|
||||
|
||||
Hypothesis::
|
||||
~Hypothesis()
|
||||
{
|
||||
Hypothesis::
|
||||
~Hypothesis()
|
||||
{
|
||||
for (unsigned i = 0; i < m_ffStates.size(); ++i)
|
||||
delete m_ffStates[i];
|
||||
|
||||
@ -125,12 +125,12 @@ namespace Moses
|
||||
delete m_arcList;
|
||||
m_arcList = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Hypothesis::
|
||||
AddArc(Hypothesis *loserHypo)
|
||||
{
|
||||
void
|
||||
Hypothesis::
|
||||
AddArc(Hypothesis *loserHypo)
|
||||
{
|
||||
if (!m_arcList) {
|
||||
if (loserHypo->m_arcList) { // we don't have an arcList, but loser does
|
||||
this->m_arcList = loserHypo->m_arcList; // take ownership, we'll delete
|
||||
@ -151,25 +151,25 @@ namespace Moses
|
||||
}
|
||||
}
|
||||
m_arcList->push_back(loserHypo);
|
||||
}
|
||||
}
|
||||
|
||||
/***
|
||||
/***
|
||||
* return the subclass of Hypothesis most appropriate to the given translation option
|
||||
*/
|
||||
Hypothesis*
|
||||
Hypothesis::
|
||||
CreateNext(const TranslationOption &transOpt) const
|
||||
{
|
||||
Hypothesis*
|
||||
Hypothesis::
|
||||
CreateNext(const TranslationOption &transOpt) const
|
||||
{
|
||||
return Create(*this, transOpt);
|
||||
}
|
||||
}
|
||||
|
||||
/***
|
||||
/***
|
||||
* return the subclass of Hypothesis most appropriate to the given translation option
|
||||
*/
|
||||
Hypothesis*
|
||||
Hypothesis::
|
||||
Create(const Hypothesis &prevHypo, const TranslationOption &transOpt)
|
||||
{
|
||||
Hypothesis*
|
||||
Hypothesis::
|
||||
Create(const Hypothesis &prevHypo, const TranslationOption &transOpt)
|
||||
{
|
||||
|
||||
#ifdef USE_HYPO_POOL
|
||||
Hypothesis *ptr = s_objectPool.getPtr();
|
||||
@ -177,33 +177,33 @@ namespace Moses
|
||||
#else
|
||||
return new Hypothesis(prevHypo, transOpt);
|
||||
#endif
|
||||
}
|
||||
/***
|
||||
}
|
||||
/***
|
||||
* return the subclass of Hypothesis most appropriate to the given target phrase
|
||||
*/
|
||||
|
||||
Hypothesis*
|
||||
Hypothesis::
|
||||
Create(Manager& manager, InputType const& m_source,
|
||||
Hypothesis*
|
||||
Hypothesis::
|
||||
Create(Manager& manager, InputType const& m_source,
|
||||
const TranslationOption &initialTransOpt)
|
||||
{
|
||||
{
|
||||
#ifdef USE_HYPO_POOL
|
||||
Hypothesis *ptr = s_objectPool.getPtr();
|
||||
return new(ptr) Hypothesis(manager, m_source, initialTransOpt);
|
||||
#else
|
||||
return new Hypothesis(manager, m_source, initialTransOpt);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
/** check, if two hypothesis can be recombined.
|
||||
/** check, if two hypothesis can be recombined.
|
||||
this is actually a sorting function that allows us to
|
||||
keep an ordered list of hypotheses. This makes recombination
|
||||
much quicker.
|
||||
*/
|
||||
int
|
||||
Hypothesis::
|
||||
RecombineCompare(const Hypothesis &compare) const
|
||||
{
|
||||
*/
|
||||
int
|
||||
Hypothesis::
|
||||
RecombineCompare(const Hypothesis &compare) const
|
||||
{
|
||||
// -1 = this < compare
|
||||
// +1 = this > compare
|
||||
// 0 = this ==compare
|
||||
@ -221,40 +221,39 @@ namespace Moses
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Hypothesis::
|
||||
EvaluateWhenApplied(StatefulFeatureFunction const& sfff,
|
||||
void
|
||||
Hypothesis::
|
||||
EvaluateWhenApplied(StatefulFeatureFunction const& sfff,
|
||||
int state_idx)
|
||||
{
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
if (! staticData.IsFeatureFunctionIgnored( sfff ))
|
||||
{
|
||||
if (! staticData.IsFeatureFunctionIgnored( sfff )) {
|
||||
m_ffStates[state_idx]
|
||||
= sfff.EvaluateWhenApplied
|
||||
(*this, m_prevHypo ? m_prevHypo->m_ffStates[state_idx] : NULL,
|
||||
&m_currScoreBreakdown);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Hypothesis::
|
||||
EvaluateWhenApplied(const StatelessFeatureFunction& slff)
|
||||
{
|
||||
void
|
||||
Hypothesis::
|
||||
EvaluateWhenApplied(const StatelessFeatureFunction& slff)
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
if (! staticData.IsFeatureFunctionIgnored( slff )) {
|
||||
slff.EvaluateWhenApplied(*this, &m_currScoreBreakdown);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/***
|
||||
/***
|
||||
* calculate the logarithm of our total translation score (sum up components)
|
||||
*/
|
||||
void
|
||||
Hypothesis::
|
||||
EvaluateWhenApplied(const SquareMatrix &futureScore)
|
||||
{
|
||||
void
|
||||
Hypothesis::
|
||||
EvaluateWhenApplied(const SquareMatrix &futureScore)
|
||||
{
|
||||
IFVERBOSE(2) {
|
||||
m_manager.GetSentenceStats().StartTimeOtherScore();
|
||||
}
|
||||
@ -299,20 +298,20 @@ namespace Moses
|
||||
IFVERBOSE(2) {
|
||||
m_manager.GetSentenceStats().StopTimeEstimateScore();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const Hypothesis* Hypothesis::GetPrevHypo()const
|
||||
{
|
||||
const Hypothesis* Hypothesis::GetPrevHypo()const
|
||||
{
|
||||
return m_prevHypo;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
/**
|
||||
* print hypothesis information for pharaoh-style logging
|
||||
*/
|
||||
void
|
||||
Hypothesis::
|
||||
PrintHypothesis() const
|
||||
{
|
||||
void
|
||||
Hypothesis::
|
||||
PrintHypothesis() const
|
||||
{
|
||||
if (!m_prevHypo) {
|
||||
TRACE_ERR(endl << "NULL hypo" << endl);
|
||||
return;
|
||||
@ -344,12 +343,12 @@ namespace Moses
|
||||
TRACE_ERR( "\tscore "<<m_totalScore - m_futureScore<<" + future cost "<<m_futureScore<<" = "<<m_totalScore<<endl);
|
||||
TRACE_ERR( "\tunweighted feature scores: " << m_currScoreBreakdown << endl);
|
||||
//PrintLMScores();
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Hypothesis::
|
||||
CleanupArcList()
|
||||
{
|
||||
void
|
||||
Hypothesis::
|
||||
CleanupArcList()
|
||||
{
|
||||
// point this hypo's main hypo to itself
|
||||
SetWinningHypo(this);
|
||||
|
||||
@ -369,8 +368,7 @@ namespace Moses
|
||||
staticData.GetOutputSearchGraphHypergraph() ||
|
||||
staticData.UseLatticeMBR());
|
||||
|
||||
if (!distinctNBest && m_arcList->size() > nBestSize * 5)
|
||||
{
|
||||
if (!distinctNBest && m_arcList->size() > nBestSize * 5) {
|
||||
// prune arc list only if there too many arcs
|
||||
NTH_ELEMENT4(m_arcList->begin(), m_arcList->begin() + nBestSize - 1,
|
||||
m_arcList->end(), CompareHypothesisTotalScore());
|
||||
@ -388,27 +386,29 @@ namespace Moses
|
||||
Hypothesis *arc = *iter;
|
||||
arc->SetWinningHypo(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TargetPhrase const&
|
||||
Hypothesis::
|
||||
GetCurrTargetPhrase() const
|
||||
{ return m_transOpt.GetTargetPhrase(); }
|
||||
TargetPhrase const&
|
||||
Hypothesis::
|
||||
GetCurrTargetPhrase() const
|
||||
{
|
||||
return m_transOpt.GetTargetPhrase();
|
||||
}
|
||||
|
||||
void
|
||||
Hypothesis::
|
||||
GetOutputPhrase(Phrase &out) const
|
||||
{
|
||||
void
|
||||
Hypothesis::
|
||||
GetOutputPhrase(Phrase &out) const
|
||||
{
|
||||
if (m_prevHypo != NULL)
|
||||
m_prevHypo->GetOutputPhrase(out);
|
||||
out.Append(GetCurrTargetPhrase());
|
||||
}
|
||||
}
|
||||
|
||||
TO_STRING_BODY(Hypothesis)
|
||||
TO_STRING_BODY(Hypothesis)
|
||||
|
||||
// friend
|
||||
ostream& operator<<(ostream& out, const Hypothesis& hypo)
|
||||
{
|
||||
// friend
|
||||
ostream& operator<<(ostream& out, const Hypothesis& hypo)
|
||||
{
|
||||
hypo.ToStream(out);
|
||||
// words bitmap
|
||||
out << "[" << hypo.m_sourceCompleted << "] ";
|
||||
@ -421,43 +421,47 @@ namespace Moses
|
||||
out << " " << hypo.GetCurrTargetPhrase().GetAlignNonTerm();
|
||||
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
std::string
|
||||
Hypothesis::
|
||||
GetSourcePhraseStringRep(const vector<FactorType> factorsToPrint) const
|
||||
{ return m_transOpt.GetInputPath().GetPhrase().GetStringRep(factorsToPrint); }
|
||||
std::string
|
||||
Hypothesis::
|
||||
GetSourcePhraseStringRep(const vector<FactorType> factorsToPrint) const
|
||||
{
|
||||
return m_transOpt.GetInputPath().GetPhrase().GetStringRep(factorsToPrint);
|
||||
}
|
||||
|
||||
std::string
|
||||
Hypothesis::
|
||||
GetTargetPhraseStringRep(const vector<FactorType> factorsToPrint) const
|
||||
{ return (m_prevHypo ? GetCurrTargetPhrase().GetStringRep(factorsToPrint) : ""); }
|
||||
std::string
|
||||
Hypothesis::
|
||||
GetTargetPhraseStringRep(const vector<FactorType> factorsToPrint) const
|
||||
{
|
||||
return (m_prevHypo ? GetCurrTargetPhrase().GetStringRep(factorsToPrint) : "");
|
||||
}
|
||||
|
||||
std::string
|
||||
Hypothesis::
|
||||
GetSourcePhraseStringRep() const
|
||||
{
|
||||
std::string
|
||||
Hypothesis::
|
||||
GetSourcePhraseStringRep() const
|
||||
{
|
||||
vector<FactorType> allFactors(MAX_NUM_FACTORS);
|
||||
for(size_t i=0; i < MAX_NUM_FACTORS; i++)
|
||||
allFactors[i] = i;
|
||||
return GetSourcePhraseStringRep(allFactors);
|
||||
}
|
||||
}
|
||||
|
||||
std::string
|
||||
Hypothesis::
|
||||
GetTargetPhraseStringRep() const
|
||||
{
|
||||
std::string
|
||||
Hypothesis::
|
||||
GetTargetPhraseStringRep() const
|
||||
{
|
||||
vector<FactorType> allFactors(MAX_NUM_FACTORS);
|
||||
for(size_t i=0; i < MAX_NUM_FACTORS; i++)
|
||||
allFactors[i] = i;
|
||||
return GetTargetPhraseStringRep(allFactors);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Hypothesis::
|
||||
OutputAlignment(std::ostream &out) const
|
||||
{
|
||||
void
|
||||
Hypothesis::
|
||||
OutputAlignment(std::ostream &out) const
|
||||
{
|
||||
std::vector<const Hypothesis *> edges;
|
||||
const Hypothesis *currentHypo = this;
|
||||
while (currentHypo) {
|
||||
@ -467,12 +471,12 @@ namespace Moses
|
||||
|
||||
OutputAlignment(out, edges);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Hypothesis::
|
||||
OutputAlignment(ostream &out, const vector<const Hypothesis *> &edges)
|
||||
{
|
||||
void
|
||||
Hypothesis::
|
||||
OutputAlignment(ostream &out, const vector<const Hypothesis *> &edges)
|
||||
{
|
||||
size_t targetOffset = 0;
|
||||
|
||||
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
|
||||
@ -485,13 +489,13 @@ namespace Moses
|
||||
targetOffset += tp.GetSize();
|
||||
}
|
||||
// Used by --print-alignment-info, so no endl
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Hypothesis::
|
||||
OutputAlignment(ostream &out, const AlignmentInfo &ai,
|
||||
void
|
||||
Hypothesis::
|
||||
OutputAlignment(ostream &out, const AlignmentInfo &ai,
|
||||
size_t sourceOffset, size_t targetOffset)
|
||||
{
|
||||
{
|
||||
typedef std::vector< const std::pair<size_t,size_t>* > AlignVec;
|
||||
AlignVec alignments = ai.GetSortedAlignments();
|
||||
|
||||
@ -501,51 +505,51 @@ namespace Moses
|
||||
out << alignment.first + sourceOffset << "-" << alignment.second + targetOffset << " ";
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Hypothesis::
|
||||
OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo)
|
||||
{
|
||||
void
|
||||
Hypothesis::
|
||||
OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo)
|
||||
{
|
||||
if (!hypo->GetPrevHypo()) return;
|
||||
OutputInput(map, hypo->GetPrevHypo());
|
||||
map[hypo->GetCurrSourceWordsRange().GetStartPos()]
|
||||
= &hypo->GetTranslationOption().GetInputPath().GetPhrase();
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Hypothesis::
|
||||
OutputInput(std::ostream& os) const
|
||||
{
|
||||
void
|
||||
Hypothesis::
|
||||
OutputInput(std::ostream& os) const
|
||||
{
|
||||
size_t len = this->GetInput().GetSize();
|
||||
std::vector<const Phrase*> inp_phrases(len, 0);
|
||||
OutputInput(inp_phrases, this);
|
||||
for (size_t i=0; i<len; ++i)
|
||||
if (inp_phrases[i]) os << *inp_phrases[i];
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Hypothesis::
|
||||
OutputBestSurface(std::ostream &out, const std::vector<FactorType> &outputFactorOrder,
|
||||
void
|
||||
Hypothesis::
|
||||
OutputBestSurface(std::ostream &out, const std::vector<FactorType> &outputFactorOrder,
|
||||
char reportSegmentation, bool reportAllFactors) const
|
||||
{
|
||||
if (m_prevHypo)
|
||||
{ // recursively retrace this best path through the lattice, starting from the end of the hypothesis sentence
|
||||
{
|
||||
if (m_prevHypo) {
|
||||
// recursively retrace this best path through the lattice, starting from the end of the hypothesis sentence
|
||||
m_prevHypo->OutputBestSurface(out, outputFactorOrder, reportSegmentation, reportAllFactors);
|
||||
}
|
||||
OutputSurface(out, *this, outputFactorOrder, reportSegmentation, reportAllFactors);
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
/***
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
/***
|
||||
* print surface factor only for the given phrase
|
||||
*/
|
||||
void
|
||||
Hypothesis::
|
||||
OutputSurface(std::ostream &out, const Hypothesis &edge,
|
||||
void
|
||||
Hypothesis::
|
||||
OutputSurface(std::ostream &out, const Hypothesis &edge,
|
||||
const std::vector<FactorType> &outputFactorOrder,
|
||||
char reportSegmentation, bool reportAllFactors) const
|
||||
{
|
||||
{
|
||||
UTIL_THROW_IF2(outputFactorOrder.size() == 0,
|
||||
"Must specific at least 1 output factor");
|
||||
const TargetPhrase& phrase = edge.GetCurrTargetPhrase();
|
||||
@ -614,12 +618,12 @@ namespace Moses
|
||||
}
|
||||
out << "| ";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::map<size_t, const Factor*>
|
||||
Hypothesis::
|
||||
GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor) const
|
||||
{
|
||||
std::map<size_t, const Factor*>
|
||||
Hypothesis::
|
||||
GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor) const
|
||||
{
|
||||
const InputPath &inputPath = hypo.GetTranslationOption().GetInputPath();
|
||||
const Phrase &inputPhrase = inputPath.GetPhrase();
|
||||
|
||||
@ -636,13 +640,13 @@ namespace Moses
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef HAVE_XMLRPC_C
|
||||
void
|
||||
Hypothesis::
|
||||
OutputLocalWordAlignment(vector<xmlrpc_c::value>& dest) const
|
||||
{
|
||||
void
|
||||
Hypothesis::
|
||||
OutputLocalWordAlignment(vector<xmlrpc_c::value>& dest) const
|
||||
{
|
||||
using namespace std;
|
||||
WordsRange const& src = this->GetCurrSourceWordsRange();
|
||||
WordsRange const& trg = this->GetCurrTargetWordsRange();
|
||||
@ -651,24 +655,23 @@ namespace Moses
|
||||
= this->GetCurrTargetPhrase().GetAlignTerm().GetSortedAlignments();
|
||||
typedef pair<size_t,size_t> item;
|
||||
map<string, xmlrpc_c::value> M;
|
||||
BOOST_FOREACH(item const* p, a)
|
||||
{
|
||||
BOOST_FOREACH(item const* p, a) {
|
||||
M["source-word"] = xmlrpc_c::value_int(src.GetStartPos() + p->first);
|
||||
M["target-word"] = xmlrpc_c::value_int(trg.GetStartPos() + p->second);
|
||||
dest.push_back(xmlrpc_c::value_struct(M));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Hypothesis::
|
||||
OutputWordAlignment(vector<xmlrpc_c::value>& out) const
|
||||
{
|
||||
void
|
||||
Hypothesis::
|
||||
OutputWordAlignment(vector<xmlrpc_c::value>& out) const
|
||||
{
|
||||
vector<Hypothesis const*> tmp;
|
||||
for (Hypothesis const* h = this; h; h = h->GetPrevHypo())
|
||||
tmp.push_back(h);
|
||||
for (size_t i = tmp.size(); i-- > 0;)
|
||||
tmp[i]->OutputLocalWordAlignment(out);
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
@ -161,7 +161,7 @@ public:
|
||||
return m_detailTreeFragmentsOutputCollector.get();
|
||||
}
|
||||
|
||||
void SetInputStreamFromString(std::istringstream &input){
|
||||
void SetInputStreamFromString(std::istringstream &input) {
|
||||
m_inputStream = &input;
|
||||
}
|
||||
|
||||
|
@ -13,12 +13,14 @@ namespace Moses
|
||||
|
||||
typedef Eigen::Map<Eigen::Matrix<int,Eigen::Dynamic,1> > EigenMap;
|
||||
|
||||
RDLM::~RDLM() {
|
||||
RDLM::~RDLM()
|
||||
{
|
||||
delete lm_head_base_instance_;
|
||||
delete lm_label_base_instance_;
|
||||
}
|
||||
|
||||
void RDLM::Load() {
|
||||
void RDLM::Load()
|
||||
{
|
||||
|
||||
lm_head_base_instance_ = new nplm::neuralTM();
|
||||
lm_head_base_instance_->read(m_path_head_lm);
|
||||
@ -202,8 +204,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
|
||||
// ignore glue rules
|
||||
if (root->GetLabel() == m_glueSymbol) {
|
||||
// recursion
|
||||
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it)
|
||||
{
|
||||
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it) {
|
||||
Score(it->get(), back_pointers, score, ancestor_heads, ancestor_labels, boundary_hash, num_virtual, rescoring_levels);
|
||||
}
|
||||
return;
|
||||
@ -260,8 +261,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
|
||||
}
|
||||
if (ancestor_labels.size() >= m_context_up && !num_virtual) {
|
||||
score[0] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size())));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
boost::hash_combine(boundary_hash, ngram_head_null.back());
|
||||
score[1] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size())));
|
||||
}
|
||||
@ -276,8 +276,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
|
||||
return;
|
||||
}
|
||||
rescoring_levels = m_context_up-1;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
@ -305,16 +304,14 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
|
||||
virtual_head = true;
|
||||
if (m_binarized == 1 || (m_binarized == 3 && head_label[2] == 'l')) {
|
||||
reached_end = 1; //indicate that we've seen the first symbol of the RHS
|
||||
}
|
||||
else if (m_binarized == 2 || (m_binarized == 3 && head_label[2] == 'r')) {
|
||||
} else if (m_binarized == 2 || (m_binarized == 3 && head_label[2] == 'r')) {
|
||||
reached_end = 2; // indicate that we've seen the last symbol of the RHS
|
||||
}
|
||||
// with 'full' binarization, direction is encoded in 2nd char
|
||||
std::string clipped_label = (m_binarized == 3) ? head_label.substr(2,head_label.size()-2) : head_label.substr(1,head_label.size()-1);
|
||||
label_idx = lm_label->lookup_input_word(clipped_label);
|
||||
label_idx_out = lm_label->lookup_output_word(clipped_label);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
reached_end = 3; // indicate that we've seen first and last symbol of the RHS
|
||||
label_idx = lm_label->lookup_input_word(head_label);
|
||||
label_idx_out = lm_label->lookup_output_word(head_label);
|
||||
@ -341,8 +338,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
|
||||
it = std::copy(ancestor_heads.end()-context_up_nonempty, ancestor_heads.end(), it);
|
||||
it = std::copy(ancestor_labels.end()-context_up_nonempty, ancestor_labels.end(), it);
|
||||
score[0] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size())));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
boost::hash_combine(boundary_hash, ngram_head_null.back());
|
||||
score[1] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram_head_null.data(), ngram_head_null.size())));
|
||||
}
|
||||
@ -362,8 +358,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
|
||||
it = std::copy(ancestor_heads.end()-context_up_nonempty, ancestor_heads.end(), it);
|
||||
it = std::copy(ancestor_labels.end()-context_up_nonempty, ancestor_labels.end(), it);
|
||||
score[2] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram_label_null.data(), ngram_label_null.size())));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
boost::hash_combine(boundary_hash, ngram_label_null.back());
|
||||
score[3] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram_label_null.data(), ngram_label_null.size())));
|
||||
}
|
||||
@ -374,8 +369,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
|
||||
|
||||
if (virtual_head) {
|
||||
num_virtual = m_context_up;
|
||||
}
|
||||
else if (num_virtual) {
|
||||
} else if (num_virtual) {
|
||||
--num_virtual;
|
||||
}
|
||||
|
||||
@ -456,8 +450,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
|
||||
if (reached_end == 2 || reached_end == 3) {
|
||||
std::fill_n(it, right_padding, static_stop_head);
|
||||
it += right_padding;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
std::copy(static_label_null.begin()+offset_up_head-m_context_right-right_padding, static_label_null.begin()-m_context_right+offset_up_head, it);
|
||||
}
|
||||
}
|
||||
@ -468,8 +461,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
|
||||
if (reached_end == 2 || reached_end == 3) {
|
||||
std::fill_n(it, right_padding, static_stop_label);
|
||||
it += right_padding;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
std::copy(static_label_null.begin()+offset_up_head-right_padding, static_label_null.begin()+offset_up_head, it);
|
||||
}
|
||||
}
|
||||
@ -478,8 +470,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
|
||||
|
||||
if (ancestor_labels.size() >= m_context_up && !num_virtual) {
|
||||
score[2] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram.data(), ngram.size())));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
boost::hash_combine(boundary_hash, ngram.back());
|
||||
score[3] += FloorScore(lm_label->lookup_ngram(EigenMap(ngram.data(), ngram.size())));
|
||||
}
|
||||
@ -492,8 +483,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
|
||||
|
||||
if (ancestor_labels.size() >= m_context_up && !num_virtual) {
|
||||
score[0] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram.data(), ngram.size())));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
boost::hash_combine(boundary_hash, ngram.back());
|
||||
score[1] += FloorScore(lm_head->lookup_ngram(EigenMap(ngram.data(), ngram.size())));
|
||||
}
|
||||
@ -519,8 +509,7 @@ void RDLM::Score(InternalTree* root, const TreePointerMap & back_pointers, boost
|
||||
return;
|
||||
}
|
||||
// recursion
|
||||
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it)
|
||||
{
|
||||
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it) {
|
||||
Score(it->get(), back_pointers, score, ancestor_heads, ancestor_labels, boundary_hash, num_virtual, rescoring_levels - 1);
|
||||
}
|
||||
ancestor_heads.pop_back();
|
||||
@ -531,12 +520,10 @@ InternalTree* RDLM::GetHead(InternalTree* root, const TreePointerMap & back_poin
|
||||
{
|
||||
InternalTree *tree;
|
||||
|
||||
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it)
|
||||
{
|
||||
for (std::vector<TreePointer>::const_iterator it = root->GetChildren().begin(); it != root->GetChildren().end(); ++it) {
|
||||
if ((*it)->IsLeafNT()) {
|
||||
tree = back_pointers.find(it->get())->second.get();
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
tree = it->get();
|
||||
}
|
||||
|
||||
@ -563,8 +550,7 @@ InternalTree* RDLM::GetHead(InternalTree* root, const TreePointerMap & back_poin
|
||||
for (std::vector<TreePointer>::const_iterator it2 = tree->GetChildren().begin(); it2 != tree->GetChildren().end(); ++it2) {
|
||||
if ((*it2)->IsLeafNT()) {
|
||||
tree2 = back_pointers.find(it2->get())->second.get();
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
tree2 = it2->get();
|
||||
}
|
||||
if (tree2->GetLabel() == "PTKVZ" && tree2->GetLength() == 1 && tree2->GetChildren()[0]->IsTerminal()) {
|
||||
@ -659,8 +645,7 @@ void RDLM::GetIDs(const std::string & head, const std::string & preterminal, std
|
||||
}
|
||||
if (m_sharedVocab) {
|
||||
IDs.second = IDs.first;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
IDs.second = lm_head_base_instance_->lookup_output_word(head);
|
||||
if (m_isPretermBackoff && IDs.second == 0) {
|
||||
IDs.second = lm_head_base_instance_->lookup_output_word(preterminal);
|
||||
@ -694,8 +679,7 @@ RDLM::TreePointerMap RDLM::AssociateLeafNTs(InternalTree* root, const std::vecto
|
||||
found = next_leafNT(it);
|
||||
if (found) {
|
||||
ret[it->get()] = *it_prev;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
std::cerr << "Warning: leaf nonterminal not found in rule; why did this happen?\n";
|
||||
}
|
||||
}
|
||||
@ -822,8 +806,7 @@ FFState* RDLM::EvaluateWhenApplied(const ChartHypothesis& cur_hypo
|
||||
}
|
||||
|
||||
return new RDLMState(mytree, score[1], score[3], boundary_hash);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
UTIL_THROW2("Error: RDLM active, but no internal tree structure found");
|
||||
}
|
||||
|
||||
|
@ -11,8 +11,9 @@
|
||||
// Sennrich, Rico (2015). Modelling and Optimizing on Syntactic N-Grams for Statistical Machine Translation. Transactions of the Association for Computational Linguistics.
|
||||
// see 'scripts/training/rdlm' for training scripts
|
||||
|
||||
namespace nplm {
|
||||
class neuralTM;
|
||||
namespace nplm
|
||||
{
|
||||
class neuralTM;
|
||||
}
|
||||
|
||||
namespace Moses
|
||||
@ -121,8 +122,7 @@ public:
|
||||
, m_normalizeLabelLM(false)
|
||||
, m_sharedVocab(false)
|
||||
, m_binarized(0)
|
||||
, m_cacheSize(1000000)
|
||||
{
|
||||
, m_cacheSize(1000000) {
|
||||
ReadParameters();
|
||||
}
|
||||
|
||||
@ -161,7 +161,9 @@ public:
|
||||
FFState* EvaluateWhenApplied(
|
||||
const Hypothesis& cur_hypo,
|
||||
const FFState* prev_state,
|
||||
ScoreComponentCollection* accumulator) const {UTIL_THROW(util::Exception, "Not implemented");};
|
||||
ScoreComponentCollection* accumulator) const {
|
||||
UTIL_THROW(util::Exception, "Not implemented");
|
||||
};
|
||||
FFState* EvaluateWhenApplied(
|
||||
const ChartHypothesis& /* cur_hypo */,
|
||||
int /* featureID - used to index the state in the previous hypotheses */,
|
||||
@ -185,8 +187,7 @@ public:
|
||||
UnbinarizedChildren(InternalTree* root, const TreePointerMap & pointers, bool binary):
|
||||
current(root),
|
||||
back_pointers(pointers),
|
||||
binarized(binary)
|
||||
{
|
||||
binarized(binary) {
|
||||
stack.reserve(10);
|
||||
_end = current->GetChildren().end();
|
||||
iter = current->GetChildren().begin();
|
||||
@ -196,8 +197,7 @@ public:
|
||||
// also go through trees or previous hypotheses to rescore nodes for which more context has become available
|
||||
if ((*iter)->IsLeafNT()) {
|
||||
current = back_pointers.find(iter->get())->second.get();
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
current = iter->get();
|
||||
}
|
||||
iter = current->GetChildren().begin();
|
||||
@ -205,8 +205,12 @@ public:
|
||||
_begin = iter;
|
||||
}
|
||||
|
||||
std::vector<TreePointer>::const_iterator begin() const { return _begin; }
|
||||
std::vector<TreePointer>::const_iterator end() const { return _end; }
|
||||
std::vector<TreePointer>::const_iterator begin() const {
|
||||
return _begin;
|
||||
}
|
||||
std::vector<TreePointer>::const_iterator end() const {
|
||||
return _end;
|
||||
}
|
||||
|
||||
std::vector<TreePointer>::const_iterator operator++() {
|
||||
iter++;
|
||||
@ -230,8 +234,7 @@ public:
|
||||
// also go through trees or previous hypotheses to rescore nodes for which more context has become available
|
||||
if ((*iter)->IsLeafNT()) {
|
||||
current = back_pointers.find(iter->get())->second.get();
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
current = iter->get();
|
||||
}
|
||||
iter = current->GetChildren().begin();
|
||||
|
@ -87,7 +87,9 @@ Manager::~Manager()
|
||||
|
||||
const InputType&
|
||||
Manager::GetSource() const
|
||||
{ return m_source ; }
|
||||
{
|
||||
return m_source ;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main decoder loop that translates a sentence by expanding
|
||||
|
@ -110,7 +110,7 @@ private:
|
||||
#endif
|
||||
|
||||
public:
|
||||
void SetOutputStream(std::ostream* outStream){
|
||||
void SetOutputStream(std::ostream* outStream) {
|
||||
m_outStream = outStream;
|
||||
}
|
||||
|
||||
|
@ -368,8 +368,7 @@ AddParam(po::options_description& optgroup,
|
||||
m_fullname[abbrevName] = paramName;
|
||||
m_description[paramName] = description;
|
||||
string optname = paramName;
|
||||
if (abbrevName.size() == 1)
|
||||
{
|
||||
if (abbrevName.size() == 1) {
|
||||
optname += string(",")+abbrevName;
|
||||
// m_confusable[abbrevName[0]].insert(paramName);
|
||||
}
|
||||
@ -429,8 +428,7 @@ LoadParam(int argc, char* xargv[])
|
||||
// legacy parameter handling: all parameters are expected
|
||||
// to start with a single dash
|
||||
char* argv[argc+1];
|
||||
for (int i = 0; i < argc; ++i)
|
||||
{
|
||||
for (int i = 0; i < argc; ++i) {
|
||||
argv[i] = xargv[i];
|
||||
if (strlen(argv[i]) > 2 && argv[i][0] == '-' && argv[i][1] == '-')
|
||||
++argv[i];
|
||||
|
@ -195,14 +195,12 @@ void ScoreComponentCollection::Save(ostream& out, bool multiline) const
|
||||
|
||||
std::vector<FeatureFunction*> const& all_ff
|
||||
= FeatureFunction::GetFeatureFunctions();
|
||||
BOOST_FOREACH(FeatureFunction const* ff, all_ff)
|
||||
{
|
||||
BOOST_FOREACH(FeatureFunction const* ff, all_ff) {
|
||||
string name = ff->GetScoreProducerDescription();
|
||||
size_t i = ff->GetIndex();
|
||||
if (ff->GetNumScoreComponents() == 1)
|
||||
out << name << sep << m_scores[i] << linesep;
|
||||
else
|
||||
{
|
||||
else {
|
||||
size_t stop = i + ff->GetNumScoreComponents();
|
||||
boost::format fmt("%s_%d");
|
||||
for (size_t k = 1; i < stop; ++i, ++k)
|
||||
|
@ -234,7 +234,7 @@ public:
|
||||
const ScoreComponentCollection& scores) {
|
||||
size_t i = sp->GetIndex();
|
||||
size_t stop = i + sp->GetNumScoreComponents();
|
||||
for (;i < stop; ++i) m_scores[i] += scores.m_scores[i];
|
||||
for (; i < stop; ++i) m_scores[i] += scores.m_scores[i];
|
||||
}
|
||||
|
||||
//! Add scores from a single FeatureFunction only
|
||||
|
@ -60,23 +60,18 @@ aux_init_partial_translation(string& line)
|
||||
string sourceCompletedStr;
|
||||
int loc1 = line.find( "|||", 0 );
|
||||
int loc2 = line.find( "|||", loc1 + 3 );
|
||||
if (loc1 > -1 && loc2 > -1)
|
||||
{
|
||||
if (loc1 > -1 && loc2 > -1) {
|
||||
m_initialTargetPhrase = Trim(line.substr(0, loc1));
|
||||
string scov = Trim(line.substr(loc1 + 3, loc2 - loc1 - 3));
|
||||
line = line.substr(loc2 + 3);
|
||||
|
||||
m_sourceCompleted.resize(scov.size());
|
||||
int contiguous = 1;
|
||||
for (size_t i = 0; i < scov.size(); ++i)
|
||||
{
|
||||
if (sourceCompletedStr.at(i) == '1')
|
||||
{
|
||||
for (size_t i = 0; i < scov.size(); ++i) {
|
||||
if (sourceCompletedStr.at(i) == '1') {
|
||||
m_sourceCompleted[i] = true;
|
||||
if (contiguous) m_frontSpanCoveredLength++;
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
m_sourceCompleted[i] = false;
|
||||
contiguous = 0;
|
||||
}
|
||||
@ -94,38 +89,31 @@ aux_interpret_sgml_markup(string& line)
|
||||
metamap::const_iterator i;
|
||||
if ((i = meta.find("id")) != meta.end())
|
||||
this->SetTranslationId(atol(i->second.c_str()));
|
||||
if ((i = meta.find("docid")) != meta.end())
|
||||
{
|
||||
if ((i = meta.find("docid")) != meta.end()) {
|
||||
this->SetDocumentId(atol(i->second.c_str()));
|
||||
this->SetUseTopicId(false);
|
||||
this->SetUseTopicIdAndProb(false);
|
||||
}
|
||||
if ((i = meta.find("topic")) != meta.end())
|
||||
{
|
||||
if ((i = meta.find("topic")) != meta.end()) {
|
||||
vector<string> topic_params;
|
||||
boost::split(topic_params, i->second, boost::is_any_of("\t "));
|
||||
if (topic_params.size() == 1)
|
||||
{
|
||||
if (topic_params.size() == 1) {
|
||||
this->SetTopicId(atol(topic_params[0].c_str()));
|
||||
this->SetUseTopicId(true);
|
||||
this->SetUseTopicIdAndProb(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
this->SetTopicIdAndProb(topic_params);
|
||||
this->SetUseTopicId(false);
|
||||
this->SetUseTopicIdAndProb(true);
|
||||
}
|
||||
}
|
||||
if ((i = meta.find("weight-setting")) != meta.end())
|
||||
{
|
||||
if ((i = meta.find("weight-setting")) != meta.end()) {
|
||||
this->SetWeightSetting(i->second);
|
||||
this->SetSpecifiesWeightSetting(true);
|
||||
StaticData::Instance().SetWeightSetting(i->second);
|
||||
// oh this is so horrible! Why does this have to be propagated globally?
|
||||
// --- UG
|
||||
}
|
||||
else this->SetSpecifiesWeightSetting(false);
|
||||
} else this->SetSpecifiesWeightSetting(false);
|
||||
}
|
||||
|
||||
void
|
||||
@ -135,21 +123,17 @@ aux_interpret_dlt(string& line) // whatever DLT means ... --- UG
|
||||
using namespace std;
|
||||
typedef map<string, string> str2str_map;
|
||||
vector<str2str_map> meta = ProcessAndStripDLT(line);
|
||||
BOOST_FOREACH(str2str_map const& M, meta)
|
||||
{
|
||||
BOOST_FOREACH(str2str_map const& M, meta) {
|
||||
str2str_map::const_iterator i,j;
|
||||
if ((i = M.find("type")) != M.end())
|
||||
{
|
||||
if ((i = M.find("type")) != M.end()) {
|
||||
j = M.find("id");
|
||||
string id = j == M.end() ? "default" : j->second;
|
||||
if (i->second == "cbtm")
|
||||
{
|
||||
if (i->second == "cbtm") {
|
||||
PhraseDictionaryDynamicCacheBased* cbtm;
|
||||
cbtm = PhraseDictionaryDynamicCacheBased::InstanceNonConst(id);
|
||||
if (cbtm) cbtm->ExecuteDlt(M);
|
||||
}
|
||||
if (i->second == "cblm")
|
||||
{
|
||||
if (i->second == "cblm") {
|
||||
DynamicCacheBasedLanguageModel* cblm;
|
||||
cblm = DynamicCacheBasedLanguageModel::InstanceNonConst(id);
|
||||
if (cblm) cblm->ExecuteDlt(M);
|
||||
@ -162,13 +146,13 @@ void
|
||||
Sentence::
|
||||
aux_interpret_xml(std::string& line, std::vector<size_t> & xmlWalls,
|
||||
std::vector<std::pair<size_t, std::string> >& placeholders)
|
||||
{ // parse XML markup in translation line
|
||||
{
|
||||
// parse XML markup in translation line
|
||||
|
||||
const StaticData &SD = StaticData::Instance();
|
||||
|
||||
using namespace std;
|
||||
if (SD.GetXmlInputType() != XmlPassThrough)
|
||||
{
|
||||
if (SD.GetXmlInputType() != XmlPassThrough) {
|
||||
int offset = SD.IsSyntax() ? 1 : 0;
|
||||
bool OK = ProcessAndStripXMLTags(line, m_xmlOptions,
|
||||
m_reorderingConstraint,
|
||||
@ -197,8 +181,7 @@ init(string line, std::vector<FactorType> const& factorOrder)
|
||||
aux_interpret_dlt(line); // some poorly documented cache-based stuff
|
||||
|
||||
// if sentences is specified as "<passthrough tag1=""/>"
|
||||
if (SD.IsPassthroughEnabled() || SD.IsPassthroughInNBestEnabled())
|
||||
{
|
||||
if (SD.IsPassthroughEnabled() || SD.IsPassthroughInNBestEnabled()) {
|
||||
string pthru = PassthroughSGML(line,"passthrough");
|
||||
this->SetPassthroughInformation(pthru);
|
||||
}
|
||||
@ -218,11 +201,9 @@ init(string line, std::vector<FactorType> const& factorOrder)
|
||||
// our XmlOptions and create TranslationOptions
|
||||
|
||||
// only fill the vector if we are parsing XML
|
||||
if (SD.GetXmlInputType() != XmlPassThrough)
|
||||
{
|
||||
if (SD.GetXmlInputType() != XmlPassThrough) {
|
||||
m_xmlCoverageMap.assign(GetSize(), false);
|
||||
BOOST_FOREACH(XmlOption* o, m_xmlOptions)
|
||||
{
|
||||
BOOST_FOREACH(XmlOption* o, m_xmlOptions) {
|
||||
WordsRange const& r = o->range;
|
||||
for(size_t j = r.GetStartPos(); j <= r.GetEndPos(); ++j)
|
||||
m_xmlCoverageMap[j]=true;
|
||||
@ -233,8 +214,7 @@ init(string line, std::vector<FactorType> const& factorOrder)
|
||||
m_reorderingConstraint.InitializeWalls(GetSize());
|
||||
|
||||
// set reordering walls, if "-monotone-at-punction" is set
|
||||
if (SD.UseReorderingConstraint() && GetSize())
|
||||
{
|
||||
if (SD.UseReorderingConstraint() && GetSize()) {
|
||||
WordsRange r(0, GetSize()-1);
|
||||
m_reorderingConstraint.SetMonotoneAtPunctuation(GetSubString(r));
|
||||
}
|
||||
|
@ -32,22 +32,22 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
class WordsRange;
|
||||
class PhraseDictionary;
|
||||
class TranslationOption;
|
||||
class TranslationOptionCollection;
|
||||
class ChartTranslationOptions;
|
||||
class TranslationTask;
|
||||
struct XmlOption;
|
||||
class WordsRange;
|
||||
class PhraseDictionary;
|
||||
class TranslationOption;
|
||||
class TranslationOptionCollection;
|
||||
class ChartTranslationOptions;
|
||||
class TranslationTask;
|
||||
struct XmlOption;
|
||||
|
||||
|
||||
/**
|
||||
/**
|
||||
* A Phrase class with an ID. Used specifically as source input so contains functionality to read
|
||||
* from IODevice and create trans opt
|
||||
*/
|
||||
class Sentence : public Phrase, public InputType
|
||||
{
|
||||
protected:
|
||||
class Sentence : public Phrase, public InputType
|
||||
{
|
||||
protected:
|
||||
|
||||
/**
|
||||
* Utility method that takes in a string representing an XML tag and the name of the attribute,
|
||||
@ -61,7 +61,7 @@ namespace Moses
|
||||
void ProcessPlaceholders(const std::vector< std::pair<size_t, std::string> > &placeholders);
|
||||
|
||||
|
||||
public:
|
||||
public:
|
||||
Sentence();
|
||||
Sentence(size_t const transId, std::string const& stext,
|
||||
std::vector<FactorType> const* IFO = NULL);
|
||||
@ -106,14 +106,15 @@ namespace Moses
|
||||
std::string const& phraseString);
|
||||
|
||||
const NonTerminalSet&
|
||||
GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const
|
||||
{ return m_defaultLabelSet; }
|
||||
GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const {
|
||||
return m_defaultLabelSet;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
init(std::string line, std::vector<FactorType> const& factorOrder);
|
||||
|
||||
private:
|
||||
private:
|
||||
// auxliliary functions for Sentence initialization
|
||||
// void aux_interpret_sgml_markup(std::string& line);
|
||||
// void aux_interpret_dlt(std::string& line);
|
||||
@ -134,7 +135,7 @@ namespace Moses
|
||||
void
|
||||
aux_init_partial_translation(std::string& line);
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
@ -476,8 +476,7 @@ public:
|
||||
// m_searchAlgorithm == SyntaxF2S;
|
||||
// }
|
||||
|
||||
bool IsSyntax(SearchAlgorithm algo = DefaultSearchAlgorithm) const
|
||||
{
|
||||
bool IsSyntax(SearchAlgorithm algo = DefaultSearchAlgorithm) const {
|
||||
if (algo == DefaultSearchAlgorithm)
|
||||
algo = m_searchAlgorithm;
|
||||
return (algo == CYKPlus || algo == ChartIncremental ||
|
||||
@ -486,8 +485,9 @@ public:
|
||||
}
|
||||
|
||||
const ScoreComponentCollection&
|
||||
GetAllWeights() const
|
||||
{ return m_allWeights; }
|
||||
GetAllWeights() const {
|
||||
return m_allWeights;
|
||||
}
|
||||
|
||||
void SetAllWeights(const ScoreComponentCollection& weights) {
|
||||
m_allWeights = weights;
|
||||
|
@ -232,11 +232,9 @@ mergescores(boost::shared_ptr<Scores> const& a,
|
||||
if (!b) return a;
|
||||
if (a->size() != b->size()) return ret;
|
||||
ret.reset(new Scores(*a));
|
||||
for (size_t i = 0; i < a->size(); ++i)
|
||||
{
|
||||
for (size_t i = 0; i < a->size(); ++i) {
|
||||
if ((*a)[i] == 0) (*a)[i] = (*b)[i];
|
||||
else if ((*b)[i])
|
||||
{
|
||||
else if ((*b)[i]) {
|
||||
UTIL_THROW_IF2((*a)[i] != (*b)[i], "can't merge feature vectors");
|
||||
}
|
||||
}
|
||||
@ -253,8 +251,7 @@ Merge(const TargetPhrase ©, const std::vector<FactorType>& factorVec)
|
||||
m_fullScore += copy.m_fullScore;
|
||||
typedef ScoreCache_t::iterator iter;
|
||||
typedef ScoreCache_t::value_type item;
|
||||
BOOST_FOREACH(item const& s, copy.m_cached_scores)
|
||||
{
|
||||
BOOST_FOREACH(item const& s, copy.m_cached_scores) {
|
||||
pair<iter,bool> foo = m_cached_scores.insert(s);
|
||||
if (foo.second == false)
|
||||
foo.first->second = mergescores(foo.first->second, s.second);
|
||||
@ -280,7 +277,9 @@ void
|
||||
TargetPhrase::
|
||||
SetExtraScores(FeatureFunction const* ff,
|
||||
boost::shared_ptr<Scores> const& s)
|
||||
{ m_cached_scores[ff] = s; }
|
||||
{
|
||||
m_cached_scores[ff] = s;
|
||||
}
|
||||
|
||||
|
||||
void TargetPhrase::SetProperties(const StringPiece &str)
|
||||
|
@ -51,7 +51,7 @@ class PhraseDictionary;
|
||||
*/
|
||||
class TargetPhrase: public Phrase
|
||||
{
|
||||
public:
|
||||
public:
|
||||
typedef std::map<FeatureFunction const*, boost::shared_ptr<Scores> >
|
||||
ScoreCache_t;
|
||||
ScoreCache_t const& GetExtraScores() const;
|
||||
@ -59,7 +59,7 @@ class TargetPhrase: public Phrase
|
||||
void SetExtraScores(FeatureFunction const* ff,
|
||||
boost::shared_ptr<Scores> const& scores);
|
||||
|
||||
private:
|
||||
private:
|
||||
ScoreCache_t m_cached_scores;
|
||||
|
||||
private:
|
||||
|
@ -26,8 +26,7 @@ public:
|
||||
|
||||
// factory function
|
||||
static boost::shared_ptr<TrainingTask>
|
||||
create(boost::shared_ptr<InputType> const& source)
|
||||
{
|
||||
create(boost::shared_ptr<InputType> const& source) {
|
||||
boost::shared_ptr<IOWrapper> nix;
|
||||
boost::shared_ptr<TrainingTask> ret(new TrainingTask(source, nix));
|
||||
ret->m_self = ret;
|
||||
@ -37,8 +36,7 @@ public:
|
||||
// factory function
|
||||
static boost::shared_ptr<TrainingTask>
|
||||
create(boost::shared_ptr<InputType> const& source,
|
||||
boost::shared_ptr<IOWrapper> const& ioWrapper)
|
||||
{
|
||||
boost::shared_ptr<IOWrapper> const& ioWrapper) {
|
||||
boost::shared_ptr<TrainingTask> ret(new TrainingTask(source, ioWrapper));
|
||||
ret->m_self = ret;
|
||||
return ret;
|
||||
|
@ -117,8 +117,7 @@ public:
|
||||
|
||||
virtual
|
||||
TargetPhraseCollection const *
|
||||
GetTargetPhraseCollectionLEGACY(ttasksptr const& ttask, const Phrase& src)
|
||||
{
|
||||
GetTargetPhraseCollectionLEGACY(ttasksptr const& ttask, const Phrase& src) {
|
||||
return GetTargetPhraseCollectionLEGACY(src);
|
||||
}
|
||||
|
||||
@ -129,8 +128,7 @@ public:
|
||||
virtual
|
||||
void
|
||||
GetTargetPhraseCollectionBatch(ttasksptr const& ttask,
|
||||
const InputPathList &inputPathQueue) const
|
||||
{
|
||||
const InputPathList &inputPathQueue) const {
|
||||
GetTargetPhraseCollectionBatch(inputPathQueue);
|
||||
}
|
||||
|
||||
|
@ -109,7 +109,7 @@ ostream& operator<<(ostream& out, const TranslationOption& possibleTranslation)
|
||||
return out;
|
||||
}
|
||||
|
||||
/** returns cached scores */
|
||||
/** returns cached scores */
|
||||
const Scores*
|
||||
TranslationOption::
|
||||
GetLexReorderingScores(LexicalReordering const* scoreProducer) const
|
||||
|
@ -626,8 +626,7 @@ CacheLexReordering()
|
||||
{
|
||||
size_t const stop = m_source.GetSize();
|
||||
typedef StatefulFeatureFunction sfFF;
|
||||
BOOST_FOREACH(sfFF const* ff, sfFF::GetStatefulFeatureFunctions())
|
||||
{
|
||||
BOOST_FOREACH(sfFF const* ff, sfFF::GetStatefulFeatureFunctions()) {
|
||||
if (typeid(*ff) != typeid(LexicalReordering)) continue;
|
||||
LexicalReordering const& lr = static_cast<const LexicalReordering&>(*ff);
|
||||
for (size_t s = 0 ; s < stop ; s++)
|
||||
|
@ -177,8 +177,7 @@ public:
|
||||
return m_inputPathQueue;
|
||||
}
|
||||
|
||||
ttasksptr GetTranslationTask() const
|
||||
{
|
||||
ttasksptr GetTranslationTask() const {
|
||||
return m_ttask.lock();
|
||||
}
|
||||
TO_STRING();
|
||||
|
@ -82,33 +82,29 @@ TranslationTask
|
||||
if (!staticData.IsSyntax(algo))
|
||||
manager.reset(new Manager(this->self())); // phrase-based
|
||||
|
||||
else if (algo == SyntaxF2S || algo == SyntaxT2S)
|
||||
{ // STSG-based tree-to-string / forest-to-string decoding (ask Phil Williams)
|
||||
else if (algo == SyntaxF2S || algo == SyntaxT2S) {
|
||||
// STSG-based tree-to-string / forest-to-string decoding (ask Phil Williams)
|
||||
typedef Syntax::F2S::RuleMatcherCallback Callback;
|
||||
typedef Syntax::F2S::RuleMatcherHyperTree<Callback> RuleMatcher;
|
||||
manager.reset(new Syntax::F2S::Manager<RuleMatcher>(this->self()));
|
||||
}
|
||||
|
||||
else if (algo == SyntaxS2T)
|
||||
{ // new-style string-to-tree decoding (ask Phil Williams)
|
||||
else if (algo == SyntaxS2T) {
|
||||
// new-style string-to-tree decoding (ask Phil Williams)
|
||||
S2TParsingAlgorithm algorithm = staticData.GetS2TParsingAlgorithm();
|
||||
if (algorithm == RecursiveCYKPlus)
|
||||
{
|
||||
if (algorithm == RecursiveCYKPlus) {
|
||||
typedef Syntax::S2T::EagerParserCallback Callback;
|
||||
typedef Syntax::S2T::RecursiveCYKPlusParser<Callback> Parser;
|
||||
manager.reset(new Syntax::S2T::Manager<Parser>(this->self()));
|
||||
}
|
||||
else if (algorithm == Scope3)
|
||||
{
|
||||
} else if (algorithm == Scope3) {
|
||||
typedef Syntax::S2T::StandardParserCallback Callback;
|
||||
typedef Syntax::S2T::Scope3Parser<Callback> Parser;
|
||||
manager.reset(new Syntax::S2T::Manager<Parser>(this->self()));
|
||||
}
|
||||
else UTIL_THROW2("ERROR: unhandled S2T parsing algorithm");
|
||||
} else UTIL_THROW2("ERROR: unhandled S2T parsing algorithm");
|
||||
}
|
||||
|
||||
else if (algo == SyntaxT2S_SCFG)
|
||||
{ // SCFG-based tree-to-string decoding (ask Phil Williams)
|
||||
else if (algo == SyntaxT2S_SCFG) {
|
||||
// SCFG-based tree-to-string decoding (ask Phil Williams)
|
||||
typedef Syntax::F2S::RuleMatcherCallback Callback;
|
||||
typedef Syntax::T2S::RuleMatcherSCFG<Callback> RuleMatcher;
|
||||
manager.reset(new Syntax::T2S::Manager<RuleMatcher>(this->self()));
|
||||
|
@ -40,7 +40,9 @@ class TranslationTask : public Moses::Task
|
||||
TranslationTask(TranslationTask const& other) { }
|
||||
|
||||
TranslationTask const&
|
||||
operator=(TranslationTask const& other) { return *this; }
|
||||
operator=(TranslationTask const& other) {
|
||||
return *this;
|
||||
}
|
||||
|
||||
protected:
|
||||
boost::weak_ptr<TranslationTask> m_self; // weak ptr to myself
|
||||
@ -68,11 +70,15 @@ protected:
|
||||
public:
|
||||
|
||||
boost::shared_ptr<TranslationTask>
|
||||
self() { return m_self.lock(); }
|
||||
self() {
|
||||
return m_self.lock();
|
||||
}
|
||||
|
||||
virtual
|
||||
boost::shared_ptr<TranslationTask const>
|
||||
self() const { return m_self.lock(); }
|
||||
self() const {
|
||||
return m_self.lock();
|
||||
}
|
||||
|
||||
// creator functions
|
||||
static boost::shared_ptr<TranslationTask> create();
|
||||
@ -92,15 +98,16 @@ public:
|
||||
virtual void Run();
|
||||
|
||||
boost::shared_ptr<Moses::InputType>
|
||||
GetSource() const { return m_source; }
|
||||
GetSource() const {
|
||||
return m_source;
|
||||
}
|
||||
|
||||
boost::shared_ptr<BaseManager>
|
||||
SetupManager(SearchAlgorithm algo = DefaultSearchAlgorithm);
|
||||
|
||||
|
||||
boost::shared_ptr<ContextScope> const&
|
||||
GetScope() const
|
||||
{
|
||||
GetScope() const {
|
||||
UTIL_THROW_IF2(m_scope == NULL, "No context scope!");
|
||||
return m_scope;
|
||||
}
|
||||
|
@ -8,7 +8,7 @@
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
class TranslationTask;
|
||||
class TranslationTask;
|
||||
//! @todo what is this?
|
||||
class XMLParseOutput
|
||||
{
|
||||
|
@ -3,36 +3,34 @@
|
||||
|
||||
namespace MosesServer
|
||||
{
|
||||
using namespace std;
|
||||
using namespace std;
|
||||
|
||||
Optimizer::
|
||||
Optimizer()
|
||||
{
|
||||
Optimizer::
|
||||
Optimizer()
|
||||
{
|
||||
// signature and help strings are documentation -- the client
|
||||
// can query this information with a system.methodSignature and
|
||||
// system.methodHelp RPC.
|
||||
this->_signature = "S:S";
|
||||
this->_help = "Optimizes multi-model translation model";
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Optimizer::
|
||||
execute(xmlrpc_c::paramList const& paramList,
|
||||
void
|
||||
Optimizer::
|
||||
execute(xmlrpc_c::paramList const& paramList,
|
||||
xmlrpc_c::value * const retvalP)
|
||||
{
|
||||
{
|
||||
#ifdef WITH_DLIB
|
||||
const params_t params = paramList.getStruct(0);
|
||||
params_t::const_iterator si;
|
||||
if ((si = params.find("model_name")) == params.end())
|
||||
{
|
||||
if ((si = params.find("model_name")) == params.end()) {
|
||||
string msg = "Missing name of model to be optimized";
|
||||
msg += " (e.g. PhraseDictionaryMultiModelCounts0)";
|
||||
throw xmlrpc_c::fault(msg, xmlrpc_c::fault::CODE_PARSE);
|
||||
}
|
||||
const string model_name = xmlrpc_c::value_string(si->second);
|
||||
|
||||
if ((si = params.find("phrase_pairs")) == params.end())
|
||||
{
|
||||
if ((si = params.find("phrase_pairs")) == params.end()) {
|
||||
throw xmlrpc_c::fault("Missing list of phrase pairs",
|
||||
xmlrpc_c::fault::CODE_PARSE);
|
||||
}
|
||||
@ -42,8 +40,7 @@ namespace MosesServer
|
||||
|
||||
xmlrpc_c::value_array pp_array = xmlrpc_c::value_array(si->second);
|
||||
vector<xmlrpc_c::value> ppValVec(pp_array.vectorValueValue());
|
||||
for (size_t i = 0; i < ppValVec.size(); ++i)
|
||||
{
|
||||
for (size_t i = 0; i < ppValVec.size(); ++i) {
|
||||
xmlrpc_c::value_array pp_array
|
||||
= xmlrpc_c::value_array(ppValVec[i]);
|
||||
vector<xmlrpc_c::value> pp(pp_array.vectorValueValue());
|
||||
@ -58,7 +55,7 @@ namespace MosesServer
|
||||
vector<float> weight_vector = pdmm->MinimizePerplexity(phrase_pairs);
|
||||
|
||||
vector<xmlrpc_c::value> weight_vector_ret;
|
||||
for (size_t i=0;i < weight_vector.size();i++)
|
||||
for (size_t i=0; i < weight_vector.size(); i++)
|
||||
weight_vector_ret.push_back(xmlrpc_c::value_double(weight_vector[i]));
|
||||
|
||||
*retvalP = xmlrpc_c::value_array(weight_vector_ret);
|
||||
@ -68,5 +65,5 @@ namespace MosesServer
|
||||
std::cerr << errmsg << std::endl;
|
||||
*retvalP = xmlrpc_c::value_string(errmsg);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -6,12 +6,12 @@
|
||||
|
||||
namespace MosesServer
|
||||
{
|
||||
class
|
||||
class
|
||||
Optimizer : public xmlrpc_c::method
|
||||
{
|
||||
public:
|
||||
{
|
||||
public:
|
||||
Optimizer();
|
||||
void execute(xmlrpc_c::paramList const& paramList,
|
||||
xmlrpc_c::value * const retvalP);
|
||||
};
|
||||
};
|
||||
}
|
||||
|
@ -3,40 +3,40 @@
|
||||
|
||||
namespace MosesServer
|
||||
{
|
||||
using namespace std;
|
||||
using Moses::Hypothesis;
|
||||
using Moses::StaticData;
|
||||
using Moses::WordsRange;
|
||||
using Moses::ChartHypothesis;
|
||||
using Moses::Phrase;
|
||||
using Moses::Manager;
|
||||
using Moses::SearchGraphNode;
|
||||
using Moses::TrellisPathList;
|
||||
using Moses::TranslationOptionCollection;
|
||||
using Moses::TranslationOptionList;
|
||||
using Moses::TranslationOption;
|
||||
using Moses::TargetPhrase;
|
||||
using Moses::FValue;
|
||||
using Moses::PhraseDictionaryMultiModel;
|
||||
using Moses::FindPhraseDictionary;
|
||||
using Moses::Sentence;
|
||||
using namespace std;
|
||||
using Moses::Hypothesis;
|
||||
using Moses::StaticData;
|
||||
using Moses::WordsRange;
|
||||
using Moses::ChartHypothesis;
|
||||
using Moses::Phrase;
|
||||
using Moses::Manager;
|
||||
using Moses::SearchGraphNode;
|
||||
using Moses::TrellisPathList;
|
||||
using Moses::TranslationOptionCollection;
|
||||
using Moses::TranslationOptionList;
|
||||
using Moses::TranslationOption;
|
||||
using Moses::TargetPhrase;
|
||||
using Moses::FValue;
|
||||
using Moses::PhraseDictionaryMultiModel;
|
||||
using Moses::FindPhraseDictionary;
|
||||
using Moses::Sentence;
|
||||
|
||||
boost::shared_ptr<TranslationRequest>
|
||||
TranslationRequest::
|
||||
create(xmlrpc_c::paramList const& paramList,
|
||||
boost::shared_ptr<TranslationRequest>
|
||||
TranslationRequest::
|
||||
create(xmlrpc_c::paramList const& paramList,
|
||||
boost::condition_variable& cond,
|
||||
boost::mutex& mut)
|
||||
{
|
||||
{
|
||||
boost::shared_ptr<TranslationRequest> ret;
|
||||
ret.reset(new TranslationRequest(paramList,cond, mut));
|
||||
ret->m_self = ret;
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
TranslationRequest::
|
||||
Run()
|
||||
{
|
||||
void
|
||||
TranslationRequest::
|
||||
Run()
|
||||
{
|
||||
parse_request(m_paramList.getStruct(0));
|
||||
|
||||
Moses::StaticData const& SD = Moses::StaticData::Instance();
|
||||
@ -60,13 +60,13 @@ namespace MosesServer
|
||||
}
|
||||
m_cond.notify_one();
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/// add phrase alignment information from a Hypothesis
|
||||
void
|
||||
TranslationRequest::
|
||||
add_phrase_aln_info(Hypothesis const& h, vector<xmlrpc_c::value>& aInfo) const
|
||||
{
|
||||
/// add phrase alignment information from a Hypothesis
|
||||
void
|
||||
TranslationRequest::
|
||||
add_phrase_aln_info(Hypothesis const& h, vector<xmlrpc_c::value>& aInfo) const
|
||||
{
|
||||
if (!m_withAlignInfo) return;
|
||||
WordsRange const& trg = h.GetCurrTargetWordsRange();
|
||||
WordsRange const& src = h.GetCurrSourceWordsRange();
|
||||
@ -76,12 +76,12 @@ namespace MosesServer
|
||||
pAlnInfo["src-start"] = xmlrpc_c::value_int(src.GetStartPos());
|
||||
pAlnInfo["src-end"] = xmlrpc_c::value_int(src.GetEndPos());
|
||||
aInfo.push_back(xmlrpc_c::value_struct(pAlnInfo));
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
TranslationRequest::
|
||||
outputChartHypo(ostream& out, const ChartHypothesis* hypo)
|
||||
{
|
||||
void
|
||||
TranslationRequest::
|
||||
outputChartHypo(ostream& out, const ChartHypothesis* hypo)
|
||||
{
|
||||
Phrase outPhrase(20);
|
||||
hypo->GetOutputPhrase(outPhrase);
|
||||
|
||||
@ -91,18 +91,20 @@ namespace MosesServer
|
||||
outPhrase.RemoveWord(outPhrase.GetSize() - 1);
|
||||
for (size_t pos = 0 ; pos < outPhrase.GetSize() ; pos++)
|
||||
out << *outPhrase.GetFactor(pos, 0) << " ";
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
TranslationRequest::
|
||||
compareSearchGraphNode(const Moses::SearchGraphNode& a,
|
||||
bool
|
||||
TranslationRequest::
|
||||
compareSearchGraphNode(const Moses::SearchGraphNode& a,
|
||||
const Moses::SearchGraphNode& b)
|
||||
{ return a.hypo->GetId() < b.hypo->GetId(); }
|
||||
{
|
||||
return a.hypo->GetId() < b.hypo->GetId();
|
||||
}
|
||||
|
||||
void
|
||||
TranslationRequest::
|
||||
insertGraphInfo(Manager& manager, map<string, xmlrpc_c::value>& retData)
|
||||
{
|
||||
void
|
||||
TranslationRequest::
|
||||
insertGraphInfo(Manager& manager, map<string, xmlrpc_c::value>& retData)
|
||||
{
|
||||
using xmlrpc_c::value_int;
|
||||
using xmlrpc_c::value_double;
|
||||
using xmlrpc_c::value_struct;
|
||||
@ -111,16 +113,14 @@ namespace MosesServer
|
||||
vector<SearchGraphNode> searchGraph;
|
||||
manager.GetSearchGraph(searchGraph);
|
||||
std::sort(searchGraph.begin(), searchGraph.end());
|
||||
BOOST_FOREACH(Moses::SearchGraphNode const& n, searchGraph)
|
||||
{
|
||||
BOOST_FOREACH(Moses::SearchGraphNode const& n, searchGraph) {
|
||||
map<string, xmlrpc_c::value> x; // search graph xml node
|
||||
x["forward"] = value_double(n.forward);
|
||||
x["fscore"] = value_double(n.fscore);
|
||||
const Hypothesis* hypo = n.hypo;
|
||||
x["hyp"] = value_int(hypo->GetId());
|
||||
x["stack"] = value_int(hypo->GetWordsBitmap().GetNumWordsCovered());
|
||||
if (hypo->GetId() != 0)
|
||||
{
|
||||
if (hypo->GetId() != 0) {
|
||||
const Hypothesis *prevHypo = hypo->GetPrevHypo();
|
||||
x["back"] = value_int(prevHypo->GetId());
|
||||
x["score"] = value_double(hypo->GetScore());
|
||||
@ -134,36 +134,32 @@ namespace MosesServer
|
||||
searchGraphXml.push_back(value_struct(x));
|
||||
}
|
||||
retData["sg"] = xmlrpc_c::value_array(searchGraphXml);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
TranslationRequest::
|
||||
output_phrase(ostream& out, Phrase const& phrase) const
|
||||
{
|
||||
if (!m_reportAllFactors)
|
||||
{
|
||||
void
|
||||
TranslationRequest::
|
||||
output_phrase(ostream& out, Phrase const& phrase) const
|
||||
{
|
||||
if (!m_reportAllFactors) {
|
||||
for (size_t i = 0 ; i < phrase.GetSize(); ++i)
|
||||
out << *phrase.GetFactor(i, 0) << " ";
|
||||
}
|
||||
else out << phrase;
|
||||
}
|
||||
} else out << phrase;
|
||||
}
|
||||
|
||||
void
|
||||
TranslationRequest::
|
||||
outputNBest(const Manager& manager, map<string, xmlrpc_c::value>& retData)
|
||||
{
|
||||
void
|
||||
TranslationRequest::
|
||||
outputNBest(const Manager& manager, map<string, xmlrpc_c::value>& retData)
|
||||
{
|
||||
TrellisPathList nBestList;
|
||||
vector<xmlrpc_c::value> nBestXml;
|
||||
manager.CalcNBest(m_nbestSize, nBestList, m_nbestDistinct);
|
||||
|
||||
BOOST_FOREACH(Moses::TrellisPath const* path, nBestList)
|
||||
{
|
||||
BOOST_FOREACH(Moses::TrellisPath const* path, nBestList) {
|
||||
vector<const Hypothesis *> const& E = path->GetEdges();
|
||||
if (!E.size()) continue;
|
||||
std::map<std::string, xmlrpc_c::value> nBestXmlItem;
|
||||
pack_hypothesis(E, "hyp", nBestXmlItem);
|
||||
if (m_withScoreBreakdown)
|
||||
{
|
||||
if (m_withScoreBreakdown) {
|
||||
// should the score breakdown be reported in a more structured manner?
|
||||
ostringstream buf;
|
||||
path->GetScoreBreakdown()->OutputAllFeatureScores(buf);
|
||||
@ -175,26 +171,23 @@ namespace MosesServer
|
||||
nBestXml.push_back(xmlrpc_c::value_struct(nBestXmlItem));
|
||||
}
|
||||
retData["nbest"] = xmlrpc_c::value_array(nBestXml);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
TranslationRequest::
|
||||
insertTranslationOptions(Moses::Manager& manager,
|
||||
void
|
||||
TranslationRequest::
|
||||
insertTranslationOptions(Moses::Manager& manager,
|
||||
std::map<std::string, xmlrpc_c::value>& retData)
|
||||
{
|
||||
{
|
||||
const TranslationOptionCollection* toptsColl
|
||||
= manager.getSntTranslationOptions();
|
||||
vector<xmlrpc_c::value> toptsXml;
|
||||
size_t const stop = toptsColl->GetSource().GetSize();
|
||||
TranslationOptionList const* tol;
|
||||
for (size_t s = 0 ; s < stop ; ++s)
|
||||
{
|
||||
for (size_t s = 0 ; s < stop ; ++s) {
|
||||
for (size_t e = s;
|
||||
(tol = toptsColl->GetTranslationOptionList(s,e)) != NULL;
|
||||
++e)
|
||||
{
|
||||
BOOST_FOREACH(TranslationOption const* topt, *tol)
|
||||
{
|
||||
++e) {
|
||||
BOOST_FOREACH(TranslationOption const* topt, *tol) {
|
||||
std::map<std::string, xmlrpc_c::value> toptXml;
|
||||
TargetPhrase const& tp = topt->GetTargetPhrase();
|
||||
StaticData const& GLOBAL = StaticData::Instance();
|
||||
@ -215,25 +208,26 @@ namespace MosesServer
|
||||
}
|
||||
}
|
||||
retData["topt"] = xmlrpc_c::value_array(toptsXml);
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
check(std::map<std::string, xmlrpc_c::value> const& params, std::string const key)
|
||||
{
|
||||
bool
|
||||
check(std::map<std::string, xmlrpc_c::value> const& params, std::string const key)
|
||||
{
|
||||
std::map<std::string, xmlrpc_c::value>::const_iterator m;
|
||||
return (params.find(key) != params.end());
|
||||
}
|
||||
}
|
||||
|
||||
TranslationRequest::
|
||||
TranslationRequest(xmlrpc_c::paramList const& paramList,
|
||||
TranslationRequest::
|
||||
TranslationRequest(xmlrpc_c::paramList const& paramList,
|
||||
boost::condition_variable& cond, boost::mutex& mut)
|
||||
: m_cond(cond), m_mutex(mut), m_done(false), m_paramList(paramList)
|
||||
{ }
|
||||
{ }
|
||||
|
||||
void
|
||||
TranslationRequest::
|
||||
parse_request(std::map<std::string, xmlrpc_c::value> const& params)
|
||||
{ // parse XMLRPC request
|
||||
void
|
||||
TranslationRequest::
|
||||
parse_request(std::map<std::string, xmlrpc_c::value> const& params)
|
||||
{
|
||||
// parse XMLRPC request
|
||||
// params_t const params = m_paramList.getStruct(0);
|
||||
m_paramList.verifyEnd(1); // ??? UG
|
||||
|
||||
@ -254,16 +248,14 @@ namespace MosesServer
|
||||
m_withScoreBreakdown = check(params, "add-score-breakdown");
|
||||
m_source.reset(new Sentence(0,m_source_string));
|
||||
si = params.find("lambda");
|
||||
if (si != params.end())
|
||||
{
|
||||
if (si != params.end()) {
|
||||
// muMo = multiModel
|
||||
xmlrpc_c::value_array muMoArray = xmlrpc_c::value_array(si->second);
|
||||
vector<xmlrpc_c::value> muMoValVec(muMoArray.vectorValueValue());
|
||||
vector<float> w(muMoValVec.size());
|
||||
for (size_t i = 0; i < muMoValVec.size(); ++i)
|
||||
w[i] = xmlrpc_c::value_double(muMoValVec[i]);
|
||||
if (w.size() && (si = params.find("model_name")) != params.end())
|
||||
{
|
||||
if (w.size() && (si = params.find("model_name")) != params.end()) {
|
||||
string const model_name = xmlrpc_c::value_string(si->second);
|
||||
PhraseDictionaryMultiModel* pdmm
|
||||
= (PhraseDictionaryMultiModel*) FindPhraseDictionary(model_name);
|
||||
@ -281,13 +273,13 @@ namespace MosesServer
|
||||
// for (size_t i = 1; i < tmp.size(); i += 2)
|
||||
// m_bias[xmlrpc_c::value_int(tmp[i-1])] = xmlrpc_c::value_double(tmp[i]);
|
||||
// }
|
||||
} // end of Translationtask::parse_request()
|
||||
} // end of Translationtask::parse_request()
|
||||
|
||||
|
||||
void
|
||||
TranslationRequest::
|
||||
run_chart_decoder()
|
||||
{
|
||||
void
|
||||
TranslationRequest::
|
||||
run_chart_decoder()
|
||||
{
|
||||
Moses::TreeInput tinput;
|
||||
istringstream buf(m_source_string + "\n");
|
||||
tinput.Read(buf, StaticData::Instance().GetInputFactorOrder());
|
||||
@ -302,27 +294,26 @@ namespace MosesServer
|
||||
m_target_string = out.str();
|
||||
m_retData["text"] = xmlrpc_c::value_string(m_target_string);
|
||||
|
||||
if (m_withGraphInfo)
|
||||
{
|
||||
if (m_withGraphInfo) {
|
||||
std::ostringstream sgstream;
|
||||
manager.OutputSearchGraphMoses(sgstream);
|
||||
m_retData["sg"] = xmlrpc_c::value_string(sgstream.str());
|
||||
}
|
||||
} // end of TranslationRequest::run_chart_decoder()
|
||||
} // end of TranslationRequest::run_chart_decoder()
|
||||
|
||||
void
|
||||
TranslationRequest::
|
||||
pack_hypothesis(vector<Hypothesis const* > const& edges, string const& key,
|
||||
void
|
||||
TranslationRequest::
|
||||
pack_hypothesis(vector<Hypothesis const* > const& edges, string const& key,
|
||||
map<string, xmlrpc_c::value> & dest) const
|
||||
{
|
||||
{
|
||||
// target string
|
||||
ostringstream target;
|
||||
BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
|
||||
output_phrase(target, e->GetCurrTargetPhrase());
|
||||
dest[key] = xmlrpc_c::value_string(target.str());
|
||||
|
||||
if (m_withAlignInfo)
|
||||
{ // phrase alignment, if requested
|
||||
if (m_withAlignInfo) {
|
||||
// phrase alignment, if requested
|
||||
|
||||
vector<xmlrpc_c::value> p_aln;
|
||||
BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
|
||||
@ -330,32 +321,32 @@ namespace MosesServer
|
||||
dest["align"] = xmlrpc_c::value_array(p_aln);
|
||||
}
|
||||
|
||||
if (m_withWordAlignInfo)
|
||||
{ // word alignment, if requested
|
||||
if (m_withWordAlignInfo) {
|
||||
// word alignment, if requested
|
||||
vector<xmlrpc_c::value> w_aln;
|
||||
BOOST_FOREACH(Hypothesis const* e, edges)
|
||||
e->OutputLocalWordAlignment(w_aln);
|
||||
dest["word-align"] = xmlrpc_c::value_array(w_aln);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
TranslationRequest::
|
||||
pack_hypothesis(Hypothesis const* h, string const& key,
|
||||
void
|
||||
TranslationRequest::
|
||||
pack_hypothesis(Hypothesis const* h, string const& key,
|
||||
map<string, xmlrpc_c::value>& dest) const
|
||||
{
|
||||
{
|
||||
using namespace std;
|
||||
vector<Hypothesis const*> edges;
|
||||
for (;h; h = h->GetPrevHypo())
|
||||
for (; h; h = h->GetPrevHypo())
|
||||
edges.push_back(h);
|
||||
pack_hypothesis(edges, key, dest);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
TranslationRequest::
|
||||
run_phrase_decoder()
|
||||
{
|
||||
void
|
||||
TranslationRequest::
|
||||
run_phrase_decoder()
|
||||
{
|
||||
Manager manager(this->self());
|
||||
// if (m_bias.size()) manager.SetBias(&m_bias);
|
||||
manager.Decode();
|
||||
@ -370,5 +361,5 @@ namespace MosesServer
|
||||
.SetOutputSearchGraph(false);
|
||||
// WTF? one more reason not to have this as global variable! --- UG
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -23,9 +23,9 @@
|
||||
#include <xmlrpc-c/base.hpp>
|
||||
namespace MosesServer
|
||||
{
|
||||
class
|
||||
class
|
||||
TranslationRequest : public virtual Moses::TranslationTask
|
||||
{
|
||||
{
|
||||
boost::condition_variable& m_cond;
|
||||
boost::mutex& m_mutex;
|
||||
bool m_done;
|
||||
@ -90,12 +90,12 @@ namespace MosesServer
|
||||
void
|
||||
insertTranslationOptions(Moses::Manager& manager,
|
||||
std::map<std::string, xmlrpc_c::value>& retData);
|
||||
protected:
|
||||
protected:
|
||||
TranslationRequest(xmlrpc_c::paramList const& paramList,
|
||||
boost::condition_variable& cond,
|
||||
boost::mutex& mut);
|
||||
|
||||
public:
|
||||
public:
|
||||
|
||||
static
|
||||
boost::shared_ptr<TranslationRequest>
|
||||
@ -105,18 +105,24 @@ namespace MosesServer
|
||||
|
||||
|
||||
virtual bool
|
||||
DeleteAfterExecution() { return false; }
|
||||
DeleteAfterExecution() {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool
|
||||
IsDone() const { return m_done; }
|
||||
IsDone() const {
|
||||
return m_done;
|
||||
}
|
||||
|
||||
std::map<std::string, xmlrpc_c::value> const&
|
||||
GetRetData() { return m_retData; }
|
||||
GetRetData() {
|
||||
return m_retData;
|
||||
}
|
||||
|
||||
void
|
||||
Run();
|
||||
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -4,25 +4,25 @@
|
||||
namespace MosesServer
|
||||
{
|
||||
|
||||
using namespace std;
|
||||
using namespace Moses;
|
||||
using namespace std;
|
||||
using namespace Moses;
|
||||
|
||||
Translator::
|
||||
Translator(size_t numThreads)
|
||||
Translator::
|
||||
Translator(size_t numThreads)
|
||||
: m_threadPool(numThreads)
|
||||
{
|
||||
{
|
||||
// signature and help strings are documentation -- the client
|
||||
// can query this information with a system.methodSignature and
|
||||
// system.methodHelp RPC.
|
||||
this->_signature = "S:S";
|
||||
this->_help = "Does translation";
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Translator::
|
||||
execute(xmlrpc_c::paramList const& paramList,
|
||||
void
|
||||
Translator::
|
||||
execute(xmlrpc_c::paramList const& paramList,
|
||||
xmlrpc_c::value * const retvalP)
|
||||
{
|
||||
{
|
||||
boost::condition_variable cond;
|
||||
boost::mutex mut;
|
||||
boost::shared_ptr<TranslationRequest> task
|
||||
@ -32,6 +32,6 @@ namespace MosesServer
|
||||
while (!task->IsDone())
|
||||
cond.wait(lock);
|
||||
*retvalP = xmlrpc_c::value_struct(task->GetRetData());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -10,17 +10,17 @@
|
||||
#endif
|
||||
namespace MosesServer
|
||||
{
|
||||
class
|
||||
class
|
||||
// MosesServer::
|
||||
Translator : public xmlrpc_c::method
|
||||
{
|
||||
public:
|
||||
{
|
||||
public:
|
||||
Translator(size_t numThreads = 10);
|
||||
|
||||
void execute(xmlrpc_c::paramList const& paramList,
|
||||
xmlrpc_c::value * const retvalP);
|
||||
private:
|
||||
private:
|
||||
Moses::ThreadPool m_threadPool;
|
||||
};
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -2,24 +2,24 @@
|
||||
|
||||
namespace MosesServer
|
||||
{
|
||||
using namespace Moses;
|
||||
using namespace std;
|
||||
using namespace Moses;
|
||||
using namespace std;
|
||||
|
||||
Updater::
|
||||
Updater()
|
||||
{
|
||||
Updater::
|
||||
Updater()
|
||||
{
|
||||
// signature and help strings are documentation -- the client
|
||||
// can query this information with a system.methodSignature and
|
||||
// system.methodHelp RPC.
|
||||
this->_signature = "S:S";
|
||||
this->_help = "Updates stuff";
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Updater::
|
||||
execute(xmlrpc_c::paramList const& paramList,
|
||||
void
|
||||
Updater::
|
||||
execute(xmlrpc_c::paramList const& paramList,
|
||||
xmlrpc_c::value * const retvalP)
|
||||
{
|
||||
{
|
||||
#if PT_UG
|
||||
const params_t params = paramList.getStruct(0);
|
||||
breakOutParams(params);
|
||||
@ -28,12 +28,12 @@ namespace MosesServer
|
||||
XVERBOSE(1,"Done inserting\n");
|
||||
*retvalP = xmlrpc_c::value_string("Phrase table updated");
|
||||
#endif
|
||||
};
|
||||
};
|
||||
|
||||
void
|
||||
Updater::
|
||||
breakOutParams(const params_t& params)
|
||||
{
|
||||
void
|
||||
Updater::
|
||||
breakOutParams(const params_t& params)
|
||||
{
|
||||
params_t::const_iterator si = params.find("source");
|
||||
if(si == params.end())
|
||||
throw xmlrpc_c::fault("Missing source sentence",
|
||||
@ -52,6 +52,6 @@ namespace MosesServer
|
||||
XVERBOSE(1,"alignment = " << m_aln << endl);
|
||||
m_bounded = ((si = params.find("bounded")) != params.end());
|
||||
m_add2ORLM = ((si = params.find("updateORLM")) != params.end());
|
||||
};
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -19,9 +19,9 @@
|
||||
|
||||
namespace MosesServer
|
||||
{
|
||||
class
|
||||
class
|
||||
Updater: public xmlrpc_c::method
|
||||
{
|
||||
{
|
||||
|
||||
typedef std::map<std::string, xmlrpc_c::value> params_t;
|
||||
|
||||
@ -29,7 +29,7 @@ namespace MosesServer
|
||||
std::string m_src, m_trg, m_aln;
|
||||
bool m_bounded, m_add2ORLM;
|
||||
|
||||
public:
|
||||
public:
|
||||
Updater();
|
||||
|
||||
void
|
||||
@ -39,6 +39,6 @@ namespace MosesServer
|
||||
void
|
||||
breakOutParams(const params_t& params);
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -18,20 +18,20 @@
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
// todo: replace this with thread lock-free containers, if a stable library can
|
||||
// be found somewhere
|
||||
// todo: replace this with thread lock-free containers, if a stable library can
|
||||
// be found somewhere
|
||||
|
||||
template<typename KEY, typename VAL, class CONTAINER = std::map<KEY,VAL> >
|
||||
class
|
||||
template<typename KEY, typename VAL, class CONTAINER = std::map<KEY,VAL> >
|
||||
class
|
||||
ThreadSafeContainer
|
||||
{
|
||||
protected:
|
||||
{
|
||||
protected:
|
||||
mutable boost::shared_mutex m_lock;
|
||||
CONTAINER m_container;
|
||||
typedef typename CONTAINER::iterator iter_t;
|
||||
typedef typename CONTAINER::const_iterator const_iter_t;
|
||||
typedef typename CONTAINER::value_type entry_t;
|
||||
public:
|
||||
public:
|
||||
|
||||
class locking_iterator
|
||||
{
|
||||
@ -49,8 +49,7 @@ namespace Moses
|
||||
: m_lock(lock), m_container(container), m_iter(iter)
|
||||
{ }
|
||||
|
||||
entry_t const& operator->()
|
||||
{
|
||||
entry_t const& operator->() {
|
||||
UTIL_THROW_IF2(m_container == NULL, "This locking iterator is invalid "
|
||||
<< "or has not been assigned.");
|
||||
return m_iter.operator->();
|
||||
@ -58,21 +57,22 @@ namespace Moses
|
||||
|
||||
// locking operators transfer the lock upon assignment and become invalid
|
||||
locking_iterator const&
|
||||
operator=(locking_iterator& other)
|
||||
{
|
||||
operator=(locking_iterator& other) {
|
||||
m_lock.swap(other.m_lock);
|
||||
m_iter = other.m_iter;
|
||||
other.m_iter = other.m_container.end();
|
||||
}
|
||||
|
||||
bool
|
||||
operator==(const_iter_t const& other)
|
||||
{
|
||||
operator==(const_iter_t const& other) {
|
||||
return m_iter == other;
|
||||
}
|
||||
|
||||
locking_iterator const&
|
||||
operator++() { ++m_iter; return *this; }
|
||||
operator++() {
|
||||
++m_iter;
|
||||
return *this;
|
||||
}
|
||||
|
||||
// DO NOT DEFINE THE POST-INCREMENT OPERATOR!
|
||||
// locking_operators are non-copyable,
|
||||
@ -82,16 +82,15 @@ namespace Moses
|
||||
operator++(int);
|
||||
};
|
||||
|
||||
const_iter_t const& end() const
|
||||
{ return m_container.end(); }
|
||||
const_iter_t const& end() const {
|
||||
return m_container.end();
|
||||
}
|
||||
|
||||
locking_iterator begin() const
|
||||
{
|
||||
locking_iterator begin() const {
|
||||
return locking_iterator(m_lock, this, m_container.begin());
|
||||
}
|
||||
|
||||
VAL const& set(KEY const& key, VAL const& val)
|
||||
{
|
||||
VAL const& set(KEY const& key, VAL const& val) {
|
||||
boost::unique_lock< boost::shared_mutex > lock(m_lock);
|
||||
entry_t entry(key,val);
|
||||
iter_t foo = m_container.insert(entry).first;
|
||||
@ -99,27 +98,24 @@ namespace Moses
|
||||
return foo->second;
|
||||
}
|
||||
|
||||
VAL const* get(KEY const& key, VAL const& default_val)
|
||||
{
|
||||
VAL const* get(KEY const& key, VAL const& default_val) {
|
||||
boost::shared_lock< boost::shared_mutex > lock(m_lock);
|
||||
entry_t entry(key, default_val);
|
||||
iter_t foo = m_container.insert(entry).first;
|
||||
return &(foo->second);
|
||||
}
|
||||
|
||||
VAL const* get(KEY const& key) const
|
||||
{
|
||||
VAL const* get(KEY const& key) const {
|
||||
boost::shared_lock< boost::shared_mutex > lock(m_lock);
|
||||
const_iter_t m = m_container.find(key);
|
||||
if (m == m_container.end()) return NULL;
|
||||
return &m->second;
|
||||
}
|
||||
|
||||
size_t erase(KEY const& key)
|
||||
{
|
||||
size_t erase(KEY const& key) {
|
||||
boost::unique_lock< boost::shared_mutex > lock(m_lock);
|
||||
return m_container.erase(key);
|
||||
}
|
||||
};
|
||||
};
|
||||
}
|
||||
#endif
|
||||
|
@ -116,7 +116,7 @@ void PropertiesConsolidator::ProcessPropertiesString(const std::string &properti
|
||||
|
||||
} else if ( !keyValue[0].compare("POS") ) {
|
||||
|
||||
/* DO NOTHING (property is not registered in the decoder at the moment)
|
||||
/* DO NOTHING (property is not registered in the decoder at the moment)
|
||||
if ( m_partsOfSpeechFlag ) {
|
||||
|
||||
// POS property: replace strings with vocabulary indices
|
||||
@ -127,7 +127,7 @@ void PropertiesConsolidator::ProcessPropertiesString(const std::string &properti
|
||||
} else { // don't process POS property
|
||||
out << " {{" << keyValue[0] << " " << keyValue[1] << "}}";
|
||||
}
|
||||
*/
|
||||
*/
|
||||
|
||||
} else {
|
||||
|
||||
|
@ -699,12 +699,12 @@ void ExtractGHKM::WriteGlueGrammar(
|
||||
// const size_t partOfSpeechSentenceStart = 0;
|
||||
// const size_t partOfSpeechSentenceEnd = 1;
|
||||
|
||||
#ifndef BOS_
|
||||
#define BOS_ "<s>" //Beginning of sentence symbol
|
||||
#endif
|
||||
#ifndef EOS_
|
||||
#define EOS_ "</s>" //End of sentence symbol
|
||||
#endif
|
||||
#ifndef BOS_
|
||||
#define BOS_ "<s>" //Beginning of sentence symbol
|
||||
#endif
|
||||
#ifndef EOS_
|
||||
#define EOS_ "</s>" //End of sentence symbol
|
||||
#endif
|
||||
|
||||
std::string sentenceStartSource = BOS_;
|
||||
std::string sentenceEndSource = EOS_;
|
||||
|
@ -25,8 +25,9 @@ namespace FilterRuleTable
|
||||
// Filters a rule table, discarding rules that cannot be applied to a given
|
||||
// test set. The rule table must have a TSG source-side and the test sentences
|
||||
// must be parse trees.
|
||||
class TreeCfgFilter : public CfgFilter {
|
||||
public:
|
||||
class TreeCfgFilter : public CfgFilter
|
||||
{
|
||||
public:
|
||||
// Initialize the filter for a given set of test sentences.
|
||||
TreeCfgFilter(const std::vector<boost::shared_ptr<StringTree> > &);
|
||||
|
||||
|
@ -15,7 +15,7 @@ namespace PostprocessEgretForests
|
||||
|
||||
class Forest
|
||||
{
|
||||
public:
|
||||
public:
|
||||
struct Vertex;
|
||||
|
||||
struct Hyperedge {
|
||||
@ -35,7 +35,7 @@ class Forest
|
||||
|
||||
std::vector<boost::shared_ptr<Vertex> > vertices;
|
||||
|
||||
private:
|
||||
private:
|
||||
// Copying is not allowed.
|
||||
Forest(const Forest &);
|
||||
Forest &operator=(const Forest &);
|
||||
|
@ -17,15 +17,18 @@ namespace PostprocessEgretForests
|
||||
{
|
||||
|
||||
ForestParser::ForestParser()
|
||||
: m_input(0) {
|
||||
: m_input(0)
|
||||
{
|
||||
}
|
||||
|
||||
ForestParser::ForestParser(std::istream &input)
|
||||
: m_input(&input) {
|
||||
: m_input(&input)
|
||||
{
|
||||
++(*this);
|
||||
}
|
||||
|
||||
ForestParser &ForestParser::operator++() {
|
||||
ForestParser &ForestParser::operator++()
|
||||
{
|
||||
if (!m_input) {
|
||||
return *this;
|
||||
}
|
||||
@ -132,12 +135,14 @@ boost::shared_ptr<Forest::Vertex> ForestParser::ParseVertex(
|
||||
return v;
|
||||
}
|
||||
|
||||
bool operator==(const ForestParser &lhs, const ForestParser &rhs) {
|
||||
bool operator==(const ForestParser &lhs, const ForestParser &rhs)
|
||||
{
|
||||
// TODO Is this right? Compare values of istreams if non-zero?
|
||||
return lhs.m_input == rhs.m_input;
|
||||
}
|
||||
|
||||
bool operator!=(const ForestParser &lhs, const ForestParser &rhs) {
|
||||
bool operator!=(const ForestParser &lhs, const ForestParser &rhs)
|
||||
{
|
||||
return !(lhs == rhs);
|
||||
}
|
||||
|
||||
|
@ -20,8 +20,9 @@ namespace Syntax
|
||||
namespace PostprocessEgretForests
|
||||
{
|
||||
|
||||
class ForestParser {
|
||||
public:
|
||||
class ForestParser
|
||||
{
|
||||
public:
|
||||
struct Entry {
|
||||
std::size_t sentNum;
|
||||
std::string sentence;
|
||||
@ -31,15 +32,19 @@ class ForestParser {
|
||||
ForestParser();
|
||||
ForestParser(std::istream &);
|
||||
|
||||
Entry &operator*() { return m_entry; }
|
||||
Entry *operator->() { return &m_entry; }
|
||||
Entry &operator*() {
|
||||
return m_entry;
|
||||
}
|
||||
Entry *operator->() {
|
||||
return &m_entry;
|
||||
}
|
||||
|
||||
ForestParser &operator++();
|
||||
|
||||
friend bool operator==(const ForestParser &, const ForestParser &);
|
||||
friend bool operator!=(const ForestParser &, const ForestParser &);
|
||||
|
||||
private:
|
||||
private:
|
||||
typedef boost::shared_ptr<Forest::Vertex> VertexSP;
|
||||
typedef boost::shared_ptr<Forest::Hyperedge> HyperedgeSP;
|
||||
|
||||
|
@ -61,7 +61,8 @@ void ForestWriter::WriteVertex(const Forest::Vertex &v)
|
||||
}
|
||||
}
|
||||
|
||||
std::string ForestWriter::PossiblyEscape(const std::string &s) const {
|
||||
std::string ForestWriter::PossiblyEscape(const std::string &s) const
|
||||
{
|
||||
if (m_options.escape) {
|
||||
return Escape(s);
|
||||
} else {
|
||||
@ -70,7 +71,8 @@ std::string ForestWriter::PossiblyEscape(const std::string &s) const {
|
||||
}
|
||||
|
||||
// Escapes XML special characters.
|
||||
std::string ForestWriter::Escape(const std::string &s) const {
|
||||
std::string ForestWriter::Escape(const std::string &s) const
|
||||
{
|
||||
std::string t;
|
||||
std::size_t len = s.size();
|
||||
t.reserve(len);
|
||||
|
@ -15,13 +15,13 @@ namespace PostprocessEgretForests
|
||||
|
||||
class ForestWriter
|
||||
{
|
||||
public:
|
||||
public:
|
||||
ForestWriter(const Options &options, std::ostream &out)
|
||||
: m_options(options), m_out(out) {}
|
||||
|
||||
void Write(const std::string &, const Forest &, std::size_t);
|
||||
|
||||
private:
|
||||
private:
|
||||
std::string Escape(const std::string &) const;
|
||||
std::string PossiblyEscape(const std::string &) const;
|
||||
void WriteHyperedgeLine(const Forest::Hyperedge &);
|
||||
|
@ -16,15 +16,18 @@ namespace PostprocessEgretForests
|
||||
{
|
||||
|
||||
SplitPointFileParser::SplitPointFileParser()
|
||||
: m_input(0) {
|
||||
: m_input(0)
|
||||
{
|
||||
}
|
||||
|
||||
SplitPointFileParser::SplitPointFileParser(std::istream &input)
|
||||
: m_input(&input) {
|
||||
: m_input(&input)
|
||||
{
|
||||
++(*this);
|
||||
}
|
||||
|
||||
SplitPointFileParser &SplitPointFileParser::operator++() {
|
||||
SplitPointFileParser &SplitPointFileParser::operator++()
|
||||
{
|
||||
if (!m_input) {
|
||||
return *this;
|
||||
}
|
||||
@ -66,13 +69,15 @@ void SplitPointFileParser::ParseLine(const std::string &line,
|
||||
}
|
||||
|
||||
bool operator==(const SplitPointFileParser &lhs,
|
||||
const SplitPointFileParser &rhs) {
|
||||
const SplitPointFileParser &rhs)
|
||||
{
|
||||
// TODO Is this right? Compare values of istreams if non-zero?
|
||||
return lhs.m_input == rhs.m_input;
|
||||
}
|
||||
|
||||
bool operator!=(const SplitPointFileParser &lhs,
|
||||
const SplitPointFileParser &rhs) {
|
||||
const SplitPointFileParser &rhs)
|
||||
{
|
||||
return !(lhs == rhs);
|
||||
}
|
||||
|
||||
|
@ -13,8 +13,9 @@ namespace Syntax
|
||||
namespace PostprocessEgretForests
|
||||
{
|
||||
|
||||
class SplitPointFileParser {
|
||||
public:
|
||||
class SplitPointFileParser
|
||||
{
|
||||
public:
|
||||
struct Entry {
|
||||
std::vector<SplitPoint> splitPoints;
|
||||
};
|
||||
@ -22,8 +23,12 @@ class SplitPointFileParser {
|
||||
SplitPointFileParser();
|
||||
SplitPointFileParser(std::istream &);
|
||||
|
||||
const Entry &operator*() const { return m_entry; }
|
||||
const Entry *operator->() const { return &m_entry; }
|
||||
const Entry &operator*() const {
|
||||
return m_entry;
|
||||
}
|
||||
const Entry *operator->() const {
|
||||
return &m_entry;
|
||||
}
|
||||
|
||||
SplitPointFileParser &operator++();
|
||||
|
||||
@ -33,7 +38,7 @@ class SplitPointFileParser {
|
||||
friend bool operator!=(const SplitPointFileParser &,
|
||||
const SplitPointFileParser &);
|
||||
|
||||
private:
|
||||
private:
|
||||
void ParseLine(const std::string &, std::vector<SplitPoint> &);
|
||||
|
||||
Entry m_entry;
|
||||
|
@ -30,14 +30,14 @@ inline bool operator==(const Symbol &s, const Symbol &t)
|
||||
}
|
||||
|
||||
struct SymbolHasher {
|
||||
public:
|
||||
public:
|
||||
std::size_t operator()(const Symbol &s) const {
|
||||
return hash_value(s);
|
||||
}
|
||||
};
|
||||
|
||||
struct SymbolEqualityPred {
|
||||
public:
|
||||
public:
|
||||
bool operator()(const Symbol &s, const Symbol &t) const {
|
||||
return s.value == t.value && s.isNonTerminal == t.isNonTerminal;
|
||||
}
|
||||
|
@ -16,10 +16,10 @@ namespace PostprocessEgretForests
|
||||
|
||||
class TopologicalSorter
|
||||
{
|
||||
public:
|
||||
public:
|
||||
void Sort(const Forest &, std::vector<const Forest::Vertex *> &);
|
||||
|
||||
private:
|
||||
private:
|
||||
typedef boost::unordered_set<const Forest::Vertex *> VertexSet;
|
||||
|
||||
void BuildPredSets(const Forest &);
|
||||
|
Loading…
Reference in New Issue
Block a user