mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
delete lineNumber variable from Manager and TranslationTask. It should always be carried by the input sentence itself
This commit is contained in:
parent
0d7048803a
commit
433186ae25
@ -33,12 +33,12 @@
|
|||||||
<link>
|
<link>
|
||||||
<name>IOWrapper.cpp</name>
|
<name>IOWrapper.cpp</name>
|
||||||
<type>1</type>
|
<type>1</type>
|
||||||
<locationURI>PARENT-3-PROJECT_LOC/moses-cmd/IOWrapper.cpp</locationURI>
|
<locationURI>PARENT-3-PROJECT_LOC/moses/IOWrapper.cpp</locationURI>
|
||||||
</link>
|
</link>
|
||||||
<link>
|
<link>
|
||||||
<name>IOWrapper.h</name>
|
<name>IOWrapper.h</name>
|
||||||
<type>1</type>
|
<type>1</type>
|
||||||
<locationURI>PARENT-3-PROJECT_LOC/moses-cmd/IOWrapper.h</locationURI>
|
<locationURI>PARENT-3-PROJECT_LOC/moses/IOWrapper.h</locationURI>
|
||||||
</link>
|
</link>
|
||||||
<link>
|
<link>
|
||||||
<name>mosesserver.cpp</name>
|
<name>mosesserver.cpp</name>
|
||||||
|
@ -292,13 +292,15 @@ public:
|
|||||||
m_retData.insert(pair<string, xmlrpc_c::value>("sg", xmlrpc_c::value_string(sgstream.str())));
|
m_retData.insert(pair<string, xmlrpc_c::value>("sg", xmlrpc_c::value_string(sgstream.str())));
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
size_t lineNumber = 0; // TODO: Include sentence request number here?
|
||||||
Sentence sentence;
|
Sentence sentence;
|
||||||
|
sentence.SetTranslationId(lineNumber);
|
||||||
|
|
||||||
const vector<FactorType> &
|
const vector<FactorType> &
|
||||||
inputFactorOrder = staticData.GetInputFactorOrder();
|
inputFactorOrder = staticData.GetInputFactorOrder();
|
||||||
stringstream in(source + "\n");
|
stringstream in(source + "\n");
|
||||||
sentence.Read(in,inputFactorOrder);
|
sentence.Read(in,inputFactorOrder);
|
||||||
size_t lineNumber = 0; // TODO: Include sentence request number here?
|
Manager manager(sentence, staticData.GetSearchAlgorithm());
|
||||||
Manager manager(lineNumber, sentence, staticData.GetSearchAlgorithm());
|
|
||||||
manager.ProcessSentence();
|
manager.ProcessSentence();
|
||||||
const Hypothesis* hypo = manager.GetBestHypothesis();
|
const Hypothesis* hypo = manager.GetBestHypothesis();
|
||||||
|
|
||||||
|
@ -143,7 +143,7 @@ vector< vector<const Word*> > MosesDecoder::runDecoder(const std::string& source
|
|||||||
string filename)
|
string filename)
|
||||||
{
|
{
|
||||||
// run the decoder
|
// run the decoder
|
||||||
m_manager = new Moses::Manager(0,*m_sentence, search);
|
m_manager = new Moses::Manager(*m_sentence, search);
|
||||||
m_manager->ProcessSentence();
|
m_manager->ProcessSentence();
|
||||||
TrellisPathList nBestList;
|
TrellisPathList nBestList;
|
||||||
m_manager->CalcNBest(nBestSize, nBestList, distinct);
|
m_manager->CalcNBest(nBestSize, nBestList, distinct);
|
||||||
|
@ -159,7 +159,7 @@ int main(int argc, char* argv[])
|
|||||||
|
|
||||||
StaticData& staticData = const_cast<StaticData&>(StaticData::Instance());
|
StaticData& staticData = const_cast<StaticData&>(StaticData::Instance());
|
||||||
staticData.SetUseLatticeMBR(true);
|
staticData.SetUseLatticeMBR(true);
|
||||||
IOWrapper* ioWrapper = GetIOWrapper(staticData);
|
IOWrapper* ioWrapper = IOWrapper::GetIOWrapper(staticData);
|
||||||
|
|
||||||
if (!ioWrapper) {
|
if (!ioWrapper) {
|
||||||
throw runtime_error("Failed to initialise IOWrapper");
|
throw runtime_error("Failed to initialise IOWrapper");
|
||||||
@ -180,8 +180,9 @@ int main(int argc, char* argv[])
|
|||||||
|
|
||||||
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
|
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
|
||||||
++lineCount;
|
++lineCount;
|
||||||
Sentence sentence;
|
source->SetTranslationId(lineCount);
|
||||||
Manager manager(lineCount, *source, staticData.GetSearchAlgorithm());
|
|
||||||
|
Manager manager(*source, staticData.GetSearchAlgorithm());
|
||||||
manager.ProcessSentence();
|
manager.ProcessSentence();
|
||||||
TrellisPathList nBestList;
|
TrellisPathList nBestList;
|
||||||
manager.CalcNBest(nBestSize, nBestList,true);
|
manager.CalcNBest(nBestSize, nBestList,true);
|
||||||
|
@ -142,6 +142,7 @@ int main(int argc, char** argv)
|
|||||||
InputType* source = NULL;
|
InputType* source = NULL;
|
||||||
size_t lineCount = staticData.GetStartTranslationId();
|
size_t lineCount = staticData.GetStartTranslationId();
|
||||||
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
|
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
|
||||||
|
source->SetTranslationId(lineCount);
|
||||||
IFVERBOSE(1) {
|
IFVERBOSE(1) {
|
||||||
ResetUserTime();
|
ResetUserTime();
|
||||||
}
|
}
|
||||||
@ -150,7 +151,7 @@ int main(int argc, char** argv)
|
|||||||
|
|
||||||
// set up task of translating one sentence
|
// set up task of translating one sentence
|
||||||
TranslationTask* task =
|
TranslationTask* task =
|
||||||
new TranslationTask(lineCount,source, *ioWrapper,
|
new TranslationTask(source, *ioWrapper,
|
||||||
staticData.GetOutputSearchGraphSLF(),
|
staticData.GetOutputSearchGraphSLF(),
|
||||||
hypergraphOutput);
|
hypergraphOutput);
|
||||||
// execute task
|
// execute task
|
||||||
|
@ -124,7 +124,7 @@ template<class M>
|
|||||||
void HypergraphOutput<M>::Write(const M& manager) const {
|
void HypergraphOutput<M>::Write(const M& manager) const {
|
||||||
|
|
||||||
stringstream fileName;
|
stringstream fileName;
|
||||||
fileName << m_hypergraphDir << "/" << manager.GetLineNumber();
|
fileName << m_hypergraphDir << "/" << manager.GetSource().GetTranslationId();
|
||||||
if ( m_appendSuffix ) {
|
if ( m_appendSuffix ) {
|
||||||
fileName << "." << m_compression;
|
fileName << "." << m_compression;
|
||||||
}
|
}
|
||||||
@ -144,7 +144,7 @@ void HypergraphOutput<M>::Write(const M& manager) const {
|
|||||||
manager.OutputSearchGraphAsHypergraph(file);
|
manager.OutputSearchGraphAsHypergraph(file);
|
||||||
file.flush();
|
file.flush();
|
||||||
} else {
|
} else {
|
||||||
TRACE_ERR("Cannot output hypergraph for line " << manager.GetLineNumber()
|
TRACE_ERR("Cannot output hypergraph for line " << manager.GetSource().GetTranslationId()
|
||||||
<< " because the output file " << fileName.str()
|
<< " because the output file " << fileName.str()
|
||||||
<< " is not open or not ready for writing"
|
<< " is not open or not ready for writing"
|
||||||
<< std::endl);
|
<< std::endl);
|
||||||
|
@ -54,12 +54,11 @@ using namespace std;
|
|||||||
|
|
||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
Manager::Manager(size_t lineNumber, InputType const& source, SearchAlgorithm searchAlgorithm)
|
Manager::Manager(InputType const& source, SearchAlgorithm searchAlgorithm)
|
||||||
:m_transOptColl(source.CreateTranslationOptionCollection())
|
:m_transOptColl(source.CreateTranslationOptionCollection())
|
||||||
,m_search(Search::CreateSearch(*this, source, searchAlgorithm, *m_transOptColl))
|
,m_search(Search::CreateSearch(*this, source, searchAlgorithm, *m_transOptColl))
|
||||||
,interrupted_flag(0)
|
,interrupted_flag(0)
|
||||||
,m_hypoId(0)
|
,m_hypoId(0)
|
||||||
,m_lineNumber(lineNumber)
|
|
||||||
,m_source(source)
|
,m_source(source)
|
||||||
{
|
{
|
||||||
StaticData::Instance().InitializeForInput(m_source);
|
StaticData::Instance().InitializeForInput(m_source);
|
||||||
@ -105,7 +104,7 @@ void Manager::ProcessSentence()
|
|||||||
// some reporting on how long this took
|
// some reporting on how long this took
|
||||||
IFVERBOSE(1) {
|
IFVERBOSE(1) {
|
||||||
GetSentenceStats().StopTimeCollectOpts();
|
GetSentenceStats().StopTimeCollectOpts();
|
||||||
TRACE_ERR("Line "<< m_lineNumber << ": Collecting options took "
|
TRACE_ERR("Line "<< m_source.GetTranslationId() << ": Collecting options took "
|
||||||
<< GetSentenceStats().GetTimeCollectOpts() << " seconds at "
|
<< GetSentenceStats().GetTimeCollectOpts() << " seconds at "
|
||||||
<< __FILE__ << ":" << __LINE__ << endl);
|
<< __FILE__ << ":" << __LINE__ << endl);
|
||||||
}
|
}
|
||||||
@ -114,7 +113,7 @@ void Manager::ProcessSentence()
|
|||||||
Timer searchTime;
|
Timer searchTime;
|
||||||
searchTime.start();
|
searchTime.start();
|
||||||
m_search->ProcessSentence();
|
m_search->ProcessSentence();
|
||||||
VERBOSE(1, "Line " << m_lineNumber << ": Search took " << searchTime << " seconds" << endl);
|
VERBOSE(1, "Line " << m_source.GetTranslationId() << ": Search took " << searchTime << " seconds" << endl);
|
||||||
IFVERBOSE(2) {
|
IFVERBOSE(2) {
|
||||||
GetSentenceStats().StopTimeTotal();
|
GetSentenceStats().StopTimeTotal();
|
||||||
TRACE_ERR(GetSentenceStats());
|
TRACE_ERR(GetSentenceStats());
|
||||||
@ -831,7 +830,7 @@ size_t Manager::OutputFeatureValuesForSLF(size_t index, bool zeros, const Hypoth
|
|||||||
void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const
|
void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const
|
||||||
{
|
{
|
||||||
|
|
||||||
VERBOSE(2,"Getting search graph to output as hypergraph for sentence " << m_lineNumber << std::endl)
|
VERBOSE(2,"Getting search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl)
|
||||||
|
|
||||||
vector<SearchGraphNode> searchGraph;
|
vector<SearchGraphNode> searchGraph;
|
||||||
GetSearchGraph(searchGraph);
|
GetSearchGraph(searchGraph);
|
||||||
@ -842,7 +841,7 @@ void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStrea
|
|||||||
set<int> terminalNodes;
|
set<int> terminalNodes;
|
||||||
multimap<int,int> hypergraphIDToArcs;
|
multimap<int,int> hypergraphIDToArcs;
|
||||||
|
|
||||||
VERBOSE(2,"Gathering information about search graph to output as hypergraph for sentence " << m_lineNumber << std::endl)
|
VERBOSE(2,"Gathering information about search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl)
|
||||||
|
|
||||||
long numNodes = 0;
|
long numNodes = 0;
|
||||||
long endNode = 0;
|
long endNode = 0;
|
||||||
@ -904,15 +903,15 @@ void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStrea
|
|||||||
// Print number of nodes and arcs
|
// Print number of nodes and arcs
|
||||||
outputSearchGraphStream << numNodes << " " << numArcs << endl;
|
outputSearchGraphStream << numNodes << " " << numArcs << endl;
|
||||||
|
|
||||||
VERBOSE(2,"Search graph to output as hypergraph for sentence " << m_lineNumber
|
VERBOSE(2,"Search graph to output as hypergraph for sentence " << m_source.GetTranslationId()
|
||||||
<< " contains " << numArcs << " arcs and " << numNodes << " nodes" << std::endl)
|
<< " contains " << numArcs << " arcs and " << numNodes << " nodes" << std::endl)
|
||||||
|
|
||||||
VERBOSE(2,"Outputting search graph to output as hypergraph for sentence " << m_lineNumber << std::endl)
|
VERBOSE(2,"Outputting search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl)
|
||||||
|
|
||||||
|
|
||||||
for (int hypergraphHypothesisID=0; hypergraphHypothesisID < endNode; hypergraphHypothesisID+=1) {
|
for (int hypergraphHypothesisID=0; hypergraphHypothesisID < endNode; hypergraphHypothesisID+=1) {
|
||||||
if (hypergraphHypothesisID % 100000 == 0) {
|
if (hypergraphHypothesisID % 100000 == 0) {
|
||||||
VERBOSE(2,"Processed " << hypergraphHypothesisID << " of " << numNodes << " hypergraph nodes for sentence " << m_lineNumber << std::endl);
|
VERBOSE(2,"Processed " << hypergraphHypothesisID << " of " << numNodes << " hypergraph nodes for sentence " << m_source.GetTranslationId() << std::endl);
|
||||||
}
|
}
|
||||||
// int mosesID = hypergraphIDToMosesID[hypergraphHypothesisID];
|
// int mosesID = hypergraphIDToMosesID[hypergraphHypothesisID];
|
||||||
size_t count = hypergraphIDToArcs.count(hypergraphHypothesisID);
|
size_t count = hypergraphIDToArcs.count(hypergraphHypothesisID);
|
||||||
@ -935,7 +934,7 @@ void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStrea
|
|||||||
// int actualHypergraphHypothesisID = mosesIDToHypergraphID[mosesHypothesisID];
|
// int actualHypergraphHypothesisID = mosesIDToHypergraphID[mosesHypothesisID];
|
||||||
UTIL_THROW_IF2(
|
UTIL_THROW_IF2(
|
||||||
(hypergraphHypothesisID != mosesIDToHypergraphID[mosesHypothesisID]),
|
(hypergraphHypothesisID != mosesIDToHypergraphID[mosesHypothesisID]),
|
||||||
"Error while writing search lattice as hypergraph for sentence " << m_lineNumber << ". " <<
|
"Error while writing search lattice as hypergraph for sentence " << m_source.GetTranslationId() << ". " <<
|
||||||
"Moses node " << mosesHypothesisID << " was expected to have hypergraph id " << hypergraphHypothesisID <<
|
"Moses node " << mosesHypothesisID << " was expected to have hypergraph id " << hypergraphHypothesisID <<
|
||||||
", but actually had hypergraph id " << mosesIDToHypergraphID[mosesHypothesisID] <<
|
", but actually had hypergraph id " << mosesIDToHypergraphID[mosesHypothesisID] <<
|
||||||
". There are " << numNodes << " nodes in the search lattice."
|
". There are " << numNodes << " nodes in the search lattice."
|
||||||
@ -950,7 +949,7 @@ void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStrea
|
|||||||
// VERBOSE(2,"Hypergraph node " << hypergraphHypothesisID << " has parent node " << startNode << std::endl)
|
// VERBOSE(2,"Hypergraph node " << hypergraphHypothesisID << " has parent node " << startNode << std::endl)
|
||||||
UTIL_THROW_IF2(
|
UTIL_THROW_IF2(
|
||||||
(startNode >= hypergraphHypothesisID),
|
(startNode >= hypergraphHypothesisID),
|
||||||
"Error while writing search lattice as hypergraph for sentence" << m_lineNumber << ". " <<
|
"Error while writing search lattice as hypergraph for sentence" << m_source.GetTranslationId() << ". " <<
|
||||||
"The nodes must be output in topological order. The code attempted to violate this restriction."
|
"The nodes must be output in topological order. The code attempted to violate this restriction."
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -118,7 +118,6 @@ protected:
|
|||||||
size_t interrupted_flag;
|
size_t interrupted_flag;
|
||||||
std::auto_ptr<SentenceStats> m_sentenceStats;
|
std::auto_ptr<SentenceStats> m_sentenceStats;
|
||||||
int m_hypoId; //used to number the hypos as they are created.
|
int m_hypoId; //used to number the hypos as they are created.
|
||||||
size_t m_lineNumber;
|
|
||||||
|
|
||||||
void GetConnectedGraph(
|
void GetConnectedGraph(
|
||||||
std::map< int, bool >* pConnected,
|
std::map< int, bool >* pConnected,
|
||||||
@ -130,7 +129,7 @@ protected:
|
|||||||
|
|
||||||
public:
|
public:
|
||||||
InputType const& m_source; /**< source sentence to be translated */
|
InputType const& m_source; /**< source sentence to be translated */
|
||||||
Manager(size_t lineNumber, InputType const& source, SearchAlgorithm searchAlgorithm);
|
Manager(InputType const& source, SearchAlgorithm searchAlgorithm);
|
||||||
~Manager();
|
~Manager();
|
||||||
const TranslationOptionCollection* getSntTranslationOptions();
|
const TranslationOptionCollection* getSntTranslationOptions();
|
||||||
|
|
||||||
@ -145,7 +144,7 @@ public:
|
|||||||
void GetOutputLanguageModelOrder( std::ostream &out, const Hypothesis *hypo );
|
void GetOutputLanguageModelOrder( std::ostream &out, const Hypothesis *hypo );
|
||||||
void GetWordGraph(long translationId, std::ostream &outputWordGraphStream) const;
|
void GetWordGraph(long translationId, std::ostream &outputWordGraphStream) const;
|
||||||
int GetNextHypoId();
|
int GetNextHypoId();
|
||||||
size_t GetLineNumber() const {return m_lineNumber;}
|
|
||||||
#ifdef HAVE_PROTOBUF
|
#ifdef HAVE_PROTOBUF
|
||||||
void SerializeSearchGraphPB(long translationId, std::ostream& outputStream) const;
|
void SerializeSearchGraphPB(long translationId, std::ostream& outputStream) const;
|
||||||
#endif
|
#endif
|
||||||
|
@ -41,7 +41,7 @@ MockHypothesisGuard::MockHypothesisGuard(
|
|||||||
m_wp("WordPenalty"),
|
m_wp("WordPenalty"),
|
||||||
m_uwp("UnknownWordPenalty"),
|
m_uwp("UnknownWordPenalty"),
|
||||||
m_dist("Distortion"),
|
m_dist("Distortion"),
|
||||||
m_manager(0,m_sentence,Normal)
|
m_manager(m_sentence,Normal)
|
||||||
{
|
{
|
||||||
BOOST_CHECK_EQUAL(alignments.size(), targetSegments.size());
|
BOOST_CHECK_EQUAL(alignments.size(), targetSegments.size());
|
||||||
|
|
||||||
|
@ -15,10 +15,10 @@ using namespace Moses;
|
|||||||
namespace MosesCmd
|
namespace MosesCmd
|
||||||
{
|
{
|
||||||
|
|
||||||
TranslationTask::TranslationTask(size_t lineNumber, InputType* source, MosesCmd::IOWrapper &ioWrapper,
|
TranslationTask::TranslationTask(InputType* source, MosesCmd::IOWrapper &ioWrapper,
|
||||||
bool outputSearchGraphSLF,
|
bool outputSearchGraphSLF,
|
||||||
boost::shared_ptr<HypergraphOutput<Manager> > hypergraphOutput) :
|
boost::shared_ptr<HypergraphOutput<Manager> > hypergraphOutput) :
|
||||||
m_source(source), m_lineNumber(lineNumber),
|
m_source(source),
|
||||||
m_ioWrapper(ioWrapper),
|
m_ioWrapper(ioWrapper),
|
||||||
m_outputSearchGraphSLF(outputSearchGraphSLF),
|
m_outputSearchGraphSLF(outputSearchGraphSLF),
|
||||||
m_hypergraphOutput(hypergraphOutput)
|
m_hypergraphOutput(hypergraphOutput)
|
||||||
@ -41,7 +41,7 @@ void TranslationTask::Run() {
|
|||||||
|
|
||||||
// report thread number
|
// report thread number
|
||||||
#if defined(WITH_THREADS) && defined(BOOST_HAS_PTHREADS)
|
#if defined(WITH_THREADS) && defined(BOOST_HAS_PTHREADS)
|
||||||
TRACE_ERR("Translating line " << m_lineNumber << " in thread id " << pthread_self() << endl);
|
TRACE_ERR("Translating line " << m_source->GetTranslationId() << " in thread id " << pthread_self() << endl);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
||||||
@ -50,8 +50,8 @@ void TranslationTask::Run() {
|
|||||||
// we still need to apply the decision rule (MAP, MBR, ...)
|
// we still need to apply the decision rule (MAP, MBR, ...)
|
||||||
Timer initTime;
|
Timer initTime;
|
||||||
initTime.start();
|
initTime.start();
|
||||||
Manager manager(m_lineNumber, *m_source,staticData.GetSearchAlgorithm());
|
Manager manager(*m_source,staticData.GetSearchAlgorithm());
|
||||||
VERBOSE(1, "Line " << m_lineNumber << ": Initialize search took " << initTime << " seconds total" << endl);
|
VERBOSE(1, "Line " << m_source->GetTranslationId() << ": Initialize search took " << initTime << " seconds total" << endl);
|
||||||
manager.ProcessSentence();
|
manager.ProcessSentence();
|
||||||
|
|
||||||
// we are done with search, let's look what we got
|
// we are done with search, let's look what we got
|
||||||
@ -62,25 +62,25 @@ void TranslationTask::Run() {
|
|||||||
if (m_ioWrapper.GetWordGraphCollector()) {
|
if (m_ioWrapper.GetWordGraphCollector()) {
|
||||||
ostringstream out;
|
ostringstream out;
|
||||||
fix(out,PRECISION);
|
fix(out,PRECISION);
|
||||||
manager.GetWordGraph(m_lineNumber, out);
|
manager.GetWordGraph(m_source->GetTranslationId(), out);
|
||||||
m_ioWrapper.GetWordGraphCollector()->Write(m_lineNumber, out.str());
|
m_ioWrapper.GetWordGraphCollector()->Write(m_source->GetTranslationId(), out.str());
|
||||||
}
|
}
|
||||||
|
|
||||||
// output search graph
|
// output search graph
|
||||||
if (m_ioWrapper.GetSearchGraphOutputCollector()) {
|
if (m_ioWrapper.GetSearchGraphOutputCollector()) {
|
||||||
ostringstream out;
|
ostringstream out;
|
||||||
fix(out,PRECISION);
|
fix(out,PRECISION);
|
||||||
manager.OutputSearchGraph(m_lineNumber, out);
|
manager.OutputSearchGraph(m_source->GetTranslationId(), out);
|
||||||
m_ioWrapper.GetSearchGraphOutputCollector()->Write(m_lineNumber, out.str());
|
m_ioWrapper.GetSearchGraphOutputCollector()->Write(m_source->GetTranslationId(), out.str());
|
||||||
|
|
||||||
#ifdef HAVE_PROTOBUF
|
#ifdef HAVE_PROTOBUF
|
||||||
if (staticData.GetOutputSearchGraphPB()) {
|
if (staticData.GetOutputSearchGraphPB()) {
|
||||||
ostringstream sfn;
|
ostringstream sfn;
|
||||||
sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_lineNumber << ".pb" << ends;
|
sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_source->GetTranslationId() << ".pb" << ends;
|
||||||
string fn = sfn.str();
|
string fn = sfn.str();
|
||||||
VERBOSE(2, "Writing search graph to " << fn << endl);
|
VERBOSE(2, "Writing search graph to " << fn << endl);
|
||||||
fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out);
|
fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out);
|
||||||
manager.SerializeSearchGraphPB(m_lineNumber, output);
|
manager.SerializeSearchGraphPB(m_source->GetTranslationId(), output);
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
@ -88,17 +88,17 @@ void TranslationTask::Run() {
|
|||||||
// Output search graph in HTK standard lattice format (SLF)
|
// Output search graph in HTK standard lattice format (SLF)
|
||||||
if (m_outputSearchGraphSLF) {
|
if (m_outputSearchGraphSLF) {
|
||||||
stringstream fileName;
|
stringstream fileName;
|
||||||
fileName << staticData.GetParam("output-search-graph-slf")[0] << "/" << m_lineNumber << ".slf";
|
fileName << staticData.GetParam("output-search-graph-slf")[0] << "/" << m_source->GetTranslationId() << ".slf";
|
||||||
ofstream *file = new ofstream;
|
ofstream *file = new ofstream;
|
||||||
file->open(fileName.str().c_str());
|
file->open(fileName.str().c_str());
|
||||||
if (file->is_open() && file->good()) {
|
if (file->is_open() && file->good()) {
|
||||||
ostringstream out;
|
ostringstream out;
|
||||||
fix(out,PRECISION);
|
fix(out,PRECISION);
|
||||||
manager.OutputSearchGraphAsSLF(m_lineNumber, out);
|
manager.OutputSearchGraphAsSLF(m_source->GetTranslationId(), out);
|
||||||
*file << out.str();
|
*file << out.str();
|
||||||
file -> flush();
|
file -> flush();
|
||||||
} else {
|
} else {
|
||||||
TRACE_ERR("Cannot output HTK standard lattice for line " << m_lineNumber << " because the output file is not open or not ready for writing" << endl);
|
TRACE_ERR("Cannot output HTK standard lattice for line " << m_source->GetTranslationId() << " because the output file is not open or not ready for writing" << endl);
|
||||||
}
|
}
|
||||||
delete file;
|
delete file;
|
||||||
}
|
}
|
||||||
@ -119,7 +119,7 @@ void TranslationTask::Run() {
|
|||||||
// all derivations - send them to debug stream
|
// all derivations - send them to debug stream
|
||||||
if (staticData.PrintAllDerivations()) {
|
if (staticData.PrintAllDerivations()) {
|
||||||
additionalReportingTime.start();
|
additionalReportingTime.start();
|
||||||
manager.PrintAllDerivations(m_lineNumber, debug);
|
manager.PrintAllDerivations(m_source->GetTranslationId(), debug);
|
||||||
additionalReportingTime.stop();
|
additionalReportingTime.stop();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -156,7 +156,7 @@ void TranslationTask::Run() {
|
|||||||
OutputAlignment(out, bestHypo);
|
OutputAlignment(out, bestHypo);
|
||||||
}
|
}
|
||||||
|
|
||||||
OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_lineNumber, bestHypo);
|
OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_source->GetTranslationId(), bestHypo);
|
||||||
IFVERBOSE(1) {
|
IFVERBOSE(1) {
|
||||||
debug << "BEST TRANSLATION: " << *bestHypo << endl;
|
debug << "BEST TRANSLATION: " << *bestHypo << endl;
|
||||||
}
|
}
|
||||||
@ -190,12 +190,12 @@ void TranslationTask::Run() {
|
|||||||
size_t n = min(nBestSize, staticData.GetNBestSize());
|
size_t n = min(nBestSize, staticData.GetNBestSize());
|
||||||
getLatticeMBRNBest(manager,nBestList,solutions,n);
|
getLatticeMBRNBest(manager,nBestList,solutions,n);
|
||||||
ostringstream out;
|
ostringstream out;
|
||||||
OutputLatticeMBRNBest(out, solutions,m_lineNumber);
|
OutputLatticeMBRNBest(out, solutions,m_source->GetTranslationId());
|
||||||
m_ioWrapper.GetNBestOutputCollector()->Write(m_lineNumber, out.str());
|
m_ioWrapper.GetNBestOutputCollector()->Write(m_source->GetTranslationId(), out.str());
|
||||||
} else {
|
} else {
|
||||||
//Lattice MBR decoding
|
//Lattice MBR decoding
|
||||||
vector<Word> mbrBestHypo = doLatticeMBR(manager,nBestList);
|
vector<Word> mbrBestHypo = doLatticeMBR(manager,nBestList);
|
||||||
OutputBestHypo(mbrBestHypo, m_lineNumber, staticData.GetReportSegmentation(),
|
OutputBestHypo(mbrBestHypo, m_source->GetTranslationId(), staticData.GetReportSegmentation(),
|
||||||
staticData.GetReportAllFactors(),out);
|
staticData.GetReportAllFactors(),out);
|
||||||
IFVERBOSE(2) {
|
IFVERBOSE(2) {
|
||||||
PrintUserTime("finished Lattice MBR decoding");
|
PrintUserTime("finished Lattice MBR decoding");
|
||||||
@ -206,10 +206,10 @@ void TranslationTask::Run() {
|
|||||||
// consensus decoding
|
// consensus decoding
|
||||||
else if (staticData.UseConsensusDecoding()) {
|
else if (staticData.UseConsensusDecoding()) {
|
||||||
const TrellisPath &conBestHypo = doConsensusDecoding(manager,nBestList);
|
const TrellisPath &conBestHypo = doConsensusDecoding(manager,nBestList);
|
||||||
OutputBestHypo(conBestHypo, m_lineNumber,
|
OutputBestHypo(conBestHypo, m_source->GetTranslationId(),
|
||||||
staticData.GetReportSegmentation(),
|
staticData.GetReportSegmentation(),
|
||||||
staticData.GetReportAllFactors(),out);
|
staticData.GetReportAllFactors(),out);
|
||||||
OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_lineNumber, conBestHypo);
|
OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_source->GetTranslationId(), conBestHypo);
|
||||||
IFVERBOSE(2) {
|
IFVERBOSE(2) {
|
||||||
PrintUserTime("finished Consensus decoding");
|
PrintUserTime("finished Consensus decoding");
|
||||||
}
|
}
|
||||||
@ -218,10 +218,10 @@ void TranslationTask::Run() {
|
|||||||
// n-best MBR decoding
|
// n-best MBR decoding
|
||||||
else {
|
else {
|
||||||
const TrellisPath &mbrBestHypo = doMBR(nBestList);
|
const TrellisPath &mbrBestHypo = doMBR(nBestList);
|
||||||
OutputBestHypo(mbrBestHypo, m_lineNumber,
|
OutputBestHypo(mbrBestHypo, m_source->GetTranslationId(),
|
||||||
staticData.GetReportSegmentation(),
|
staticData.GetReportSegmentation(),
|
||||||
staticData.GetReportAllFactors(),out);
|
staticData.GetReportAllFactors(),out);
|
||||||
OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_lineNumber, mbrBestHypo);
|
OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_source->GetTranslationId(), mbrBestHypo);
|
||||||
IFVERBOSE(2) {
|
IFVERBOSE(2) {
|
||||||
PrintUserTime("finished MBR decoding");
|
PrintUserTime("finished MBR decoding");
|
||||||
}
|
}
|
||||||
@ -229,10 +229,10 @@ void TranslationTask::Run() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// report best translation to output collector
|
// report best translation to output collector
|
||||||
m_ioWrapper.GetSingleBestOutputCollector()->Write(m_lineNumber,out.str(),debug.str());
|
m_ioWrapper.GetSingleBestOutputCollector()->Write(m_source->GetTranslationId(),out.str(),debug.str());
|
||||||
|
|
||||||
decisionRuleTime.stop();
|
decisionRuleTime.stop();
|
||||||
VERBOSE(1, "Line " << m_lineNumber << ": Decision rule took " << decisionRuleTime << " seconds total" << endl);
|
VERBOSE(1, "Line " << m_source->GetTranslationId() << ": Decision rule took " << decisionRuleTime << " seconds total" << endl);
|
||||||
}
|
}
|
||||||
|
|
||||||
additionalReportingTime.start();
|
additionalReportingTime.start();
|
||||||
@ -242,9 +242,9 @@ void TranslationTask::Run() {
|
|||||||
TrellisPathList nBestList;
|
TrellisPathList nBestList;
|
||||||
ostringstream out;
|
ostringstream out;
|
||||||
manager.CalcNBest(staticData.GetNBestSize(), nBestList,staticData.GetDistinctNBest());
|
manager.CalcNBest(staticData.GetNBestSize(), nBestList,staticData.GetDistinctNBest());
|
||||||
OutputNBest(out, nBestList, staticData.GetOutputFactorOrder(), m_lineNumber,
|
OutputNBest(out, nBestList, staticData.GetOutputFactorOrder(), m_source->GetTranslationId(),
|
||||||
staticData.GetReportSegmentation());
|
staticData.GetReportSegmentation());
|
||||||
m_ioWrapper.GetNBestOutputCollector()->Write(m_lineNumber, out.str());
|
m_ioWrapper.GetNBestOutputCollector()->Write(m_source->GetTranslationId(), out.str());
|
||||||
}
|
}
|
||||||
|
|
||||||
//lattice samples
|
//lattice samples
|
||||||
@ -252,9 +252,9 @@ void TranslationTask::Run() {
|
|||||||
TrellisPathList latticeSamples;
|
TrellisPathList latticeSamples;
|
||||||
ostringstream out;
|
ostringstream out;
|
||||||
manager.CalcLatticeSamples(staticData.GetLatticeSamplesSize(), latticeSamples);
|
manager.CalcLatticeSamples(staticData.GetLatticeSamplesSize(), latticeSamples);
|
||||||
OutputNBest(out,latticeSamples, staticData.GetOutputFactorOrder(), m_lineNumber,
|
OutputNBest(out,latticeSamples, staticData.GetOutputFactorOrder(), m_source->GetTranslationId(),
|
||||||
staticData.GetReportSegmentation());
|
staticData.GetReportSegmentation());
|
||||||
m_ioWrapper.GetLatticeSamplesCollector()->Write(m_lineNumber, out.str());
|
m_ioWrapper.GetLatticeSamplesCollector()->Write(m_source->GetTranslationId(), out.str());
|
||||||
}
|
}
|
||||||
|
|
||||||
// detailed translation reporting
|
// detailed translation reporting
|
||||||
@ -262,7 +262,7 @@ void TranslationTask::Run() {
|
|||||||
ostringstream out;
|
ostringstream out;
|
||||||
fix(out,PRECISION);
|
fix(out,PRECISION);
|
||||||
TranslationAnalysis::PrintTranslationAnalysis(out, manager.GetBestHypothesis());
|
TranslationAnalysis::PrintTranslationAnalysis(out, manager.GetBestHypothesis());
|
||||||
m_ioWrapper.GetDetailedTranslationCollector()->Write(m_lineNumber,out.str());
|
m_ioWrapper.GetDetailedTranslationCollector()->Write(m_source->GetTranslationId(),out.str());
|
||||||
}
|
}
|
||||||
|
|
||||||
//list of unknown words
|
//list of unknown words
|
||||||
@ -273,13 +273,13 @@ void TranslationTask::Run() {
|
|||||||
out << *(unknowns[i]);
|
out << *(unknowns[i]);
|
||||||
}
|
}
|
||||||
out << endl;
|
out << endl;
|
||||||
m_ioWrapper.GetUnknownsCollector()->Write(m_lineNumber, out.str());
|
m_ioWrapper.GetUnknownsCollector()->Write(m_source->GetTranslationId(), out.str());
|
||||||
}
|
}
|
||||||
|
|
||||||
// report additional statistics
|
// report additional statistics
|
||||||
manager.CalcDecoderStatistics();
|
manager.CalcDecoderStatistics();
|
||||||
VERBOSE(1, "Line " << m_lineNumber << ": Additional reporting took " << additionalReportingTime << " seconds total" << endl);
|
VERBOSE(1, "Line " << m_source->GetTranslationId() << ": Additional reporting took " << additionalReportingTime << " seconds total" << endl);
|
||||||
VERBOSE(1, "Line " << m_lineNumber << ": Translation took " << translationTime << " seconds total" << endl);
|
VERBOSE(1, "Line " << m_source->GetTranslationId() << ": Translation took " << translationTime << " seconds total" << endl);
|
||||||
IFVERBOSE(2) {
|
IFVERBOSE(2) {
|
||||||
PrintUserTime("Sentence Decoding Time:");
|
PrintUserTime("Sentence Decoding Time:");
|
||||||
}
|
}
|
||||||
|
@ -26,7 +26,7 @@ class TranslationTask : public Moses::Task
|
|||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
TranslationTask(size_t lineNumber, Moses::InputType* source, MosesCmd::IOWrapper &ioWrapper,
|
TranslationTask(Moses::InputType* source, MosesCmd::IOWrapper &ioWrapper,
|
||||||
bool outputSearchGraphSLF,
|
bool outputSearchGraphSLF,
|
||||||
boost::shared_ptr<Moses::HypergraphOutput<Moses::Manager> > hypergraphOutput);
|
boost::shared_ptr<Moses::HypergraphOutput<Moses::Manager> > hypergraphOutput);
|
||||||
|
|
||||||
@ -39,7 +39,6 @@ public:
|
|||||||
|
|
||||||
private:
|
private:
|
||||||
Moses::InputType* m_source;
|
Moses::InputType* m_source;
|
||||||
size_t m_lineNumber;
|
|
||||||
MosesCmd::IOWrapper &m_ioWrapper;
|
MosesCmd::IOWrapper &m_ioWrapper;
|
||||||
|
|
||||||
bool m_outputSearchGraphSLF;
|
bool m_outputSearchGraphSLF;
|
||||||
|
Loading…
Reference in New Issue
Block a user