diff --git a/contrib/other-builds/server/.project b/contrib/other-builds/server/.project index dc06f1c6d..549a67e6e 100644 --- a/contrib/other-builds/server/.project +++ b/contrib/other-builds/server/.project @@ -33,12 +33,12 @@ IOWrapper.cpp 1 - PARENT-3-PROJECT_LOC/moses-cmd/IOWrapper.cpp + PARENT-3-PROJECT_LOC/moses/IOWrapper.cpp IOWrapper.h 1 - PARENT-3-PROJECT_LOC/moses-cmd/IOWrapper.h + PARENT-3-PROJECT_LOC/moses/IOWrapper.h mosesserver.cpp diff --git a/contrib/server/mosesserver.cpp b/contrib/server/mosesserver.cpp index f184e1426..7a3f8dc8c 100644 --- a/contrib/server/mosesserver.cpp +++ b/contrib/server/mosesserver.cpp @@ -292,13 +292,15 @@ public: m_retData.insert(pair("sg", xmlrpc_c::value_string(sgstream.str()))); } } else { + size_t lineNumber = 0; // TODO: Include sentence request number here? Sentence sentence; + sentence.SetTranslationId(lineNumber); + const vector & inputFactorOrder = staticData.GetInputFactorOrder(); stringstream in(source + "\n"); sentence.Read(in,inputFactorOrder); - size_t lineNumber = 0; // TODO: Include sentence request number here? - Manager manager(lineNumber, sentence, staticData.GetSearchAlgorithm()); + Manager manager(sentence, staticData.GetSearchAlgorithm()); manager.ProcessSentence(); const Hypothesis* hypo = manager.GetBestHypothesis(); diff --git a/mira/Decoder.cpp b/mira/Decoder.cpp index 122106b96..529ae8ad1 100644 --- a/mira/Decoder.cpp +++ b/mira/Decoder.cpp @@ -143,7 +143,7 @@ vector< vector > MosesDecoder::runDecoder(const std::string& source string filename) { // run the decoder - m_manager = new Moses::Manager(0,*m_sentence, search); + m_manager = new Moses::Manager(*m_sentence, search); m_manager->ProcessSentence(); TrellisPathList nBestList; m_manager->CalcNBest(nBestSize, nBestList, distinct); diff --git a/moses-cmd/LatticeMBRGrid.cpp b/moses-cmd/LatticeMBRGrid.cpp index 3573d0697..677adb791 100644 --- a/moses-cmd/LatticeMBRGrid.cpp +++ b/moses-cmd/LatticeMBRGrid.cpp @@ -159,7 +159,7 @@ int main(int argc, char* argv[]) StaticData& staticData = const_cast(StaticData::Instance()); staticData.SetUseLatticeMBR(true); - IOWrapper* ioWrapper = GetIOWrapper(staticData); + IOWrapper* ioWrapper = IOWrapper::GetIOWrapper(staticData); if (!ioWrapper) { throw runtime_error("Failed to initialise IOWrapper"); @@ -180,8 +180,9 @@ int main(int argc, char* argv[]) while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) { ++lineCount; - Sentence sentence; - Manager manager(lineCount, *source, staticData.GetSearchAlgorithm()); + source->SetTranslationId(lineCount); + + Manager manager(*source, staticData.GetSearchAlgorithm()); manager.ProcessSentence(); TrellisPathList nBestList; manager.CalcNBest(nBestSize, nBestList,true); diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 994837d4a..513c3a187 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -142,6 +142,7 @@ int main(int argc, char** argv) InputType* source = NULL; size_t lineCount = staticData.GetStartTranslationId(); while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) { + source->SetTranslationId(lineCount); IFVERBOSE(1) { ResetUserTime(); } @@ -150,7 +151,7 @@ int main(int argc, char** argv) // set up task of translating one sentence TranslationTask* task = - new TranslationTask(lineCount,source, *ioWrapper, + new TranslationTask(source, *ioWrapper, staticData.GetOutputSearchGraphSLF(), hypergraphOutput); // execute task diff --git a/moses/HypergraphOutput.cpp b/moses/HypergraphOutput.cpp index da7e804dc..5c689eaee 100644 --- a/moses/HypergraphOutput.cpp +++ b/moses/HypergraphOutput.cpp @@ -124,7 +124,7 @@ template void HypergraphOutput::Write(const M& manager) const { stringstream fileName; - fileName << m_hypergraphDir << "/" << manager.GetLineNumber(); + fileName << m_hypergraphDir << "/" << manager.GetSource().GetTranslationId(); if ( m_appendSuffix ) { fileName << "." << m_compression; } @@ -144,7 +144,7 @@ void HypergraphOutput::Write(const M& manager) const { manager.OutputSearchGraphAsHypergraph(file); file.flush(); } else { - TRACE_ERR("Cannot output hypergraph for line " << manager.GetLineNumber() + TRACE_ERR("Cannot output hypergraph for line " << manager.GetSource().GetTranslationId() << " because the output file " << fileName.str() << " is not open or not ready for writing" << std::endl); diff --git a/moses/Manager.cpp b/moses/Manager.cpp index 5ebd0b9c4..7a27dcaaf 100644 --- a/moses/Manager.cpp +++ b/moses/Manager.cpp @@ -54,12 +54,11 @@ using namespace std; namespace Moses { -Manager::Manager(size_t lineNumber, InputType const& source, SearchAlgorithm searchAlgorithm) +Manager::Manager(InputType const& source, SearchAlgorithm searchAlgorithm) :m_transOptColl(source.CreateTranslationOptionCollection()) ,m_search(Search::CreateSearch(*this, source, searchAlgorithm, *m_transOptColl)) ,interrupted_flag(0) ,m_hypoId(0) - ,m_lineNumber(lineNumber) ,m_source(source) { StaticData::Instance().InitializeForInput(m_source); @@ -105,7 +104,7 @@ void Manager::ProcessSentence() // some reporting on how long this took IFVERBOSE(1) { GetSentenceStats().StopTimeCollectOpts(); - TRACE_ERR("Line "<< m_lineNumber << ": Collecting options took " + TRACE_ERR("Line "<< m_source.GetTranslationId() << ": Collecting options took " << GetSentenceStats().GetTimeCollectOpts() << " seconds at " << __FILE__ << ":" << __LINE__ << endl); } @@ -114,7 +113,7 @@ void Manager::ProcessSentence() Timer searchTime; searchTime.start(); m_search->ProcessSentence(); - VERBOSE(1, "Line " << m_lineNumber << ": Search took " << searchTime << " seconds" << endl); + VERBOSE(1, "Line " << m_source.GetTranslationId() << ": Search took " << searchTime << " seconds" << endl); IFVERBOSE(2) { GetSentenceStats().StopTimeTotal(); TRACE_ERR(GetSentenceStats()); @@ -831,7 +830,7 @@ size_t Manager::OutputFeatureValuesForSLF(size_t index, bool zeros, const Hypoth void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const { - VERBOSE(2,"Getting search graph to output as hypergraph for sentence " << m_lineNumber << std::endl) + VERBOSE(2,"Getting search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl) vector searchGraph; GetSearchGraph(searchGraph); @@ -842,7 +841,7 @@ void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStrea set terminalNodes; multimap hypergraphIDToArcs; - VERBOSE(2,"Gathering information about search graph to output as hypergraph for sentence " << m_lineNumber << std::endl) + VERBOSE(2,"Gathering information about search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl) long numNodes = 0; long endNode = 0; @@ -904,15 +903,15 @@ void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStrea // Print number of nodes and arcs outputSearchGraphStream << numNodes << " " << numArcs << endl; - VERBOSE(2,"Search graph to output as hypergraph for sentence " << m_lineNumber + VERBOSE(2,"Search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << " contains " << numArcs << " arcs and " << numNodes << " nodes" << std::endl) - VERBOSE(2,"Outputting search graph to output as hypergraph for sentence " << m_lineNumber << std::endl) + VERBOSE(2,"Outputting search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl) for (int hypergraphHypothesisID=0; hypergraphHypothesisID < endNode; hypergraphHypothesisID+=1) { if (hypergraphHypothesisID % 100000 == 0) { - VERBOSE(2,"Processed " << hypergraphHypothesisID << " of " << numNodes << " hypergraph nodes for sentence " << m_lineNumber << std::endl); + VERBOSE(2,"Processed " << hypergraphHypothesisID << " of " << numNodes << " hypergraph nodes for sentence " << m_source.GetTranslationId() << std::endl); } // int mosesID = hypergraphIDToMosesID[hypergraphHypothesisID]; size_t count = hypergraphIDToArcs.count(hypergraphHypothesisID); @@ -935,7 +934,7 @@ void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStrea // int actualHypergraphHypothesisID = mosesIDToHypergraphID[mosesHypothesisID]; UTIL_THROW_IF2( (hypergraphHypothesisID != mosesIDToHypergraphID[mosesHypothesisID]), - "Error while writing search lattice as hypergraph for sentence " << m_lineNumber << ". " << + "Error while writing search lattice as hypergraph for sentence " << m_source.GetTranslationId() << ". " << "Moses node " << mosesHypothesisID << " was expected to have hypergraph id " << hypergraphHypothesisID << ", but actually had hypergraph id " << mosesIDToHypergraphID[mosesHypothesisID] << ". There are " << numNodes << " nodes in the search lattice." @@ -950,7 +949,7 @@ void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStrea // VERBOSE(2,"Hypergraph node " << hypergraphHypothesisID << " has parent node " << startNode << std::endl) UTIL_THROW_IF2( (startNode >= hypergraphHypothesisID), - "Error while writing search lattice as hypergraph for sentence" << m_lineNumber << ". " << + "Error while writing search lattice as hypergraph for sentence" << m_source.GetTranslationId() << ". " << "The nodes must be output in topological order. The code attempted to violate this restriction." ); diff --git a/moses/Manager.h b/moses/Manager.h index 9512bb472..ef4612de1 100644 --- a/moses/Manager.h +++ b/moses/Manager.h @@ -118,7 +118,6 @@ protected: size_t interrupted_flag; std::auto_ptr m_sentenceStats; int m_hypoId; //used to number the hypos as they are created. - size_t m_lineNumber; void GetConnectedGraph( std::map< int, bool >* pConnected, @@ -130,7 +129,7 @@ protected: public: InputType const& m_source; /**< source sentence to be translated */ - Manager(size_t lineNumber, InputType const& source, SearchAlgorithm searchAlgorithm); + Manager(InputType const& source, SearchAlgorithm searchAlgorithm); ~Manager(); const TranslationOptionCollection* getSntTranslationOptions(); @@ -145,7 +144,7 @@ public: void GetOutputLanguageModelOrder( std::ostream &out, const Hypothesis *hypo ); void GetWordGraph(long translationId, std::ostream &outputWordGraphStream) const; int GetNextHypoId(); - size_t GetLineNumber() const {return m_lineNumber;} + #ifdef HAVE_PROTOBUF void SerializeSearchGraphPB(long translationId, std::ostream& outputStream) const; #endif diff --git a/moses/MockHypothesis.cpp b/moses/MockHypothesis.cpp index 3f68bd9a8..c18b58a5e 100644 --- a/moses/MockHypothesis.cpp +++ b/moses/MockHypothesis.cpp @@ -41,7 +41,7 @@ MockHypothesisGuard::MockHypothesisGuard( m_wp("WordPenalty"), m_uwp("UnknownWordPenalty"), m_dist("Distortion"), - m_manager(0,m_sentence,Normal) + m_manager(m_sentence,Normal) { BOOST_CHECK_EQUAL(alignments.size(), targetSegments.size()); diff --git a/moses/TranslationTask.cpp b/moses/TranslationTask.cpp index a107ff58a..c2f33db27 100644 --- a/moses/TranslationTask.cpp +++ b/moses/TranslationTask.cpp @@ -15,10 +15,10 @@ using namespace Moses; namespace MosesCmd { -TranslationTask::TranslationTask(size_t lineNumber, InputType* source, MosesCmd::IOWrapper &ioWrapper, +TranslationTask::TranslationTask(InputType* source, MosesCmd::IOWrapper &ioWrapper, bool outputSearchGraphSLF, boost::shared_ptr > hypergraphOutput) : - m_source(source), m_lineNumber(lineNumber), + m_source(source), m_ioWrapper(ioWrapper), m_outputSearchGraphSLF(outputSearchGraphSLF), m_hypergraphOutput(hypergraphOutput) @@ -41,7 +41,7 @@ void TranslationTask::Run() { // report thread number #if defined(WITH_THREADS) && defined(BOOST_HAS_PTHREADS) - TRACE_ERR("Translating line " << m_lineNumber << " in thread id " << pthread_self() << endl); + TRACE_ERR("Translating line " << m_source->GetTranslationId() << " in thread id " << pthread_self() << endl); #endif @@ -50,8 +50,8 @@ void TranslationTask::Run() { // we still need to apply the decision rule (MAP, MBR, ...) Timer initTime; initTime.start(); - Manager manager(m_lineNumber, *m_source,staticData.GetSearchAlgorithm()); - VERBOSE(1, "Line " << m_lineNumber << ": Initialize search took " << initTime << " seconds total" << endl); + Manager manager(*m_source,staticData.GetSearchAlgorithm()); + VERBOSE(1, "Line " << m_source->GetTranslationId() << ": Initialize search took " << initTime << " seconds total" << endl); manager.ProcessSentence(); // we are done with search, let's look what we got @@ -62,25 +62,25 @@ void TranslationTask::Run() { if (m_ioWrapper.GetWordGraphCollector()) { ostringstream out; fix(out,PRECISION); - manager.GetWordGraph(m_lineNumber, out); - m_ioWrapper.GetWordGraphCollector()->Write(m_lineNumber, out.str()); + manager.GetWordGraph(m_source->GetTranslationId(), out); + m_ioWrapper.GetWordGraphCollector()->Write(m_source->GetTranslationId(), out.str()); } // output search graph if (m_ioWrapper.GetSearchGraphOutputCollector()) { ostringstream out; fix(out,PRECISION); - manager.OutputSearchGraph(m_lineNumber, out); - m_ioWrapper.GetSearchGraphOutputCollector()->Write(m_lineNumber, out.str()); + manager.OutputSearchGraph(m_source->GetTranslationId(), out); + m_ioWrapper.GetSearchGraphOutputCollector()->Write(m_source->GetTranslationId(), out.str()); #ifdef HAVE_PROTOBUF if (staticData.GetOutputSearchGraphPB()) { ostringstream sfn; - sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_lineNumber << ".pb" << ends; + sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_source->GetTranslationId() << ".pb" << ends; string fn = sfn.str(); VERBOSE(2, "Writing search graph to " << fn << endl); fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out); - manager.SerializeSearchGraphPB(m_lineNumber, output); + manager.SerializeSearchGraphPB(m_source->GetTranslationId(), output); } #endif } @@ -88,17 +88,17 @@ void TranslationTask::Run() { // Output search graph in HTK standard lattice format (SLF) if (m_outputSearchGraphSLF) { stringstream fileName; - fileName << staticData.GetParam("output-search-graph-slf")[0] << "/" << m_lineNumber << ".slf"; + fileName << staticData.GetParam("output-search-graph-slf")[0] << "/" << m_source->GetTranslationId() << ".slf"; ofstream *file = new ofstream; file->open(fileName.str().c_str()); if (file->is_open() && file->good()) { ostringstream out; fix(out,PRECISION); - manager.OutputSearchGraphAsSLF(m_lineNumber, out); + manager.OutputSearchGraphAsSLF(m_source->GetTranslationId(), out); *file << out.str(); file -> flush(); } else { - TRACE_ERR("Cannot output HTK standard lattice for line " << m_lineNumber << " because the output file is not open or not ready for writing" << endl); + TRACE_ERR("Cannot output HTK standard lattice for line " << m_source->GetTranslationId() << " because the output file is not open or not ready for writing" << endl); } delete file; } @@ -119,7 +119,7 @@ void TranslationTask::Run() { // all derivations - send them to debug stream if (staticData.PrintAllDerivations()) { additionalReportingTime.start(); - manager.PrintAllDerivations(m_lineNumber, debug); + manager.PrintAllDerivations(m_source->GetTranslationId(), debug); additionalReportingTime.stop(); } @@ -156,7 +156,7 @@ void TranslationTask::Run() { OutputAlignment(out, bestHypo); } - OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_lineNumber, bestHypo); + OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_source->GetTranslationId(), bestHypo); IFVERBOSE(1) { debug << "BEST TRANSLATION: " << *bestHypo << endl; } @@ -190,12 +190,12 @@ void TranslationTask::Run() { size_t n = min(nBestSize, staticData.GetNBestSize()); getLatticeMBRNBest(manager,nBestList,solutions,n); ostringstream out; - OutputLatticeMBRNBest(out, solutions,m_lineNumber); - m_ioWrapper.GetNBestOutputCollector()->Write(m_lineNumber, out.str()); + OutputLatticeMBRNBest(out, solutions,m_source->GetTranslationId()); + m_ioWrapper.GetNBestOutputCollector()->Write(m_source->GetTranslationId(), out.str()); } else { //Lattice MBR decoding vector mbrBestHypo = doLatticeMBR(manager,nBestList); - OutputBestHypo(mbrBestHypo, m_lineNumber, staticData.GetReportSegmentation(), + OutputBestHypo(mbrBestHypo, m_source->GetTranslationId(), staticData.GetReportSegmentation(), staticData.GetReportAllFactors(),out); IFVERBOSE(2) { PrintUserTime("finished Lattice MBR decoding"); @@ -206,10 +206,10 @@ void TranslationTask::Run() { // consensus decoding else if (staticData.UseConsensusDecoding()) { const TrellisPath &conBestHypo = doConsensusDecoding(manager,nBestList); - OutputBestHypo(conBestHypo, m_lineNumber, + OutputBestHypo(conBestHypo, m_source->GetTranslationId(), staticData.GetReportSegmentation(), staticData.GetReportAllFactors(),out); - OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_lineNumber, conBestHypo); + OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_source->GetTranslationId(), conBestHypo); IFVERBOSE(2) { PrintUserTime("finished Consensus decoding"); } @@ -218,10 +218,10 @@ void TranslationTask::Run() { // n-best MBR decoding else { const TrellisPath &mbrBestHypo = doMBR(nBestList); - OutputBestHypo(mbrBestHypo, m_lineNumber, + OutputBestHypo(mbrBestHypo, m_source->GetTranslationId(), staticData.GetReportSegmentation(), staticData.GetReportAllFactors(),out); - OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_lineNumber, mbrBestHypo); + OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_source->GetTranslationId(), mbrBestHypo); IFVERBOSE(2) { PrintUserTime("finished MBR decoding"); } @@ -229,10 +229,10 @@ void TranslationTask::Run() { } // report best translation to output collector - m_ioWrapper.GetSingleBestOutputCollector()->Write(m_lineNumber,out.str(),debug.str()); + m_ioWrapper.GetSingleBestOutputCollector()->Write(m_source->GetTranslationId(),out.str(),debug.str()); decisionRuleTime.stop(); - VERBOSE(1, "Line " << m_lineNumber << ": Decision rule took " << decisionRuleTime << " seconds total" << endl); + VERBOSE(1, "Line " << m_source->GetTranslationId() << ": Decision rule took " << decisionRuleTime << " seconds total" << endl); } additionalReportingTime.start(); @@ -242,9 +242,9 @@ void TranslationTask::Run() { TrellisPathList nBestList; ostringstream out; manager.CalcNBest(staticData.GetNBestSize(), nBestList,staticData.GetDistinctNBest()); - OutputNBest(out, nBestList, staticData.GetOutputFactorOrder(), m_lineNumber, + OutputNBest(out, nBestList, staticData.GetOutputFactorOrder(), m_source->GetTranslationId(), staticData.GetReportSegmentation()); - m_ioWrapper.GetNBestOutputCollector()->Write(m_lineNumber, out.str()); + m_ioWrapper.GetNBestOutputCollector()->Write(m_source->GetTranslationId(), out.str()); } //lattice samples @@ -252,9 +252,9 @@ void TranslationTask::Run() { TrellisPathList latticeSamples; ostringstream out; manager.CalcLatticeSamples(staticData.GetLatticeSamplesSize(), latticeSamples); - OutputNBest(out,latticeSamples, staticData.GetOutputFactorOrder(), m_lineNumber, + OutputNBest(out,latticeSamples, staticData.GetOutputFactorOrder(), m_source->GetTranslationId(), staticData.GetReportSegmentation()); - m_ioWrapper.GetLatticeSamplesCollector()->Write(m_lineNumber, out.str()); + m_ioWrapper.GetLatticeSamplesCollector()->Write(m_source->GetTranslationId(), out.str()); } // detailed translation reporting @@ -262,7 +262,7 @@ void TranslationTask::Run() { ostringstream out; fix(out,PRECISION); TranslationAnalysis::PrintTranslationAnalysis(out, manager.GetBestHypothesis()); - m_ioWrapper.GetDetailedTranslationCollector()->Write(m_lineNumber,out.str()); + m_ioWrapper.GetDetailedTranslationCollector()->Write(m_source->GetTranslationId(),out.str()); } //list of unknown words @@ -273,13 +273,13 @@ void TranslationTask::Run() { out << *(unknowns[i]); } out << endl; - m_ioWrapper.GetUnknownsCollector()->Write(m_lineNumber, out.str()); + m_ioWrapper.GetUnknownsCollector()->Write(m_source->GetTranslationId(), out.str()); } // report additional statistics manager.CalcDecoderStatistics(); - VERBOSE(1, "Line " << m_lineNumber << ": Additional reporting took " << additionalReportingTime << " seconds total" << endl); - VERBOSE(1, "Line " << m_lineNumber << ": Translation took " << translationTime << " seconds total" << endl); + VERBOSE(1, "Line " << m_source->GetTranslationId() << ": Additional reporting took " << additionalReportingTime << " seconds total" << endl); + VERBOSE(1, "Line " << m_source->GetTranslationId() << ": Translation took " << translationTime << " seconds total" << endl); IFVERBOSE(2) { PrintUserTime("Sentence Decoding Time:"); } diff --git a/moses/TranslationTask.h b/moses/TranslationTask.h index edd2ca347..39a66a74a 100644 --- a/moses/TranslationTask.h +++ b/moses/TranslationTask.h @@ -26,7 +26,7 @@ class TranslationTask : public Moses::Task public: - TranslationTask(size_t lineNumber, Moses::InputType* source, MosesCmd::IOWrapper &ioWrapper, + TranslationTask(Moses::InputType* source, MosesCmd::IOWrapper &ioWrapper, bool outputSearchGraphSLF, boost::shared_ptr > hypergraphOutput); @@ -39,7 +39,6 @@ public: private: Moses::InputType* m_source; - size_t m_lineNumber; MosesCmd::IOWrapper &m_ioWrapper; bool m_outputSearchGraphSLF;