moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-08-17 23:40:50 +03:00
Code Issues Packages Projects Releases Wiki Activity

delete lineNumber variable from Manager and TranslationTask. It should always be carried by the input sentence itself

Browse Source
This commit is contained in:
Hieu Hoang 2014-10-07 19:16:30 +01:00
parent 0d7048803a
commit 433186ae25
11 changed files with 62 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
contrib/other-builds/server/.project
View File

@ -33,12 +33,12 @@
<link>
<name>IOWrapper.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses-cmd/IOWrapper.cpp</locationURI>
<locationURI>PARENT-3-PROJECT_LOC/moses/IOWrapper.cpp</locationURI>
</link>
<link>
<name>IOWrapper.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses-cmd/IOWrapper.h</locationURI>
<locationURI>PARENT-3-PROJECT_LOC/moses/IOWrapper.h</locationURI>
</link>
<link>
<name>mosesserver.cpp</name>

6
contrib/server/mosesserver.cpp
View File

@ -292,13 +292,15 @@ public:
m_retData.insert(pair<string, xmlrpc_c::value>("sg", xmlrpc_c::value_string(sgstream.str())));
}
} else {
size_t lineNumber = 0; // TODO: Include sentence request number here?
Sentence sentence;
sentence.SetTranslationId(lineNumber);
const vector<FactorType> &
inputFactorOrder = staticData.GetInputFactorOrder();
stringstream in(source + "\n");
sentence.Read(in,inputFactorOrder);
size_t lineNumber = 0; // TODO: Include sentence request number here?
Manager manager(lineNumber, sentence, staticData.GetSearchAlgorithm());
Manager manager(sentence, staticData.GetSearchAlgorithm());
manager.ProcessSentence();
const Hypothesis* hypo = manager.GetBestHypothesis();

2
mira/Decoder.cpp
View File

@ -143,7 +143,7 @@ vector< vector<const Word*> > MosesDecoder::runDecoder(const std::string& source
string filename)
{
// run the decoder
m_manager = new Moses::Manager(0,*m_sentence, search);
m_manager = new Moses::Manager(*m_sentence, search);
m_manager->ProcessSentence();
TrellisPathList nBestList;
m_manager->CalcNBest(nBestSize, nBestList, distinct);

7
moses-cmd/LatticeMBRGrid.cpp
View File

@ -159,7 +159,7 @@ int main(int argc, char* argv[])
StaticData& staticData = const_cast<StaticData&>(StaticData::Instance());
staticData.SetUseLatticeMBR(true);
IOWrapper* ioWrapper = GetIOWrapper(staticData);
IOWrapper* ioWrapper = IOWrapper::GetIOWrapper(staticData);
if (!ioWrapper) {
throw runtime_error("Failed to initialise IOWrapper");
@ -180,8 +180,9 @@ int main(int argc, char* argv[])
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
++lineCount;
Sentence sentence;
Manager manager(lineCount, *source, staticData.GetSearchAlgorithm());
source->SetTranslationId(lineCount);
Manager manager(*source, staticData.GetSearchAlgorithm());
manager.ProcessSentence();
TrellisPathList nBestList;
manager.CalcNBest(nBestSize, nBestList,true);

3
moses-cmd/Main.cpp
View File

@ -142,6 +142,7 @@ int main(int argc, char** argv)
InputType* source = NULL;
size_t lineCount = staticData.GetStartTranslationId();
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
source->SetTranslationId(lineCount);
IFVERBOSE(1) {
ResetUserTime();
}
@ -150,7 +151,7 @@ int main(int argc, char** argv)
// set up task of translating one sentence
TranslationTask* task =
new TranslationTask(lineCount,source, *ioWrapper,
new TranslationTask(source, *ioWrapper,
staticData.GetOutputSearchGraphSLF(),
hypergraphOutput);
// execute task

4
moses/HypergraphOutput.cpp
View File

@ -124,7 +124,7 @@ template<class M>
void HypergraphOutput<M>::Write(const M& manager) const {
stringstream fileName;
fileName << m_hypergraphDir << "/" << manager.GetLineNumber();
fileName << m_hypergraphDir << "/" << manager.GetSource().GetTranslationId();
if ( m_appendSuffix ) {
fileName << "." << m_compression;
}
@ -144,7 +144,7 @@ void HypergraphOutput<M>::Write(const M& manager) const {
manager.OutputSearchGraphAsHypergraph(file);
file.flush();
} else {
TRACE_ERR("Cannot output hypergraph for line " << manager.GetLineNumber()
TRACE_ERR("Cannot output hypergraph for line " << manager.GetSource().GetTranslationId()
<< " because the output file " << fileName.str()
<< " is not open or not ready for writing"
<< std::endl);

21
moses/Manager.cpp
View File

@ -54,12 +54,11 @@ using namespace std;
namespace Moses
{
Manager::Manager(size_t lineNumber, InputType const& source, SearchAlgorithm searchAlgorithm)
Manager::Manager(InputType const& source, SearchAlgorithm searchAlgorithm)
:m_transOptColl(source.CreateTranslationOptionCollection())
,m_search(Search::CreateSearch(*this, source, searchAlgorithm, *m_transOptColl))
,interrupted_flag(0)
,m_hypoId(0)
,m_lineNumber(lineNumber)
,m_source(source)
{
StaticData::Instance().InitializeForInput(m_source);
@ -105,7 +104,7 @@ void Manager::ProcessSentence()
// some reporting on how long this took
IFVERBOSE(1) {
GetSentenceStats().StopTimeCollectOpts();
TRACE_ERR("Line "<< m_lineNumber << ": Collecting options took "
TRACE_ERR("Line "<< m_source.GetTranslationId() << ": Collecting options took "
<< GetSentenceStats().GetTimeCollectOpts() << " seconds at "
<< __FILE__ << ":" << __LINE__ << endl);
}
@ -114,7 +113,7 @@ void Manager::ProcessSentence()
Timer searchTime;
searchTime.start();
m_search->ProcessSentence();
VERBOSE(1, "Line " << m_lineNumber << ": Search took " << searchTime << " seconds" << endl);
VERBOSE(1, "Line " << m_source.GetTranslationId() << ": Search took " << searchTime << " seconds" << endl);
IFVERBOSE(2) {
GetSentenceStats().StopTimeTotal();
TRACE_ERR(GetSentenceStats());
@ -831,7 +830,7 @@ size_t Manager::OutputFeatureValuesForSLF(size_t index, bool zeros, const Hypoth
void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const
{
VERBOSE(2,"Getting search graph to output as hypergraph for sentence " << m_lineNumber << std::endl)
VERBOSE(2,"Getting search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl)
vector<SearchGraphNode> searchGraph;
GetSearchGraph(searchGraph);
@ -842,7 +841,7 @@ void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStrea
set<int> terminalNodes;
multimap<int,int> hypergraphIDToArcs;
VERBOSE(2,"Gathering information about search graph to output as hypergraph for sentence " << m_lineNumber << std::endl)
VERBOSE(2,"Gathering information about search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl)
long numNodes = 0;
long endNode = 0;
@ -904,15 +903,15 @@ void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStrea
// Print number of nodes and arcs
outputSearchGraphStream << numNodes << " " << numArcs << endl;
VERBOSE(2,"Search graph to output as hypergraph for sentence " << m_lineNumber
VERBOSE(2,"Search graph to output as hypergraph for sentence " << m_source.GetTranslationId()
<< " contains " << numArcs << " arcs and " << numNodes << " nodes" << std::endl)
VERBOSE(2,"Outputting search graph to output as hypergraph for sentence " << m_lineNumber << std::endl)
VERBOSE(2,"Outputting search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl)
for (int hypergraphHypothesisID=0; hypergraphHypothesisID < endNode; hypergraphHypothesisID+=1) {
if (hypergraphHypothesisID % 100000 == 0) {
VERBOSE(2,"Processed " << hypergraphHypothesisID << " of " << numNodes << " hypergraph nodes for sentence " << m_lineNumber << std::endl);
VERBOSE(2,"Processed " << hypergraphHypothesisID << " of " << numNodes << " hypergraph nodes for sentence " << m_source.GetTranslationId() << std::endl);
}
// int mosesID = hypergraphIDToMosesID[hypergraphHypothesisID];
size_t count = hypergraphIDToArcs.count(hypergraphHypothesisID);
@ -935,7 +934,7 @@ void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStrea
// int actualHypergraphHypothesisID = mosesIDToHypergraphID[mosesHypothesisID];
UTIL_THROW_IF2(
(hypergraphHypothesisID != mosesIDToHypergraphID[mosesHypothesisID]),
"Error while writing search lattice as hypergraph for sentence " << m_lineNumber << ". " <<
"Error while writing search lattice as hypergraph for sentence " << m_source.GetTranslationId() << ". " <<
"Moses node " << mosesHypothesisID << " was expected to have hypergraph id " << hypergraphHypothesisID <<
", but actually had hypergraph id " << mosesIDToHypergraphID[mosesHypothesisID] <<
". There are " << numNodes << " nodes in the search lattice."
@ -950,7 +949,7 @@ void Manager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStrea
// VERBOSE(2,"Hypergraph node " << hypergraphHypothesisID << " has parent node " << startNode << std::endl)
UTIL_THROW_IF2(
(startNode >= hypergraphHypothesisID),
"Error while writing search lattice as hypergraph for sentence" << m_lineNumber << ". " <<
"Error while writing search lattice as hypergraph for sentence" << m_source.GetTranslationId() << ". " <<
"The nodes must be output in topological order. The code attempted to violate this restriction."
);

5
moses/Manager.h
View File

@ -118,7 +118,6 @@ protected:
size_t interrupted_flag;
std::auto_ptr<SentenceStats> m_sentenceStats;
int m_hypoId; //used to number the hypos as they are created.
size_t m_lineNumber;
void GetConnectedGraph(
std::map< int, bool >* pConnected,
@ -130,7 +129,7 @@ protected:
public:
InputType const& m_source; /**< source sentence to be translated */
Manager(size_t lineNumber, InputType const& source, SearchAlgorithm searchAlgorithm);
Manager(InputType const& source, SearchAlgorithm searchAlgorithm);
~Manager();
const TranslationOptionCollection* getSntTranslationOptions();
@ -145,7 +144,7 @@ public:
void GetOutputLanguageModelOrder( std::ostream &out, const Hypothesis *hypo );
void GetWordGraph(long translationId, std::ostream &outputWordGraphStream) const;
int GetNextHypoId();
size_t GetLineNumber() const {return m_lineNumber;}
#ifdef HAVE_PROTOBUF
void SerializeSearchGraphPB(long translationId, std::ostream& outputStream) const;
#endif

2
moses/MockHypothesis.cpp
View File

@ -41,7 +41,7 @@ MockHypothesisGuard::MockHypothesisGuard(
m_wp("WordPenalty"),
m_uwp("UnknownWordPenalty"),
m_dist("Distortion"),
m_manager(0,m_sentence,Normal)
m_manager(m_sentence,Normal)
{
BOOST_CHECK_EQUAL(alignments.size(), targetSegments.size());

66
moses/TranslationTask.cpp
View File

@ -15,10 +15,10 @@ using namespace Moses;
namespace MosesCmd
{
TranslationTask::TranslationTask(size_t lineNumber, InputType* source, MosesCmd::IOWrapper &ioWrapper,
TranslationTask::TranslationTask(InputType* source, MosesCmd::IOWrapper &ioWrapper,
bool outputSearchGraphSLF,
boost::shared_ptr<HypergraphOutput<Manager> > hypergraphOutput) :
m_source(source), m_lineNumber(lineNumber),
m_source(source),
m_ioWrapper(ioWrapper),
m_outputSearchGraphSLF(outputSearchGraphSLF),
m_hypergraphOutput(hypergraphOutput)
@ -41,7 +41,7 @@ void TranslationTask::Run() {
// report thread number
#if defined(WITH_THREADS) && defined(BOOST_HAS_PTHREADS)
TRACE_ERR("Translating line " << m_lineNumber << " in thread id " << pthread_self() << endl);
TRACE_ERR("Translating line " << m_source->GetTranslationId() << " in thread id " << pthread_self() << endl);
#endif
@ -50,8 +50,8 @@ void TranslationTask::Run() {
// we still need to apply the decision rule (MAP, MBR, ...)
Timer initTime;
initTime.start();
Manager manager(m_lineNumber, *m_source,staticData.GetSearchAlgorithm());
VERBOSE(1, "Line " << m_lineNumber << ": Initialize search took " << initTime << " seconds total" << endl);
Manager manager(*m_source,staticData.GetSearchAlgorithm());
VERBOSE(1, "Line " << m_source->GetTranslationId() << ": Initialize search took " << initTime << " seconds total" << endl);
manager.ProcessSentence();
// we are done with search, let's look what we got
@ -62,25 +62,25 @@ void TranslationTask::Run() {
if (m_ioWrapper.GetWordGraphCollector()) {
ostringstream out;
fix(out,PRECISION);
manager.GetWordGraph(m_lineNumber, out);
m_ioWrapper.GetWordGraphCollector()->Write(m_lineNumber, out.str());
manager.GetWordGraph(m_source->GetTranslationId(), out);
m_ioWrapper.GetWordGraphCollector()->Write(m_source->GetTranslationId(), out.str());
}
// output search graph
if (m_ioWrapper.GetSearchGraphOutputCollector()) {
ostringstream out;
fix(out,PRECISION);
manager.OutputSearchGraph(m_lineNumber, out);
m_ioWrapper.GetSearchGraphOutputCollector()->Write(m_lineNumber, out.str());
manager.OutputSearchGraph(m_source->GetTranslationId(), out);
m_ioWrapper.GetSearchGraphOutputCollector()->Write(m_source->GetTranslationId(), out.str());
#ifdef HAVE_PROTOBUF
if (staticData.GetOutputSearchGraphPB()) {
ostringstream sfn;
sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_lineNumber << ".pb" << ends;
sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_source->GetTranslationId() << ".pb" << ends;
string fn = sfn.str();
VERBOSE(2, "Writing search graph to " << fn << endl);
fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out);
manager.SerializeSearchGraphPB(m_lineNumber, output);
manager.SerializeSearchGraphPB(m_source->GetTranslationId(), output);
}
#endif
}
@ -88,17 +88,17 @@ void TranslationTask::Run() {
// Output search graph in HTK standard lattice format (SLF)
if (m_outputSearchGraphSLF) {
stringstream fileName;
fileName << staticData.GetParam("output-search-graph-slf")[0] << "/" << m_lineNumber << ".slf";
fileName << staticData.GetParam("output-search-graph-slf")[0] << "/" << m_source->GetTranslationId() << ".slf";
ofstream *file = new ofstream;
file->open(fileName.str().c_str());
if (file->is_open() && file->good()) {
ostringstream out;
fix(out,PRECISION);
manager.OutputSearchGraphAsSLF(m_lineNumber, out);
manager.OutputSearchGraphAsSLF(m_source->GetTranslationId(), out);
*file << out.str();
file -> flush();
} else {
TRACE_ERR("Cannot output HTK standard lattice for line " << m_lineNumber << " because the output file is not open or not ready for writing" << endl);
TRACE_ERR("Cannot output HTK standard lattice for line " << m_source->GetTranslationId() << " because the output file is not open or not ready for writing" << endl);
}
delete file;
}
@ -119,7 +119,7 @@ void TranslationTask::Run() {
// all derivations - send them to debug stream
if (staticData.PrintAllDerivations()) {
additionalReportingTime.start();
manager.PrintAllDerivations(m_lineNumber, debug);
manager.PrintAllDerivations(m_source->GetTranslationId(), debug);
additionalReportingTime.stop();
}
@ -156,7 +156,7 @@ void TranslationTask::Run() {
OutputAlignment(out, bestHypo);
}
OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_lineNumber, bestHypo);
OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_source->GetTranslationId(), bestHypo);
IFVERBOSE(1) {
debug << "BEST TRANSLATION: " << *bestHypo << endl;
}
@ -190,12 +190,12 @@ void TranslationTask::Run() {
size_t n = min(nBestSize, staticData.GetNBestSize());
getLatticeMBRNBest(manager,nBestList,solutions,n);
ostringstream out;
OutputLatticeMBRNBest(out, solutions,m_lineNumber);
m_ioWrapper.GetNBestOutputCollector()->Write(m_lineNumber, out.str());
OutputLatticeMBRNBest(out, solutions,m_source->GetTranslationId());
m_ioWrapper.GetNBestOutputCollector()->Write(m_source->GetTranslationId(), out.str());
} else {
//Lattice MBR decoding
vector<Word> mbrBestHypo = doLatticeMBR(manager,nBestList);
OutputBestHypo(mbrBestHypo, m_lineNumber, staticData.GetReportSegmentation(),
OutputBestHypo(mbrBestHypo, m_source->GetTranslationId(), staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),out);
IFVERBOSE(2) {
PrintUserTime("finished Lattice MBR decoding");
@ -206,10 +206,10 @@ void TranslationTask::Run() {
// consensus decoding
else if (staticData.UseConsensusDecoding()) {
const TrellisPath &conBestHypo = doConsensusDecoding(manager,nBestList);
OutputBestHypo(conBestHypo, m_lineNumber,
OutputBestHypo(conBestHypo, m_source->GetTranslationId(),
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),out);
OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_lineNumber, conBestHypo);
OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_source->GetTranslationId(), conBestHypo);
IFVERBOSE(2) {
PrintUserTime("finished Consensus decoding");
}
@ -218,10 +218,10 @@ void TranslationTask::Run() {
// n-best MBR decoding
else {
const TrellisPath &mbrBestHypo = doMBR(nBestList);
OutputBestHypo(mbrBestHypo, m_lineNumber,
OutputBestHypo(mbrBestHypo, m_source->GetTranslationId(),
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),out);
OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_lineNumber, mbrBestHypo);
OutputAlignment(m_ioWrapper.GetAlignmentInfoCollector(), m_source->GetTranslationId(), mbrBestHypo);
IFVERBOSE(2) {
PrintUserTime("finished MBR decoding");
}
@ -229,10 +229,10 @@ void TranslationTask::Run() {
}
// report best translation to output collector
m_ioWrapper.GetSingleBestOutputCollector()->Write(m_lineNumber,out.str(),debug.str());
m_ioWrapper.GetSingleBestOutputCollector()->Write(m_source->GetTranslationId(),out.str(),debug.str());
decisionRuleTime.stop();
VERBOSE(1, "Line " << m_lineNumber << ": Decision rule took " << decisionRuleTime << " seconds total" << endl);
VERBOSE(1, "Line " << m_source->GetTranslationId() << ": Decision rule took " << decisionRuleTime << " seconds total" << endl);
}
additionalReportingTime.start();
@ -242,9 +242,9 @@ void TranslationTask::Run() {
TrellisPathList nBestList;
ostringstream out;
manager.CalcNBest(staticData.GetNBestSize(), nBestList,staticData.GetDistinctNBest());
OutputNBest(out, nBestList, staticData.GetOutputFactorOrder(), m_lineNumber,
OutputNBest(out, nBestList, staticData.GetOutputFactorOrder(), m_source->GetTranslationId(),
staticData.GetReportSegmentation());
m_ioWrapper.GetNBestOutputCollector()->Write(m_lineNumber, out.str());
m_ioWrapper.GetNBestOutputCollector()->Write(m_source->GetTranslationId(), out.str());
}
//lattice samples
@ -252,9 +252,9 @@ void TranslationTask::Run() {
TrellisPathList latticeSamples;
ostringstream out;
manager.CalcLatticeSamples(staticData.GetLatticeSamplesSize(), latticeSamples);
OutputNBest(out,latticeSamples, staticData.GetOutputFactorOrder(), m_lineNumber,
OutputNBest(out,latticeSamples, staticData.GetOutputFactorOrder(), m_source->GetTranslationId(),
staticData.GetReportSegmentation());
m_ioWrapper.GetLatticeSamplesCollector()->Write(m_lineNumber, out.str());
m_ioWrapper.GetLatticeSamplesCollector()->Write(m_source->GetTranslationId(), out.str());
}
// detailed translation reporting
@ -262,7 +262,7 @@ void TranslationTask::Run() {
ostringstream out;
fix(out,PRECISION);
TranslationAnalysis::PrintTranslationAnalysis(out, manager.GetBestHypothesis());
m_ioWrapper.GetDetailedTranslationCollector()->Write(m_lineNumber,out.str());
m_ioWrapper.GetDetailedTranslationCollector()->Write(m_source->GetTranslationId(),out.str());
}
//list of unknown words
@ -273,13 +273,13 @@ void TranslationTask::Run() {
out << *(unknowns[i]);
}
out << endl;
m_ioWrapper.GetUnknownsCollector()->Write(m_lineNumber, out.str());
m_ioWrapper.GetUnknownsCollector()->Write(m_source->GetTranslationId(), out.str());
}
// report additional statistics
manager.CalcDecoderStatistics();
VERBOSE(1, "Line " << m_lineNumber << ": Additional reporting took " << additionalReportingTime << " seconds total" << endl);
VERBOSE(1, "Line " << m_lineNumber << ": Translation took " << translationTime << " seconds total" << endl);
VERBOSE(1, "Line " << m_source->GetTranslationId() << ": Additional reporting took " << additionalReportingTime << " seconds total" << endl);
VERBOSE(1, "Line " << m_source->GetTranslationId() << ": Translation took " << translationTime << " seconds total" << endl);
IFVERBOSE(2) {
PrintUserTime("Sentence Decoding Time:");
}

3
moses/TranslationTask.h
View File

@ -26,7 +26,7 @@ class TranslationTask : public Moses::Task
public:
TranslationTask(size_t lineNumber, Moses::InputType* source, MosesCmd::IOWrapper &ioWrapper,
TranslationTask(Moses::InputType* source, MosesCmd::IOWrapper &ioWrapper,
bool outputSearchGraphSLF,
boost::shared_ptr<Moses::HypergraphOutput<Moses::Manager> > hypergraphOutput);
@ -39,7 +39,6 @@ public:
private:
Moses::InputType* m_source;
size_t m_lineNumber;
MosesCmd::IOWrapper &m_ioWrapper;
bool m_outputSearchGraphSLF;