moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 19:27:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

Write alignment info through OutputCollector so it gets ordered

Browse Source 
correctly when run with multiple threads.


git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3882 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
bhaddow 2011-02-16 16:50:55 +00:00
parent df901e7ce6
commit 6b8415bffb
3 changed files with 30 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
moses-cmd/src/IOWrapper.cpp
View File

@ -223,8 +223,9 @@ void OutputSurface(std::ostream &out, const Phrase &phrase, const std::vector<Fa
}
}
void OutputAlignment(std::ofstream *alignmentStream, const vector<const Hypothesis *> &edges)
void OutputAlignment(OutputCollector* collector, size_t lineNo , const vector<const Hypothesis *> &edges)
{
ostringstream out;
size_t targetOffset = 0;
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--)
@ -235,16 +236,17 @@ void OutputAlignment(std::ofstream *alignmentStream, const vector<const Hypothes
AlignmentInfo::const_iterator it;
for (it = tp.GetAlignmentInfo().begin(); it != tp.GetAlignmentInfo().end(); ++it)
{
*alignmentStream << it->first + sourceOffset << "-" << it->second + targetOffset << " ";
out << it->first + sourceOffset << "-" << it->second + targetOffset << " ";
}
targetOffset += tp.GetSize();
}
*alignmentStream << std::endl;
out << std::endl;
collector->Write(lineNo,out.str());
}
void OutputAlignment(std::ofstream *alignmentStream, const Hypothesis *hypo)
void OutputAlignment(OutputCollector* collector, size_t lineNo , const Hypothesis *hypo)
{
if (hypo && ! StaticData::Instance().GetAlignmentOutputFile().empty() && alignmentStream)
if (collector)
{
std::vector<const Hypothesis *> edges;
const Hypothesis *currentHypo = hypo;
@ -254,15 +256,15 @@ void OutputAlignment(std::ofstream *alignmentStream, const Hypothesis *hypo)
currentHypo = currentHypo->GetPrevHypo();
}
OutputAlignment(alignmentStream, edges);
OutputAlignment(collector,lineNo, edges);
}
}
void OutputAlignment(std::ofstream *alignmentStream, const TrellisPath &path)
void OutputAlignment(OutputCollector* collector, size_t lineNo , const TrellisPath &path)
{
if (! StaticData::Instance().GetAlignmentOutputFile().empty() && alignmentStream)
if (collector)
{
OutputAlignment(alignmentStream, path.GetEdges());
OutputAlignment(collector,lineNo, path.GetEdges());
}
}

5
moses-cmd/src/IOWrapper.h
View File

@ -45,6 +45,7 @@ POSSIBILITY OF SUCH DAMAGE.
#include "FactorTypeSet.h"
#include "FactorCollection.h"
#include "Hypothesis.h"
#include "OutputCollector.h"
#include "TrellisPathList.h"
#include "InputFileStream.h"
#include "InputType.h"
@ -128,7 +129,7 @@ void OutputBestHypo(const std::vector<Moses::Word>& mbrBestHypo, long /*transla
bool reportSegmentation, bool reportAllFactors, std::ostream& out);
void OutputBestHypo(const Moses::TrellisPath &path, long /*translationId*/,bool reportSegmentation, bool reportAllFactors, std::ostream &out);
void OutputInput(std::ostream& os, const Hypothesis* hypo);
void OutputAlignment(std::ofstream *alignmentStream, const Hypothesis *hypo);
void OutputAlignment(std::ofstream *alignmentStream, const TrellisPath &path);
void OutputAlignment(OutputCollector* collector, size_t lineNo, const Hypothesis *hypo);
void OutputAlignment(OutputCollector* collector, size_t lineNo, const TrellisPath &path);
#endif

23
moses-cmd/src/Main.cpp
View File

@ -68,12 +68,13 @@ class TranslationTask : public Task {
TranslationTask(size_t lineNumber,
InputType* source, OutputCollector* outputCollector, OutputCollector* nbestCollector,
OutputCollector* wordGraphCollector, OutputCollector* searchGraphCollector,
OutputCollector* detailedTranslationCollector, std::ofstream *alignmentStream ) :
OutputCollector* detailedTranslationCollector,
OutputCollector* alignmentInfoCollector ) :
m_source(source), m_lineNumber(lineNumber),
m_outputCollector(outputCollector), m_nbestCollector(nbestCollector),
m_wordGraphCollector(wordGraphCollector), m_searchGraphCollector(searchGraphCollector),
m_detailedTranslationCollector(detailedTranslationCollector),
m_alignmentStream(alignmentStream) {}
m_alignmentInfoCollector(alignmentInfoCollector) {}
void Run()
{
@ -140,7 +141,7 @@ class TranslationTask : public Task {
staticData.GetOutputFactorOrder(),
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors());
OutputAlignment(m_alignmentStream, bestHypo);
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, bestHypo);
IFVERBOSE(1) {
debug << "BEST TRANSLATION: " << *bestHypo << endl;
}
@ -186,7 +187,7 @@ class TranslationTask : public Task {
OutputBestHypo(conBestHypo, m_lineNumber,
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),out);
OutputAlignment(m_alignmentStream, conBestHypo);
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, conBestHypo);
IFVERBOSE(2) { PrintUserTime("finished Consensus decoding"); }
}
else
@ -196,7 +197,7 @@ class TranslationTask : public Task {
OutputBestHypo(mbrBestHypo, m_lineNumber,
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),out);
OutputAlignment(m_alignmentStream, mbrBestHypo);
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, mbrBestHypo);
IFVERBOSE(2) { PrintUserTime("finished MBR decoding"); }
}
@ -234,6 +235,7 @@ class TranslationTask : public Task {
OutputCollector* m_wordGraphCollector;
OutputCollector* m_searchGraphCollector;
OutputCollector* m_detailedTranslationCollector;
OutputCollector* m_alignmentInfoCollector;
std::ofstream *m_alignmentStream;
@ -381,14 +383,21 @@ int main(int argc, char** argv) {
if (staticData.IsDetailedTranslationReportingEnabled()) {
detailedTranslationCollector.reset(new OutputCollector(&(ioWrapper->GetDetailedTranslationReportingStream())));
}
auto_ptr<OutputCollector> alignmentInfoCollector;
if (!staticData.GetAlignmentOutputFile().empty()) {
alignmentInfoCollector.reset(new OutputCollector(ioWrapper->GetAlignmentOutputStream()));
}
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
IFVERBOSE(1) {
ResetUserTime();
}
TranslationTask* task =
new TranslationTask(lineCount,source, outputCollector.get(), nbestCollector.get(), wordGraphCollector.get(),
searchGraphCollector.get(), detailedTranslationCollector.get(), ioWrapper->GetAlignmentOutputStream());
new TranslationTask(lineCount,source, outputCollector.get(),
nbestCollector.get(), wordGraphCollector.get(),
searchGraphCollector.get(),
detailedTranslationCollector.get(),
alignmentInfoCollector.get() );
#ifdef WITH_THREADS
pool.Submit(task);