2006-07-04 22:04:38 +04:00
|
|
|
// $Id$
|
|
|
|
|
|
|
|
/***********************************************************************
|
|
|
|
Moses - factored phrase-based language decoder
|
|
|
|
Copyright (c) 2006 University of Edinburgh
|
|
|
|
All rights reserved.
|
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
Redistribution and use in source and binary forms, with or without modification,
|
2006-07-04 22:04:38 +04:00
|
|
|
are permitted provided that the following conditions are met:
|
|
|
|
|
2012-12-04 21:54:39 +04:00
|
|
|
* Redistributions of source code must retain the above copyright notice,
|
2006-07-04 22:04:38 +04:00
|
|
|
this list of conditions and the following disclaimer.
|
2012-12-04 21:54:39 +04:00
|
|
|
* Redistributions in binary form must reproduce the above copyright notice,
|
2011-02-24 15:39:29 +03:00
|
|
|
this list of conditions and the following disclaimer in the documentation
|
2006-07-04 22:04:38 +04:00
|
|
|
and/or other materials provided with the distribution.
|
2012-12-04 21:54:39 +04:00
|
|
|
* Neither the name of the University of Edinburgh nor the names of its contributors
|
2011-02-24 15:39:29 +03:00
|
|
|
may be used to endorse or promote products derived from this software
|
2006-07-04 22:04:38 +04:00
|
|
|
without specific prior written permission.
|
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
|
|
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
|
|
|
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
|
|
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
|
|
|
|
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
|
|
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
|
|
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
|
|
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
|
|
|
|
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
|
|
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
2006-07-04 22:04:38 +04:00
|
|
|
POSSIBILITY OF SUCH DAMAGE.
|
2012-12-04 21:54:39 +04:00
|
|
|
***********************************************************************/
|
2006-07-04 22:04:38 +04:00
|
|
|
|
|
|
|
// example file on how to use moses library
|
|
|
|
|
|
|
|
#include <iostream>
|
2011-08-18 01:13:21 +04:00
|
|
|
#include <stack>
|
2011-11-24 23:27:12 +04:00
|
|
|
#include <boost/algorithm/string.hpp>
|
2006-07-04 22:04:38 +04:00
|
|
|
|
2012-11-13 00:21:32 +04:00
|
|
|
#include "moses/TypeDef.h"
|
|
|
|
#include "moses/Util.h"
|
|
|
|
#include "moses/Hypothesis.h"
|
|
|
|
#include "moses/WordsRange.h"
|
|
|
|
#include "moses/TrellisPathList.h"
|
|
|
|
#include "moses/StaticData.h"
|
|
|
|
#include "moses/FeatureVector.h"
|
|
|
|
#include "moses/InputFileStream.h"
|
2013-09-25 19:57:01 +04:00
|
|
|
#include "moses/FF/StatefulFeatureFunction.h"
|
|
|
|
#include "moses/FF/StatelessFeatureFunction.h"
|
2013-11-18 22:27:01 +04:00
|
|
|
#include "util/exception.hh"
|
2013-09-25 19:57:01 +04:00
|
|
|
|
2012-11-13 00:21:32 +04:00
|
|
|
#include "IOWrapper.h"
|
|
|
|
|
2006-07-04 22:04:38 +04:00
|
|
|
using namespace std;
|
2008-10-09 03:51:26 +04:00
|
|
|
using namespace Moses;
|
2006-07-04 22:04:38 +04:00
|
|
|
|
2012-07-02 20:05:11 +04:00
|
|
|
namespace MosesCmd
|
|
|
|
{
|
|
|
|
|
2008-08-05 04:24:45 +04:00
|
|
|
IOWrapper::IOWrapper(
|
2013-05-29 21:16:15 +04:00
|
|
|
const vector<FactorType> &inputFactorOrder
|
|
|
|
, const vector<FactorType> &outputFactorOrder
|
|
|
|
, const FactorMask &inputFactorUsed
|
|
|
|
, size_t nBestSize
|
|
|
|
, const string &nBestFilePath)
|
|
|
|
:m_inputFactorOrder(inputFactorOrder)
|
|
|
|
,m_outputFactorOrder(outputFactorOrder)
|
|
|
|
,m_inputFactorUsed(inputFactorUsed)
|
|
|
|
,m_inputFile(NULL)
|
|
|
|
,m_inputStream(&std::cin)
|
|
|
|
,m_nBestStream(NULL)
|
|
|
|
,m_outputWordGraphStream(NULL)
|
|
|
|
,m_outputSearchGraphStream(NULL)
|
|
|
|
,m_detailedTranslationReportingStream(NULL)
|
|
|
|
,m_alignmentOutputStream(NULL)
|
2006-07-04 22:04:38 +04:00
|
|
|
{
|
2011-02-24 15:39:29 +03:00
|
|
|
Initialization(inputFactorOrder, outputFactorOrder
|
2013-05-29 21:16:15 +04:00
|
|
|
, inputFactorUsed
|
|
|
|
, nBestSize, nBestFilePath);
|
2006-07-04 22:04:38 +04:00
|
|
|
}
|
|
|
|
|
2008-08-05 04:24:45 +04:00
|
|
|
IOWrapper::IOWrapper(const std::vector<FactorType> &inputFactorOrder
|
2013-05-29 21:16:15 +04:00
|
|
|
, const std::vector<FactorType> &outputFactorOrder
|
|
|
|
, const FactorMask &inputFactorUsed
|
|
|
|
, size_t nBestSize
|
|
|
|
, const std::string &nBestFilePath
|
|
|
|
, const std::string &inputFilePath)
|
|
|
|
:m_inputFactorOrder(inputFactorOrder)
|
|
|
|
,m_outputFactorOrder(outputFactorOrder)
|
|
|
|
,m_inputFactorUsed(inputFactorUsed)
|
|
|
|
,m_inputFilePath(inputFilePath)
|
|
|
|
,m_inputFile(new InputFileStream(inputFilePath))
|
|
|
|
,m_nBestStream(NULL)
|
|
|
|
,m_outputWordGraphStream(NULL)
|
|
|
|
,m_outputSearchGraphStream(NULL)
|
|
|
|
,m_detailedTranslationReportingStream(NULL)
|
|
|
|
,m_alignmentOutputStream(NULL)
|
2006-11-22 02:06:30 +03:00
|
|
|
{
|
2011-02-24 15:39:29 +03:00
|
|
|
Initialization(inputFactorOrder, outputFactorOrder
|
2013-05-29 21:16:15 +04:00
|
|
|
, inputFactorUsed
|
|
|
|
, nBestSize, nBestFilePath);
|
2008-03-01 02:08:58 +03:00
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
m_inputStream = m_inputFile;
|
2008-03-01 02:08:58 +03:00
|
|
|
}
|
|
|
|
|
2008-08-05 04:24:45 +04:00
|
|
|
IOWrapper::~IOWrapper()
|
2008-03-01 02:08:58 +03:00
|
|
|
{
|
2011-02-24 15:39:29 +03:00
|
|
|
if (m_inputFile != NULL)
|
|
|
|
delete m_inputFile;
|
|
|
|
if (m_nBestStream != NULL && !m_surpressSingleBestOutput) {
|
|
|
|
// outputting n-best to file, rather than stdout. need to close file and delete obj
|
|
|
|
delete m_nBestStream;
|
|
|
|
}
|
|
|
|
if (m_outputWordGraphStream != NULL) {
|
|
|
|
delete m_outputWordGraphStream;
|
|
|
|
}
|
|
|
|
if (m_outputSearchGraphStream != NULL) {
|
|
|
|
delete m_outputSearchGraphStream;
|
|
|
|
}
|
2010-05-08 19:51:59 +04:00
|
|
|
delete m_detailedTranslationReportingStream;
|
2011-02-24 15:39:29 +03:00
|
|
|
delete m_alignmentOutputStream;
|
2008-03-01 02:08:58 +03:00
|
|
|
}
|
2006-11-22 02:06:30 +03:00
|
|
|
|
2010-05-11 01:18:47 +04:00
|
|
|
void IOWrapper::Initialization(const std::vector<FactorType> &/*inputFactorOrder*/
|
2013-05-29 21:16:15 +04:00
|
|
|
, const std::vector<FactorType> &/*outputFactorOrder*/
|
|
|
|
, const FactorMask &/*inputFactorUsed*/
|
|
|
|
, size_t nBestSize
|
|
|
|
, const std::string &nBestFilePath)
|
2008-03-01 02:08:58 +03:00
|
|
|
{
|
2011-02-24 15:39:29 +03:00
|
|
|
const StaticData &staticData = StaticData::Instance();
|
|
|
|
|
|
|
|
// n-best
|
|
|
|
m_surpressSingleBestOutput = false;
|
|
|
|
|
|
|
|
if (nBestSize > 0) {
|
|
|
|
if (nBestFilePath == "-" || nBestFilePath == "/dev/stdout") {
|
|
|
|
m_nBestStream = &std::cout;
|
|
|
|
m_surpressSingleBestOutput = true;
|
|
|
|
} else {
|
|
|
|
std::ofstream *file = new std::ofstream;
|
|
|
|
m_nBestStream = file;
|
|
|
|
file->open(nBestFilePath.c_str());
|
|
|
|
}
|
|
|
|
}
|
2006-11-22 02:06:30 +03:00
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
// wordgraph output
|
|
|
|
if (staticData.GetOutputWordGraph()) {
|
|
|
|
string fileName = staticData.GetParam("output-word-graph")[0];
|
|
|
|
std::ofstream *file = new std::ofstream;
|
|
|
|
m_outputWordGraphStream = file;
|
|
|
|
file->open(fileName.c_str());
|
|
|
|
}
|
2008-03-18 00:34:19 +03:00
|
|
|
|
2011-08-18 01:13:21 +04:00
|
|
|
|
2012-12-04 21:54:39 +04:00
|
|
|
// search graph output
|
2011-02-24 15:39:29 +03:00
|
|
|
if (staticData.GetOutputSearchGraph()) {
|
|
|
|
string fileName;
|
|
|
|
if (staticData.GetOutputSearchGraphExtended())
|
|
|
|
fileName = staticData.GetParam("output-search-graph-extended")[0];
|
|
|
|
else
|
|
|
|
fileName = staticData.GetParam("output-search-graph")[0];
|
|
|
|
std::ofstream *file = new std::ofstream;
|
|
|
|
m_outputSearchGraphStream = file;
|
|
|
|
file->open(fileName.c_str());
|
|
|
|
}
|
2010-05-08 19:51:59 +04:00
|
|
|
|
|
|
|
// detailed translation reporting
|
2011-02-24 15:39:29 +03:00
|
|
|
if (staticData.IsDetailedTranslationReportingEnabled()) {
|
2010-05-08 19:51:59 +04:00
|
|
|
const std::string &path = staticData.GetDetailedTranslationReportingFilePath();
|
|
|
|
m_detailedTranslationReportingStream = new std::ofstream(path.c_str());
|
2013-11-18 22:27:01 +04:00
|
|
|
UTIL_THROW_IF(!m_detailedTranslationReportingStream->good(),
|
|
|
|
util::FileOpenException,
|
|
|
|
"File for output of detailed translation report could not be open");
|
2010-05-08 19:51:59 +04:00
|
|
|
}
|
2011-02-24 15:39:29 +03:00
|
|
|
|
2011-08-18 01:13:21 +04:00
|
|
|
// sentence alignment output
|
2011-02-24 15:39:29 +03:00
|
|
|
if (! staticData.GetAlignmentOutputFile().empty()) {
|
|
|
|
m_alignmentOutputStream = new ofstream(staticData.GetAlignmentOutputFile().c_str());
|
2013-11-18 22:27:01 +04:00
|
|
|
UTIL_THROW_IF(!m_alignmentOutputStream->good(),
|
|
|
|
util::FileOpenException,
|
|
|
|
"File for output of word alignment could not be open");
|
2011-08-18 01:13:21 +04:00
|
|
|
}
|
|
|
|
|
2006-11-22 02:06:30 +03:00
|
|
|
}
|
|
|
|
|
2014-03-11 17:53:54 +04:00
|
|
|
InputType*
|
|
|
|
IOWrapper::
|
|
|
|
GetInput(InputType* inputType)
|
2006-07-04 22:04:38 +04:00
|
|
|
{
|
2011-02-24 15:39:29 +03:00
|
|
|
if(inputType->Read(*m_inputStream, m_inputFactorOrder)) {
|
|
|
|
if (long x = inputType->GetTranslationId()) {
|
|
|
|
if (x>=m_translationId) m_translationId = x+1;
|
|
|
|
} else inputType->SetTranslationId(m_translationId++);
|
|
|
|
|
|
|
|
return inputType;
|
|
|
|
} else {
|
|
|
|
delete inputType;
|
|
|
|
return NULL;
|
|
|
|
}
|
2006-07-04 22:04:38 +04:00
|
|
|
}
|
|
|
|
|
2013-09-19 18:33:21 +04:00
|
|
|
std::map<size_t, const Factor*> GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor)
|
|
|
|
{
|
|
|
|
const InputPath &inputPath = hypo.GetTranslationOption().GetInputPath();
|
|
|
|
const Phrase &inputPhrase = inputPath.GetPhrase();
|
|
|
|
|
|
|
|
std::map<size_t, const Factor*> ret;
|
|
|
|
|
|
|
|
for (size_t sourcePos = 0; sourcePos < inputPhrase.GetSize(); ++sourcePos) {
|
2013-09-27 12:35:24 +04:00
|
|
|
const Factor *factor = inputPhrase.GetFactor(sourcePos, placeholderFactor);
|
|
|
|
if (factor) {
|
|
|
|
std::set<size_t> targetPos = hypo.GetTranslationOption().GetTargetPhrase().GetAlignTerm().GetAlignmentsForSource(sourcePos);
|
2013-11-23 00:27:46 +04:00
|
|
|
UTIL_THROW_IF2(targetPos.size() != 1,
|
2013-11-18 22:27:01 +04:00
|
|
|
"Placeholder should be aligned to 1, and only 1, word");
|
2013-09-27 12:35:24 +04:00
|
|
|
ret[*targetPos.begin()] = factor;
|
|
|
|
}
|
2013-09-19 18:33:21 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2006-08-12 01:04:38 +04:00
|
|
|
/***
|
|
|
|
* print surface factor only for the given phrase
|
|
|
|
*/
|
2012-04-19 01:09:02 +04:00
|
|
|
void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector<FactorType> &outputFactorOrder,
|
2013-08-07 08:31:45 +04:00
|
|
|
char reportSegmentation, bool reportAllFactors)
|
2006-07-04 22:04:38 +04:00
|
|
|
{
|
2013-11-23 00:27:46 +04:00
|
|
|
UTIL_THROW_IF2(outputFactorOrder.size() == 0,
|
2013-11-18 22:27:01 +04:00
|
|
|
"Must specific at least 1 output factor");
|
2013-09-19 18:33:21 +04:00
|
|
|
const TargetPhrase& phrase = edge.GetCurrTargetPhrase();
|
2013-08-15 18:56:20 +04:00
|
|
|
bool markUnknown = StaticData::Instance().GetMarkUnknown();
|
2011-02-24 15:39:29 +03:00
|
|
|
if (reportAllFactors == true) {
|
|
|
|
out << phrase;
|
|
|
|
} else {
|
2013-09-22 17:09:49 +04:00
|
|
|
FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor();
|
2013-09-19 18:33:21 +04:00
|
|
|
|
|
|
|
std::map<size_t, const Factor*> placeholders;
|
|
|
|
if (placeholderFactor != NOT_FOUND) {
|
2013-09-27 12:35:24 +04:00
|
|
|
// creates map of target position -> factor for placeholders
|
|
|
|
placeholders = GetPlaceholders(edge, placeholderFactor);
|
2013-09-19 18:33:21 +04:00
|
|
|
}
|
2013-07-19 20:58:52 +04:00
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
size_t size = phrase.GetSize();
|
|
|
|
for (size_t pos = 0 ; pos < size ; pos++) {
|
2013-08-01 17:26:54 +04:00
|
|
|
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
|
2013-07-19 20:58:52 +04:00
|
|
|
|
2013-09-19 18:33:21 +04:00
|
|
|
if (placeholders.size()) {
|
2013-09-27 12:35:24 +04:00
|
|
|
// do placeholders
|
|
|
|
std::map<size_t, const Factor*>::const_iterator iter = placeholders.find(pos);
|
|
|
|
if (iter != placeholders.end()) {
|
|
|
|
factor = iter->second;
|
|
|
|
}
|
2013-07-19 20:58:52 +04:00
|
|
|
}
|
2013-09-19 18:33:21 +04:00
|
|
|
|
2013-11-23 00:27:46 +04:00
|
|
|
UTIL_THROW_IF2(factor == NULL,
|
2013-11-18 22:27:01 +04:00
|
|
|
"No factor 0 at position " << pos);
|
2011-02-24 15:39:29 +03:00
|
|
|
|
2013-08-15 18:56:20 +04:00
|
|
|
//preface surface form with UNK if marking unknowns
|
|
|
|
const Word &word = phrase.GetWord(pos);
|
|
|
|
if(markUnknown && word.IsOOV()) {
|
2013-08-23 16:53:30 +04:00
|
|
|
out << "UNK" << *factor;
|
|
|
|
} else {
|
|
|
|
out << *factor;
|
2013-08-15 18:56:20 +04:00
|
|
|
}
|
2013-08-23 16:53:30 +04:00
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
for (size_t i = 1 ; i < outputFactorOrder.size() ; i++) {
|
|
|
|
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]);
|
2013-11-23 00:27:46 +04:00
|
|
|
UTIL_THROW_IF2(factor == NULL,
|
2013-11-18 22:27:01 +04:00
|
|
|
"No factor " << i << " at position " << pos);
|
2012-05-17 19:25:53 +04:00
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
out << "|" << *factor;
|
|
|
|
}
|
|
|
|
out << " ";
|
|
|
|
}
|
|
|
|
}
|
2012-04-19 01:09:02 +04:00
|
|
|
|
2013-10-13 09:59:05 +04:00
|
|
|
// trace ("report segmentation") option "-t" / "-tt"
|
2013-08-07 08:31:45 +04:00
|
|
|
if (reportSegmentation > 0 && phrase.GetSize() > 0) {
|
|
|
|
const WordsRange &sourceRange = edge.GetCurrSourceWordsRange();
|
|
|
|
const int sourceStart = sourceRange.GetStartPos();
|
|
|
|
const int sourceEnd = sourceRange.GetEndPos();
|
2013-10-13 09:59:05 +04:00
|
|
|
out << "|" << sourceStart << "-" << sourceEnd; // enriched "-tt"
|
2013-08-07 08:31:45 +04:00
|
|
|
if (reportSegmentation == 2) {
|
2013-10-13 09:59:05 +04:00
|
|
|
out << ",wa=";
|
2013-08-07 08:31:45 +04:00
|
|
|
const AlignmentInfo &ai = edge.GetCurrTargetPhrase().GetAlignTerm();
|
|
|
|
OutputAlignment(out, ai, 0, 0);
|
2013-10-13 09:59:05 +04:00
|
|
|
out << ",total=";
|
2013-10-13 12:04:03 +04:00
|
|
|
out << edge.GetScore() - edge.GetPrevHypo()->GetScore();
|
2013-10-13 09:59:05 +04:00
|
|
|
out << ",";
|
|
|
|
ScoreComponentCollection scoreBreakdown(edge.GetScoreBreakdown());
|
|
|
|
scoreBreakdown.MinusEquals(edge.GetPrevHypo()->GetScoreBreakdown());
|
2013-10-14 01:05:07 +04:00
|
|
|
OutputAllFeatureScores(scoreBreakdown, out);
|
2013-08-07 08:31:45 +04:00
|
|
|
}
|
|
|
|
out << "| ";
|
2012-04-19 01:09:02 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void OutputBestSurface(std::ostream &out, const Hypothesis *hypo, const std::vector<FactorType> &outputFactorOrder,
|
2013-08-07 08:31:45 +04:00
|
|
|
char reportSegmentation, bool reportAllFactors)
|
2012-04-19 01:09:02 +04:00
|
|
|
{
|
|
|
|
if (hypo != NULL) {
|
|
|
|
// recursively retrace this best path through the lattice, starting from the end of the hypothesis sentence
|
|
|
|
OutputBestSurface(out, hypo->GetPrevHypo(), outputFactorOrder, reportSegmentation, reportAllFactors);
|
|
|
|
OutputSurface(out, *hypo, outputFactorOrder, reportSegmentation, reportAllFactors);
|
|
|
|
}
|
2006-07-04 22:04:38 +04:00
|
|
|
}
|
|
|
|
|
2011-08-26 06:37:52 +04:00
|
|
|
void OutputAlignment(ostream &out, const AlignmentInfo &ai, size_t sourceOffset, size_t targetOffset)
|
|
|
|
{
|
|
|
|
typedef std::vector< const std::pair<size_t,size_t>* > AlignVec;
|
|
|
|
AlignVec alignments = ai.GetSortedAlignments();
|
2012-12-04 21:54:39 +04:00
|
|
|
|
2011-08-26 06:37:52 +04:00
|
|
|
AlignVec::const_iterator it;
|
|
|
|
for (it = alignments.begin(); it != alignments.end(); ++it) {
|
|
|
|
const std::pair<size_t,size_t> &alignment = **it;
|
|
|
|
out << alignment.first + sourceOffset << "-" << alignment.second + targetOffset << " ";
|
|
|
|
}
|
2012-12-04 21:54:39 +04:00
|
|
|
|
2011-08-26 06:37:52 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
void OutputAlignment(ostream &out, const vector<const Hypothesis *> &edges)
|
2011-08-18 01:13:21 +04:00
|
|
|
{
|
2011-02-24 15:39:29 +03:00
|
|
|
size_t targetOffset = 0;
|
|
|
|
|
|
|
|
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
|
|
|
|
const Hypothesis &edge = *edges[currEdge];
|
|
|
|
const TargetPhrase &tp = edge.GetCurrTargetPhrase();
|
|
|
|
size_t sourceOffset = edge.GetCurrSourceWordsRange().GetStartPos();
|
2012-12-04 21:54:39 +04:00
|
|
|
|
2012-10-19 18:10:10 +04:00
|
|
|
OutputAlignment(out, tp.GetAlignTerm(), sourceOffset, targetOffset);
|
2011-08-26 06:37:52 +04:00
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
targetOffset += tp.GetSize();
|
|
|
|
}
|
|
|
|
out << std::endl;
|
2011-08-26 06:37:52 +04:00
|
|
|
}
|
|
|
|
|
2013-03-13 16:12:33 +04:00
|
|
|
void OutputAlignment(std::ostream &out, const Moses::Hypothesis *hypo)
|
|
|
|
{
|
|
|
|
std::vector<const Hypothesis *> edges;
|
|
|
|
const Hypothesis *currentHypo = hypo;
|
|
|
|
while (currentHypo) {
|
|
|
|
edges.push_back(currentHypo);
|
|
|
|
currentHypo = currentHypo->GetPrevHypo();
|
|
|
|
}
|
|
|
|
|
|
|
|
OutputAlignment(out, edges);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2011-08-26 06:37:52 +04:00
|
|
|
void OutputAlignment(OutputCollector* collector, size_t lineNo , const vector<const Hypothesis *> &edges)
|
|
|
|
{
|
|
|
|
ostringstream out;
|
|
|
|
OutputAlignment(out, edges);
|
2012-12-04 21:54:39 +04:00
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
collector->Write(lineNo,out.str());
|
2011-08-18 01:13:21 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
void OutputAlignment(OutputCollector* collector, size_t lineNo , const Hypothesis *hypo)
|
|
|
|
{
|
2011-02-24 15:39:29 +03:00
|
|
|
if (collector) {
|
|
|
|
std::vector<const Hypothesis *> edges;
|
|
|
|
const Hypothesis *currentHypo = hypo;
|
|
|
|
while (currentHypo) {
|
|
|
|
edges.push_back(currentHypo);
|
|
|
|
currentHypo = currentHypo->GetPrevHypo();
|
2011-08-18 01:13:21 +04:00
|
|
|
}
|
2011-02-24 15:39:29 +03:00
|
|
|
|
|
|
|
OutputAlignment(collector,lineNo, edges);
|
|
|
|
}
|
2011-08-18 01:13:21 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
void OutputAlignment(OutputCollector* collector, size_t lineNo , const TrellisPath &path)
|
|
|
|
{
|
2011-02-24 15:39:29 +03:00
|
|
|
if (collector) {
|
|
|
|
OutputAlignment(collector,lineNo, path.GetEdges());
|
|
|
|
}
|
2011-08-18 01:13:21 +04:00
|
|
|
}
|
|
|
|
|
2013-08-07 08:31:45 +04:00
|
|
|
void OutputBestHypo(const Moses::TrellisPath &path, long /*translationId*/, char reportSegmentation, bool reportAllFactors, std::ostream &out)
|
2011-02-24 15:39:29 +03:00
|
|
|
{
|
|
|
|
const std::vector<const Hypothesis *> &edges = path.GetEdges();
|
|
|
|
|
|
|
|
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
|
|
|
|
const Hypothesis &edge = *edges[currEdge];
|
2012-04-19 01:09:02 +04:00
|
|
|
OutputSurface(out, edge, StaticData::Instance().GetOutputFactorOrder(), reportSegmentation, reportAllFactors);
|
2011-02-24 15:39:29 +03:00
|
|
|
}
|
2010-05-11 01:18:47 +04:00
|
|
|
out << endl;
|
|
|
|
}
|
2008-09-12 22:09:06 +04:00
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
void IOWrapper::Backtrack(const Hypothesis *hypo)
|
|
|
|
{
|
2006-07-15 01:51:05 +04:00
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
if (hypo->GetPrevHypo() != NULL) {
|
|
|
|
VERBOSE(3,hypo->GetId() << " <= ");
|
|
|
|
Backtrack(hypo->GetPrevHypo());
|
|
|
|
}
|
2006-07-15 01:51:05 +04:00
|
|
|
}
|
2011-02-24 15:39:29 +03:00
|
|
|
|
2013-08-07 08:31:45 +04:00
|
|
|
void OutputBestHypo(const std::vector<Word>& mbrBestHypo, long /*translationId*/, char /*reportSegmentation*/, bool /*reportAllFactors*/, ostream& out)
|
2010-02-03 20:04:05 +03:00
|
|
|
{
|
2011-02-24 15:39:29 +03:00
|
|
|
|
|
|
|
for (size_t i = 0 ; i < mbrBestHypo.size() ; i++) {
|
|
|
|
const Factor *factor = mbrBestHypo[i].GetFactor(StaticData::Instance().GetOutputFactorOrder()[0]);
|
2013-11-23 00:27:46 +04:00
|
|
|
UTIL_THROW_IF2(factor == NULL,
|
2013-11-18 22:27:01 +04:00
|
|
|
"No factor 0 at position " << i);
|
2011-10-03 20:11:39 +04:00
|
|
|
if (i>0) out << " " << *factor;
|
|
|
|
else out << *factor;
|
2011-02-24 15:39:29 +03:00
|
|
|
}
|
|
|
|
out << endl;
|
|
|
|
}
|
2010-02-12 18:56:37 +03:00
|
|
|
|
2010-02-03 20:04:05 +03:00
|
|
|
|
2007-09-28 20:43:33 +04:00
|
|
|
void OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo)
|
|
|
|
{
|
2011-02-24 15:39:29 +03:00
|
|
|
if (hypo->GetPrevHypo()) {
|
2013-08-07 17:18:12 +04:00
|
|
|
OutputInput(map, hypo->GetPrevHypo());
|
2013-08-13 23:36:32 +04:00
|
|
|
map[hypo->GetCurrSourceWordsRange().GetStartPos()] = &hypo->GetTranslationOption().GetInputPath().GetPhrase();
|
2011-02-24 15:39:29 +03:00
|
|
|
}
|
2007-09-28 20:43:33 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
void OutputInput(std::ostream& os, const Hypothesis* hypo)
|
|
|
|
{
|
2011-02-24 15:39:29 +03:00
|
|
|
size_t len = hypo->GetInput().GetSize();
|
|
|
|
std::vector<const Phrase*> inp_phrases(len, 0);
|
|
|
|
OutputInput(inp_phrases, hypo);
|
|
|
|
for (size_t i=0; i<len; ++i)
|
|
|
|
if (inp_phrases[i]) os << *inp_phrases[i];
|
2007-09-28 20:43:33 +04:00
|
|
|
}
|
|
|
|
|
2013-08-07 08:31:45 +04:00
|
|
|
void IOWrapper::OutputBestHypo(const Hypothesis *hypo, long /*translationId*/, char reportSegmentation, bool reportAllFactors)
|
2006-07-04 22:04:38 +04:00
|
|
|
{
|
2011-02-24 15:39:29 +03:00
|
|
|
if (hypo != NULL) {
|
|
|
|
VERBOSE(1,"BEST TRANSLATION: " << *hypo << endl);
|
|
|
|
VERBOSE(3,"Best path: ");
|
|
|
|
Backtrack(hypo);
|
|
|
|
VERBOSE(3,"0" << std::endl);
|
|
|
|
if (!m_surpressSingleBestOutput) {
|
2013-08-16 00:14:04 +04:00
|
|
|
if (StaticData::Instance().GetOutputHypoScore()) {
|
|
|
|
cout << hypo->GetTotalScore() << " ";
|
|
|
|
}
|
2013-08-15 23:20:44 +04:00
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
if (StaticData::Instance().IsPathRecoveryEnabled()) {
|
|
|
|
OutputInput(cout, hypo);
|
|
|
|
cout << "||| ";
|
|
|
|
}
|
2012-04-19 01:09:02 +04:00
|
|
|
OutputBestSurface(cout, hypo, m_outputFactorOrder, reportSegmentation, reportAllFactors);
|
2011-02-24 15:39:29 +03:00
|
|
|
cout << endl;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
VERBOSE(1, "NO BEST TRANSLATION" << endl);
|
|
|
|
if (!m_surpressSingleBestOutput) {
|
|
|
|
cout << endl;
|
|
|
|
}
|
|
|
|
}
|
2006-07-04 22:04:38 +04:00
|
|
|
}
|
|
|
|
|
2012-12-07 17:34:44 +04:00
|
|
|
void OutputNBest(std::ostream& out
|
2013-05-29 21:16:15 +04:00
|
|
|
, const Moses::TrellisPathList &nBestList
|
|
|
|
, const std::vector<Moses::FactorType>& outputFactorOrder
|
|
|
|
, long translationId
|
2013-08-07 08:31:45 +04:00
|
|
|
, char reportSegmentation)
|
2006-07-04 22:04:38 +04:00
|
|
|
{
|
2011-02-24 15:39:29 +03:00
|
|
|
const StaticData &staticData = StaticData::Instance();
|
|
|
|
bool reportAllFactors = staticData.GetReportAllFactorsNBest();
|
2012-11-14 20:57:33 +04:00
|
|
|
bool includeSegmentation = staticData.NBestIncludesSegmentation();
|
2011-02-24 15:39:29 +03:00
|
|
|
bool includeWordAlignment = staticData.PrintAlignmentInfoInNbest();
|
|
|
|
|
|
|
|
TrellisPathList::const_iterator iter;
|
|
|
|
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) {
|
|
|
|
const TrellisPath &path = **iter;
|
|
|
|
const std::vector<const Hypothesis *> &edges = path.GetEdges();
|
|
|
|
|
|
|
|
// print the surface factor of the translation
|
|
|
|
out << translationId << " ||| ";
|
|
|
|
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
|
|
|
|
const Hypothesis &edge = *edges[currEdge];
|
2012-04-19 01:09:02 +04:00
|
|
|
OutputSurface(out, edge, outputFactorOrder, reportSegmentation, reportAllFactors);
|
2011-02-24 15:39:29 +03:00
|
|
|
}
|
|
|
|
out << " |||";
|
2009-05-26 23:30:35 +04:00
|
|
|
|
2011-08-07 04:58:56 +04:00
|
|
|
// print scores with feature names
|
2013-05-11 17:13:26 +04:00
|
|
|
OutputAllFeatureScores(path.GetScoreBreakdown(), out );
|
2011-02-24 15:39:29 +03:00
|
|
|
|
|
|
|
// total
|
|
|
|
out << " ||| " << path.GetTotalScore();
|
|
|
|
|
2012-11-14 20:57:33 +04:00
|
|
|
//phrase-to-phrase segmentation
|
|
|
|
if (includeSegmentation) {
|
2011-02-24 15:39:29 +03:00
|
|
|
out << " |||";
|
|
|
|
for (int currEdge = (int)edges.size() - 2 ; currEdge >= 0 ; currEdge--) {
|
|
|
|
const Hypothesis &edge = *edges[currEdge];
|
|
|
|
const WordsRange &sourceRange = edge.GetCurrSourceWordsRange();
|
|
|
|
WordsRange targetRange = path.GetTargetWordsRange(edge);
|
|
|
|
out << " " << sourceRange.GetStartPos();
|
|
|
|
if (sourceRange.GetStartPos() < sourceRange.GetEndPos()) {
|
|
|
|
out << "-" << sourceRange.GetEndPos();
|
|
|
|
}
|
|
|
|
out<< "=" << targetRange.GetStartPos();
|
|
|
|
if (targetRange.GetStartPos() < targetRange.GetEndPos()) {
|
|
|
|
out<< "-" << targetRange.GetEndPos();
|
2010-03-16 14:34:50 +03:00
|
|
|
}
|
2011-02-24 15:39:29 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (includeWordAlignment) {
|
2011-09-02 15:40:32 +04:00
|
|
|
out << " ||| ";
|
2011-02-24 15:39:29 +03:00
|
|
|
for (int currEdge = (int)edges.size() - 2 ; currEdge >= 0 ; currEdge--) {
|
|
|
|
const Hypothesis &edge = *edges[currEdge];
|
|
|
|
const WordsRange &sourceRange = edge.GetCurrSourceWordsRange();
|
|
|
|
WordsRange targetRange = path.GetTargetWordsRange(edge);
|
|
|
|
const int sourceOffset = sourceRange.GetStartPos();
|
|
|
|
const int targetOffset = targetRange.GetStartPos();
|
2012-10-19 18:10:10 +04:00
|
|
|
const AlignmentInfo &ai = edge.GetCurrTargetPhrase().GetAlignTerm();
|
2012-12-04 21:54:39 +04:00
|
|
|
|
2011-08-26 06:37:52 +04:00
|
|
|
OutputAlignment(out, ai, sourceOffset, targetOffset);
|
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (StaticData::Instance().IsPathRecoveryEnabled()) {
|
2013-08-07 14:37:40 +04:00
|
|
|
out << " ||| ";
|
2011-02-24 15:39:29 +03:00
|
|
|
OutputInput(out, edges[0]);
|
|
|
|
}
|
|
|
|
|
|
|
|
out << endl;
|
|
|
|
}
|
|
|
|
|
2011-08-19 17:55:32 +04:00
|
|
|
out << std::flush;
|
2011-08-07 04:58:56 +04:00
|
|
|
}
|
|
|
|
|
2013-05-11 17:13:26 +04:00
|
|
|
void OutputAllFeatureScores(const Moses::ScoreComponentCollection &features
|
|
|
|
, std::ostream &out)
|
2011-08-19 17:55:32 +04:00
|
|
|
{
|
|
|
|
std::string lastName = "";
|
2012-12-31 04:57:21 +04:00
|
|
|
const vector<const StatefulFeatureFunction*>& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions();
|
2012-12-16 22:29:53 +04:00
|
|
|
for( size_t i=0; i<sff.size(); i++ ) {
|
|
|
|
const StatefulFeatureFunction *ff = sff[i];
|
|
|
|
if (ff->GetScoreProducerDescription() != "BleuScoreFeature"
|
|
|
|
&& ff->IsTuneable()) {
|
|
|
|
OutputFeatureScores( out, features, ff, lastName );
|
|
|
|
}
|
|
|
|
}
|
2012-12-31 04:57:21 +04:00
|
|
|
const vector<const StatelessFeatureFunction*>& slf = StatelessFeatureFunction::GetStatelessFeatureFunctions();
|
2012-12-16 22:29:53 +04:00
|
|
|
for( size_t i=0; i<slf.size(); i++ ) {
|
|
|
|
const StatelessFeatureFunction *ff = slf[i];
|
|
|
|
if (ff->IsTuneable()) {
|
|
|
|
OutputFeatureScores( out, features, ff, lastName );
|
|
|
|
}
|
|
|
|
}
|
2011-08-07 04:58:56 +04:00
|
|
|
}
|
2008-09-12 22:09:06 +04:00
|
|
|
|
2012-12-07 17:34:44 +04:00
|
|
|
void OutputFeatureScores( std::ostream& out
|
2013-05-29 21:16:15 +04:00
|
|
|
, const ScoreComponentCollection &features
|
|
|
|
, const FeatureFunction *ff
|
|
|
|
, std::string &lastName )
|
2011-08-07 04:58:56 +04:00
|
|
|
{
|
2011-08-19 17:55:32 +04:00
|
|
|
const StaticData &staticData = StaticData::Instance();
|
2011-08-07 04:58:56 +04:00
|
|
|
bool labeledOutput = staticData.IsLabeledNBestList();
|
|
|
|
|
|
|
|
// regular features (not sparse)
|
2013-05-08 18:34:56 +04:00
|
|
|
if (ff->GetNumScoreComponents() != 0) {
|
2012-12-04 21:09:23 +04:00
|
|
|
if( labeledOutput && lastName != ff->GetScoreProducerDescription() ) {
|
|
|
|
lastName = ff->GetScoreProducerDescription();
|
2012-12-12 16:22:13 +04:00
|
|
|
out << " " << lastName << "=";
|
2011-08-19 17:55:32 +04:00
|
|
|
}
|
2012-12-07 17:34:44 +04:00
|
|
|
vector<float> scores = features.GetScoresForProducer( ff );
|
2011-08-19 17:55:32 +04:00
|
|
|
for (size_t j = 0; j<scores.size(); ++j) {
|
|
|
|
out << " " << scores[j];
|
|
|
|
}
|
2011-08-07 04:58:56 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
// sparse features
|
2013-06-09 21:48:09 +04:00
|
|
|
const FVector scores = features.GetVectorForProducer( ff );
|
|
|
|
for(FVector::FNVmap::const_iterator i = scores.cbegin(); i != scores.cend(); i++) {
|
|
|
|
out << " " << i->first << "= " << i->second;
|
2011-08-07 04:58:56 +04:00
|
|
|
}
|
2009-08-31 16:55:33 +04:00
|
|
|
}
|
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
void OutputLatticeMBRNBest(std::ostream& out, const vector<LatticeMBRSolution>& solutions,long translationId)
|
|
|
|
{
|
|
|
|
for (vector<LatticeMBRSolution>::const_iterator si = solutions.begin(); si != solutions.end(); ++si) {
|
|
|
|
out << translationId;
|
2011-10-03 20:11:39 +04:00
|
|
|
out << " |||";
|
2011-02-24 15:39:29 +03:00
|
|
|
const vector<Word> mbrHypo = si->GetWords();
|
|
|
|
for (size_t i = 0 ; i < mbrHypo.size() ; i++) {
|
|
|
|
const Factor *factor = mbrHypo[i].GetFactor(StaticData::Instance().GetOutputFactorOrder()[0]);
|
2011-10-03 20:11:39 +04:00
|
|
|
if (i>0) out << " " << *factor;
|
|
|
|
else out << *factor;
|
2011-02-24 15:39:29 +03:00
|
|
|
}
|
2011-10-03 20:11:39 +04:00
|
|
|
out << " |||";
|
|
|
|
out << " map: " << si->GetMapScore();
|
2011-02-24 15:39:29 +03:00
|
|
|
out << " w: " << mbrHypo.size();
|
|
|
|
const vector<float>& ngramScores = si->GetNgramScores();
|
|
|
|
for (size_t i = 0; i < ngramScores.size(); ++i) {
|
|
|
|
out << " " << ngramScores[i];
|
2010-03-16 14:34:50 +03:00
|
|
|
}
|
2011-10-03 20:11:39 +04:00
|
|
|
out << " ||| " << si->GetScore();
|
2011-02-24 15:39:29 +03:00
|
|
|
|
|
|
|
out << endl;
|
|
|
|
}
|
2010-03-16 14:34:50 +03:00
|
|
|
}
|
|
|
|
|
2009-08-07 20:47:54 +04:00
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
void IOWrapper::OutputLatticeMBRNBestList(const vector<LatticeMBRSolution>& solutions,long translationId)
|
|
|
|
{
|
|
|
|
OutputLatticeMBRNBest(*m_nBestStream, solutions,translationId);
|
2010-03-16 14:34:50 +03:00
|
|
|
}
|
|
|
|
|
2009-08-07 20:47:54 +04:00
|
|
|
bool ReadInput(IOWrapper &ioWrapper, InputTypeEnum inputType, InputType*& source)
|
|
|
|
{
|
2014-03-11 17:53:54 +04:00
|
|
|
if (source) delete source;
|
2011-02-24 15:39:29 +03:00
|
|
|
switch(inputType) {
|
|
|
|
case SentenceInput:
|
2011-11-21 15:14:05 +04:00
|
|
|
source = ioWrapper.GetInput(new Sentence);
|
2011-02-24 15:39:29 +03:00
|
|
|
break;
|
|
|
|
case ConfusionNetworkInput:
|
|
|
|
source = ioWrapper.GetInput(new ConfusionNet);
|
|
|
|
break;
|
|
|
|
case WordLatticeInput:
|
|
|
|
source = ioWrapper.GetInput(new WordLattice);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
TRACE_ERR("Unknown input type: " << inputType << "\n");
|
2014-03-11 17:53:54 +04:00
|
|
|
source = NULL;
|
2011-02-24 15:39:29 +03:00
|
|
|
}
|
|
|
|
return (source ? true : false);
|
2009-08-07 20:47:54 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
2012-06-29 07:19:28 +04:00
|
|
|
IOWrapper *GetIOWrapper(const StaticData &staticData)
|
2009-08-07 20:47:54 +04:00
|
|
|
{
|
2011-02-24 15:39:29 +03:00
|
|
|
IOWrapper *ioWrapper;
|
|
|
|
const std::vector<FactorType> &inputFactorOrder = staticData.GetInputFactorOrder()
|
2013-05-29 21:16:15 +04:00
|
|
|
,&outputFactorOrder = staticData.GetOutputFactorOrder();
|
2011-02-24 15:39:29 +03:00
|
|
|
FactorMask inputFactorUsed(inputFactorOrder);
|
|
|
|
|
|
|
|
// io
|
|
|
|
if (staticData.GetParam("input-file").size() == 1) {
|
|
|
|
VERBOSE(2,"IO from File" << endl);
|
|
|
|
string filePath = staticData.GetParam("input-file")[0];
|
|
|
|
|
|
|
|
ioWrapper = new IOWrapper(inputFactorOrder, outputFactorOrder, inputFactorUsed
|
2013-05-29 21:16:15 +04:00
|
|
|
, staticData.GetNBestSize()
|
|
|
|
, staticData.GetNBestFilePath()
|
|
|
|
, filePath);
|
2011-02-24 15:39:29 +03:00
|
|
|
} else {
|
|
|
|
VERBOSE(1,"IO from STDOUT/STDIN" << endl);
|
|
|
|
ioWrapper = new IOWrapper(inputFactorOrder, outputFactorOrder, inputFactorUsed
|
2013-05-29 21:16:15 +04:00
|
|
|
, staticData.GetNBestSize()
|
|
|
|
, staticData.GetNBestFilePath());
|
2011-02-24 15:39:29 +03:00
|
|
|
}
|
|
|
|
ioWrapper->ResetTranslationId();
|
|
|
|
|
|
|
|
IFVERBOSE(1)
|
|
|
|
PrintUserTime("Created input-output object");
|
|
|
|
|
|
|
|
return ioWrapper;
|
2009-08-07 20:47:54 +04:00
|
|
|
}
|
2012-07-02 20:05:11 +04:00
|
|
|
|
|
|
|
}
|
|
|
|
|