2006-07-04 22:04:38 +04:00
|
|
|
// $Id$
|
|
|
|
|
|
|
|
/***********************************************************************
|
|
|
|
Moses - factored phrase-based language decoder
|
|
|
|
Copyright (c) 2006 University of Edinburgh
|
|
|
|
All rights reserved.
|
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
Redistribution and use in source and binary forms, with or without modification,
|
2006-07-04 22:04:38 +04:00
|
|
|
are permitted provided that the following conditions are met:
|
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
* Redistributions of source code must retain the above copyright notice,
|
2006-07-04 22:04:38 +04:00
|
|
|
this list of conditions and the following disclaimer.
|
2011-02-24 15:39:29 +03:00
|
|
|
* Redistributions in binary form must reproduce the above copyright notice,
|
|
|
|
this list of conditions and the following disclaimer in the documentation
|
2006-07-04 22:04:38 +04:00
|
|
|
and/or other materials provided with the distribution.
|
2011-02-24 15:39:29 +03:00
|
|
|
* Neither the name of the University of Edinburgh nor the names of its contributors
|
|
|
|
may be used to endorse or promote products derived from this software
|
2006-07-04 22:04:38 +04:00
|
|
|
without specific prior written permission.
|
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
|
|
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
|
|
|
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
|
|
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
|
|
|
|
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
|
|
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
|
|
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
|
|
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
|
|
|
|
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
|
|
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
2006-07-04 22:04:38 +04:00
|
|
|
POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
***********************************************************************/
|
|
|
|
|
|
|
|
// example file on how to use moses library
|
|
|
|
|
2014-10-13 15:46:58 +04:00
|
|
|
#pragma once
|
2006-07-04 22:04:38 +04:00
|
|
|
|
2012-05-28 20:29:46 +04:00
|
|
|
#include <cassert>
|
2006-07-04 22:04:38 +04:00
|
|
|
#include <fstream>
|
2010-02-17 20:25:56 +03:00
|
|
|
#include <ostream>
|
2006-07-04 22:04:38 +04:00
|
|
|
#include <vector>
|
2010-02-17 20:25:56 +03:00
|
|
|
|
2012-11-13 00:21:32 +04:00
|
|
|
#include "moses/TypeDef.h"
|
|
|
|
#include "moses/Sentence.h"
|
|
|
|
#include "moses/FactorTypeSet.h"
|
|
|
|
#include "moses/FactorCollection.h"
|
|
|
|
#include "moses/Hypothesis.h"
|
|
|
|
#include "moses/OutputCollector.h"
|
|
|
|
#include "moses/TrellisPathList.h"
|
|
|
|
#include "moses/InputFileStream.h"
|
|
|
|
#include "moses/InputType.h"
|
|
|
|
#include "moses/WordLattice.h"
|
2014-09-29 20:34:11 +04:00
|
|
|
#include "moses/LatticeMBR.h"
|
2014-10-09 19:42:16 +04:00
|
|
|
#include "moses/ChartKBestExtractor.h"
|
2014-11-07 22:51:18 +03:00
|
|
|
#include "moses/Syntax/KBestExtractor.h"
|
2014-10-09 19:42:16 +04:00
|
|
|
|
|
|
|
#include "search/applied.hh"
|
2006-07-04 22:04:38 +04:00
|
|
|
|
2012-12-07 17:34:44 +04:00
|
|
|
namespace Moses
|
|
|
|
{
|
|
|
|
class ScoreComponentCollection;
|
2013-09-19 18:33:21 +04:00
|
|
|
class Hypothesis;
|
2014-10-09 19:42:16 +04:00
|
|
|
class ChartHypothesis;
|
2013-09-19 18:33:21 +04:00
|
|
|
class Factor;
|
2012-07-02 20:05:11 +04:00
|
|
|
|
2014-11-07 22:51:18 +03:00
|
|
|
namespace Syntax
|
|
|
|
{
|
|
|
|
struct SHyperedge;
|
|
|
|
}
|
|
|
|
|
2012-06-29 07:19:28 +04:00
|
|
|
/** Helper class that holds misc variables to write data out to command line.
|
|
|
|
*/
|
2008-08-05 04:24:45 +04:00
|
|
|
class IOWrapper
|
2006-07-04 22:04:38 +04:00
|
|
|
{
|
|
|
|
protected:
|
2011-02-24 15:39:29 +03:00
|
|
|
|
|
|
|
const std::vector<Moses::FactorType> &m_inputFactorOrder;
|
|
|
|
const std::vector<Moses::FactorType> &m_outputFactorOrder;
|
|
|
|
const Moses::FactorMask &m_inputFactorUsed;
|
|
|
|
std::string m_inputFilePath;
|
|
|
|
Moses::InputFileStream *m_inputFile;
|
|
|
|
std::istream *m_inputStream;
|
2014-10-02 17:57:04 +04:00
|
|
|
std::ostream *m_nBestStream;
|
|
|
|
std::ostream *m_outputWordGraphStream;
|
|
|
|
std::ostream *m_detailedTranslationReportingStream;
|
2014-10-02 14:22:52 +04:00
|
|
|
std::ofstream *m_alignmentInfoStream;
|
2014-10-02 15:20:49 +04:00
|
|
|
std::ostream *m_unknownsStream;
|
|
|
|
std::ostream *m_outputSearchGraphStream;
|
2014-10-03 18:30:06 +04:00
|
|
|
std::ofstream *m_latticeSamplesStream;
|
2014-10-10 14:35:14 +04:00
|
|
|
std::ostream *m_detailedTreeFragmentsTranslationReportingStream;
|
2014-10-01 22:24:58 +04:00
|
|
|
|
2014-10-02 20:08:29 +04:00
|
|
|
bool m_surpressSingleBestOutput;
|
2011-02-24 15:39:29 +03:00
|
|
|
|
2014-10-02 19:00:36 +04:00
|
|
|
Moses::OutputCollector *m_singleBestOutputCollector;
|
|
|
|
Moses::OutputCollector *m_nBestOutputCollector;
|
|
|
|
Moses::OutputCollector *m_unknownsCollector;
|
|
|
|
Moses::OutputCollector *m_alignmentInfoCollector;
|
|
|
|
Moses::OutputCollector *m_searchGraphOutputCollector;
|
2014-10-02 17:57:04 +04:00
|
|
|
Moses::OutputCollector *m_detailedTranslationCollector;
|
|
|
|
Moses::OutputCollector *m_wordGraphCollector;
|
2014-10-03 15:30:18 +04:00
|
|
|
Moses::OutputCollector *m_latticeSamplesCollector;
|
2014-10-10 14:35:14 +04:00
|
|
|
Moses::OutputCollector *m_detailTreeFragmentsOutputCollector;
|
|
|
|
|
|
|
|
// CHART
|
|
|
|
typedef std::vector<std::pair<Moses::Word, Moses::WordsRange> > ApplicationContext;
|
|
|
|
typedef std::set< std::pair<size_t, size_t> > Alignments;
|
|
|
|
|
|
|
|
void Backtrack(const ChartHypothesis *hypo);
|
|
|
|
void OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
|
|
|
void OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
|
|
|
|
void OutputTranslationOption(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
|
|
|
void OutputTranslationOption(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
|
|
|
|
|
|
|
|
void ReconstructApplicationContext(const Moses::ChartHypothesis &hypo,
|
|
|
|
const Moses::Sentence &sentence,
|
|
|
|
ApplicationContext &context);
|
|
|
|
void ReconstructApplicationContext(const search::Applied *applied,
|
|
|
|
const Moses::Sentence &sentence,
|
|
|
|
ApplicationContext &context);
|
|
|
|
void WriteApplicationContext(std::ostream &out,
|
|
|
|
const ApplicationContext &context);
|
|
|
|
void OutputTreeFragmentsTranslationOptions(std::ostream &out,
|
|
|
|
ApplicationContext &applicationContext,
|
|
|
|
const Moses::ChartHypothesis *hypo,
|
|
|
|
const Moses::Sentence &sentence,
|
|
|
|
long translationId);
|
|
|
|
void OutputTreeFragmentsTranslationOptions(std::ostream &out,
|
|
|
|
ApplicationContext &applicationContext,
|
|
|
|
const search::Applied *applied,
|
|
|
|
const Moses::Sentence &sentence,
|
|
|
|
long translationId);
|
|
|
|
|
|
|
|
void OutputSurface(std::ostream &out, const Phrase &phrase, const std::vector<FactorType> &outputFactorOrder, bool reportAllFactors);
|
2014-10-13 15:46:58 +04:00
|
|
|
void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector<FactorType> &outputFactorOrder,
|
|
|
|
char reportSegmentation, bool reportAllFactors);
|
2014-10-10 14:35:14 +04:00
|
|
|
|
|
|
|
size_t OutputAlignment(Alignments &retAlign, const Moses::ChartHypothesis *hypo, size_t startTarget);
|
|
|
|
size_t OutputAlignmentNBest(Alignments &retAlign,
|
|
|
|
const Moses::ChartKBestExtractor::Derivation &derivation,
|
|
|
|
size_t startTarget);
|
2014-11-07 22:51:18 +03:00
|
|
|
std::size_t OutputAlignmentNBest(Alignments &retAlign, const Moses::Syntax::KBestExtractor::Derivation &derivation, std::size_t startTarget);
|
2014-10-10 14:35:14 +04:00
|
|
|
|
|
|
|
size_t CalcSourceSize(const Moses::ChartHypothesis *hypo);
|
2014-11-07 22:51:18 +03:00
|
|
|
size_t CalcSourceSize(const Syntax::KBestExtractor::Derivation &d) const;
|
2014-10-10 14:35:14 +04:00
|
|
|
|
|
|
|
template <class T>
|
|
|
|
void ShiftOffsets(std::vector<T> &offsets, T shift)
|
|
|
|
{
|
|
|
|
T currPos = shift;
|
|
|
|
for (size_t i = 0; i < offsets.size(); ++i) {
|
|
|
|
if (offsets[i] == 0) {
|
|
|
|
offsets[i] = currPos;
|
|
|
|
++currPos;
|
|
|
|
} else {
|
|
|
|
currPos += offsets[i];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2014-10-01 20:43:51 +04:00
|
|
|
|
2006-07-04 22:04:38 +04:00
|
|
|
public:
|
2014-10-03 19:08:15 +04:00
|
|
|
static IOWrapper *GetIOWrapper(const Moses::StaticData &staticData);
|
2014-10-10 14:35:14 +04:00
|
|
|
static void FixPrecision(std::ostream &, size_t size=3);
|
2014-10-03 19:08:15 +04:00
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
IOWrapper(const std::vector<Moses::FactorType> &inputFactorOrder
|
|
|
|
, const std::vector<Moses::FactorType> &outputFactorOrder
|
|
|
|
, const Moses::FactorMask &inputFactorUsed
|
|
|
|
, size_t nBestSize
|
|
|
|
, const std::string &nBestFilePath
|
2014-10-02 19:00:36 +04:00
|
|
|
, const std::string &inputFilePath = "");
|
2011-02-24 15:39:29 +03:00
|
|
|
~IOWrapper();
|
|
|
|
|
|
|
|
Moses::InputType* GetInput(Moses::InputType *inputType);
|
2014-10-13 15:46:58 +04:00
|
|
|
bool ReadInput(Moses::InputTypeEnum inputType, Moses::InputType*& source);
|
2011-02-24 15:39:29 +03:00
|
|
|
|
2013-08-07 08:31:45 +04:00
|
|
|
void OutputBestHypo(const Moses::Hypothesis *hypo, long translationId, char reportSegmentation, bool reportAllFactors);
|
2010-05-11 01:18:47 +04:00
|
|
|
void OutputLatticeMBRNBestList(const std::vector<LatticeMBRSolution>& solutions,long translationId);
|
2011-02-24 15:39:29 +03:00
|
|
|
void Backtrack(const Moses::Hypothesis *hypo);
|
|
|
|
|
2014-10-01 20:43:51 +04:00
|
|
|
Moses::OutputCollector *GetSingleBestOutputCollector() {
|
|
|
|
return m_singleBestOutputCollector;
|
|
|
|
}
|
2014-10-01 21:21:57 +04:00
|
|
|
|
|
|
|
Moses::OutputCollector *GetNBestOutputCollector() {
|
|
|
|
return m_nBestOutputCollector;
|
|
|
|
}
|
|
|
|
|
2014-10-01 22:24:58 +04:00
|
|
|
Moses::OutputCollector *GetUnknownsCollector() {
|
|
|
|
return m_unknownsCollector;
|
|
|
|
}
|
2014-10-02 14:22:52 +04:00
|
|
|
|
|
|
|
Moses::OutputCollector *GetAlignmentInfoCollector() {
|
|
|
|
return m_alignmentInfoCollector;
|
|
|
|
}
|
|
|
|
|
2014-10-02 15:20:49 +04:00
|
|
|
Moses::OutputCollector *GetSearchGraphOutputCollector() {
|
|
|
|
return m_searchGraphOutputCollector;
|
|
|
|
}
|
2014-10-02 14:22:52 +04:00
|
|
|
|
2014-10-02 17:18:12 +04:00
|
|
|
Moses::OutputCollector *GetDetailedTranslationCollector() {
|
|
|
|
return m_detailedTranslationCollector;
|
|
|
|
}
|
|
|
|
|
2014-10-02 17:57:04 +04:00
|
|
|
Moses::OutputCollector *GetWordGraphCollector() {
|
|
|
|
return m_wordGraphCollector;
|
|
|
|
}
|
|
|
|
|
2014-10-03 15:30:18 +04:00
|
|
|
Moses::OutputCollector *GetLatticeSamplesCollector() {
|
|
|
|
return m_latticeSamplesCollector;
|
|
|
|
}
|
|
|
|
|
2014-10-10 14:35:14 +04:00
|
|
|
// CHART
|
|
|
|
void OutputBestHypo(const Moses::ChartHypothesis *hypo, long translationId);
|
|
|
|
void OutputBestHypo(search::Applied applied, long translationId);
|
2014-11-07 22:51:18 +03:00
|
|
|
void OutputBestHypo(const Moses::Syntax::SHyperedge *, long translationId);
|
|
|
|
|
2014-10-10 14:35:14 +04:00
|
|
|
void OutputBestNone(long translationId);
|
2014-11-07 22:51:18 +03:00
|
|
|
|
2014-10-10 14:35:14 +04:00
|
|
|
void OutputNBestList(const std::vector<boost::shared_ptr<Moses::ChartKBestExtractor::Derivation> > &nBestList, long translationId);
|
|
|
|
void OutputNBestList(const std::vector<search::Applied> &nbest, long translationId);
|
2014-11-07 22:51:18 +03:00
|
|
|
void OutputNBestList(const Moses::Syntax::KBestExtractor::KBestVec &nBestList, long translationId);
|
|
|
|
|
2014-10-10 14:35:14 +04:00
|
|
|
void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
|
|
|
void OutputDetailedTranslationReport(const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
|
2014-11-07 22:51:18 +03:00
|
|
|
void OutputDetailedTranslationReport(const Moses::Syntax::SHyperedge *, long translationId);
|
|
|
|
|
2014-10-10 14:35:14 +04:00
|
|
|
void OutputDetailedAllTranslationReport(const std::vector<boost::shared_ptr<Moses::ChartKBestExtractor::Derivation> > &nBestList, const Moses::ChartManager &manager, const Moses::Sentence &sentence, long translationId);
|
|
|
|
|
|
|
|
void OutputAlignment(size_t translationId , const Moses::ChartHypothesis *hypo);
|
|
|
|
void OutputUnknowns(const std::vector<Moses::Phrase*> &, long);
|
2014-11-07 22:51:18 +03:00
|
|
|
void OutputUnknowns(const std::set<Moses::Word> &, long);
|
2014-10-10 14:35:14 +04:00
|
|
|
|
|
|
|
void OutputDetailedTreeFragmentsTranslationReport(const Moses::ChartHypothesis *hypo,
|
|
|
|
const Moses::Sentence &sentence,
|
|
|
|
long translationId);
|
|
|
|
void OutputDetailedTreeFragmentsTranslationReport(const search::Applied *applied,
|
|
|
|
const Moses::Sentence &sentence,
|
|
|
|
long translationId);
|
|
|
|
|
2014-10-13 15:46:58 +04:00
|
|
|
// phrase-based
|
|
|
|
void OutputBestSurface(std::ostream &out, const Moses::Hypothesis *hypo, const std::vector<Moses::FactorType> &outputFactorOrder, char reportSegmentation, bool reportAllFactors);
|
|
|
|
void OutputLatticeMBRNBest(std::ostream& out, const std::vector<LatticeMBRSolution>& solutions,long translationId);
|
|
|
|
void OutputBestHypo(const std::vector<Moses::Word>& mbrBestHypo, long /*translationId*/,
|
|
|
|
char reportSegmentation, bool reportAllFactors, std::ostream& out);
|
|
|
|
void OutputBestHypo(const Moses::TrellisPath &path, long /*translationId*/,char reportSegmentation, bool reportAllFactors, std::ostream &out);
|
|
|
|
void OutputInput(std::ostream& os, const Moses::Hypothesis* hypo);
|
|
|
|
void OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo);
|
|
|
|
|
|
|
|
void OutputAlignment(Moses::OutputCollector* collector, size_t lineNo, const Moses::Hypothesis *hypo);
|
|
|
|
void OutputAlignment(Moses::OutputCollector* collector, size_t lineNo, const Moses::TrellisPath &path);
|
|
|
|
void OutputAlignment(OutputCollector* collector, size_t lineNo , const std::vector<const Hypothesis *> &edges);
|
|
|
|
|
|
|
|
static void OutputAlignment(std::ostream &out, const Moses::Hypothesis *hypo);
|
|
|
|
static void OutputAlignment(std::ostream &out, const std::vector<const Hypothesis *> &edges);
|
|
|
|
static void OutputAlignment(std::ostream &out, const Moses::AlignmentInfo &ai, size_t sourceOffset, size_t targetOffset);
|
|
|
|
|
|
|
|
void OutputNBest(std::ostream& out
|
|
|
|
, const Moses::TrellisPathList &nBestList
|
|
|
|
, const std::vector<Moses::FactorType>& outputFactorOrder
|
|
|
|
, long translationId
|
|
|
|
, char reportSegmentation);
|
|
|
|
|
|
|
|
static void OutputAllFeatureScores(const Moses::ScoreComponentCollection &features
|
|
|
|
, std::ostream &out);
|
|
|
|
static void OutputFeatureScores( std::ostream& out
|
|
|
|
, const Moses::ScoreComponentCollection &features
|
|
|
|
, const Moses::FeatureFunction *ff
|
|
|
|
, std::string &lastName );
|
|
|
|
|
|
|
|
// creates a map of TARGET positions which should be replaced by word using placeholder
|
|
|
|
std::map<size_t, const Moses::Factor*> GetPlaceholders(const Moses::Hypothesis &hypo, Moses::FactorType placeholderFactor);
|
|
|
|
|
2006-11-22 02:06:30 +03:00
|
|
|
};
|
2009-08-07 20:47:54 +04:00
|
|
|
|
2013-09-19 18:33:21 +04:00
|
|
|
|
2014-10-09 19:42:16 +04:00
|
|
|
|
2012-07-02 20:05:11 +04:00
|
|
|
}
|
|
|
|
|