moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-10-05 15:58:03 +03:00
Code Issues Packages Projects Releases Wiki Activity

add OutputSearchGraphHypergraph() to API framework. Move m_source to BaseManager

Browse Source
This commit is contained in:
Hieu Hoang 2014-12-05 21:33:59 +00:00
parent 6a77fd0ce3
commit 4b10c59bea
18 changed files with 84 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
OnDiskPt/Jamfile
View File

@ -1,5 +1,5 @@
fakelib OnDiskPt : OnDiskWrapper.cpp SourcePhrase.cpp TargetPhrase.cpp Word.cpp Phrase.cpp PhraseNode.cpp TargetPhraseCollection.cpp Vocab.cpp OnDiskQuery.cpp ../moses//headers ;
exe CreateOnDiskPt : Main.cpp ../moses//moses OnDiskPt ;
exe queryOnDiskPt : queryOnDiskPt.cpp ../moses//moses OnDiskPt ;
exe CreateOnDiskPt : Main.cpp ..//boost_filesystem ../moses//moses OnDiskPt ;
exe queryOnDiskPt : queryOnDiskPt.cpp ..//boost_filesystem ../moses//moses OnDiskPt ;

26
misc/Jamfile
View File

@ -1,24 +1,24 @@
exe processPhraseTable : GenerateTuples.cpp processPhraseTable.cpp ../moses//moses ;
exe processPhraseTable : GenerateTuples.cpp processPhraseTable.cpp ..//boost_filesystem ../moses//moses ;
exe processLexicalTable : processLexicalTable.cpp ../moses//moses ;
exe processLexicalTable : processLexicalTable.cpp ..//boost_filesystem ../moses//moses ;
exe queryPhraseTable : queryPhraseTable.cpp ../moses//moses ;
exe queryPhraseTable : queryPhraseTable.cpp ..//boost_filesystem ../moses//moses ;
exe queryLexicalTable : queryLexicalTable.cpp ../moses//moses ;
exe queryLexicalTable : queryLexicalTable.cpp ..//boost_filesystem ../moses//moses ;
exe generateSequences : GenerateSequences.cpp ../moses//moses ;
exe generateSequences : GenerateSequences.cpp ..//boost_filesystem ../moses//moses ;
exe TMining : TransliterationMining.cpp ../moses//moses ;
exe TMining : TransliterationMining.cpp ..//boost_filesystem ../moses//moses ;
exe 1-1-Extraction : 1-1-Extraction.cpp ../moses//moses ;
exe 1-1-Extraction : 1-1-Extraction.cpp ..//boost_filesystem ../moses//moses ;
exe prunePhraseTable : prunePhraseTable.cpp ../moses//moses ..//boost_program_options ;
exe prunePhraseTable : prunePhraseTable.cpp ..//boost_filesystem ../moses//moses ..//boost_program_options ;
local with-cmph = [ option.get "with-cmph" ] ;
if $(with-cmph) {
exe processPhraseTableMin : processPhraseTableMin.cpp ../moses//moses ;
exe processLexicalTableMin : processLexicalTableMin.cpp ../moses//moses ;
exe queryPhraseTableMin : queryPhraseTableMin.cpp ../moses//moses ;
exe processPhraseTableMin : processPhraseTableMin.cpp ..//boost_filesystem ../moses//moses ;
exe processLexicalTableMin : processLexicalTableMin.cpp ..//boost_filesystem ../moses//moses ;
exe queryPhraseTableMin : queryPhraseTableMin.cpp ..//boost_filesystem ../moses//moses ;
alias programsMin : processPhraseTableMin processLexicalTableMin queryPhraseTableMin ;
# alias programsMin : processPhraseTableMin processLexicalTableMin ;
@ -29,8 +29,8 @@ else {
if [ option.get "with-probing-pt" : : "yes" ]
{
exe CreateProbingPT : CreateProbingPT.cpp ../moses//moses ;
exe QueryProbingPT : QueryProbingPT.cpp ../moses//moses ;
exe CreateProbingPT : CreateProbingPT.cpp ..//boost_filesystem ../moses//moses ;
exe QueryProbingPT : QueryProbingPT.cpp ..//boost_filesystem ../moses//moses ;
alias programsProbing : CreateProbingPT QueryProbingPT ;
}

18
moses-cmd/Main.cpp
View File

@ -36,7 +36,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "moses/IOWrapper.h"
#include "moses/Hypothesis.h"
#include "moses/HypergraphOutput.h"
#include "moses/Manager.h"
#include "moses/StaticData.h"
#include "moses/TypeDef.h"
@ -139,18 +138,6 @@ int main(int argc, char** argv)
TRACE_ERR("\n");
}
boost::shared_ptr<HypergraphOutput<Manager> > hypergraphOutput;
boost::shared_ptr<HypergraphOutput<ChartManager> > hypergraphOutputChart;
if (staticData.GetOutputSearchGraphHypergraph()) {
if (staticData.IsChart()) {
hypergraphOutputChart.reset(new HypergraphOutput<ChartManager>(PRECISION));
}
else {
hypergraphOutput.reset(new HypergraphOutput<Manager>(PRECISION));
}
}
#ifdef WITH_THREADS
ThreadPool pool(staticData.ThreadCount());
#endif
@ -170,12 +157,11 @@ int main(int argc, char** argv)
TranslationTask* task;
if (staticData.IsChart()) {
// scfg
task = new TranslationTask(source, *ioWrapper, hypergraphOutputChart);
task = new TranslationTask(source, *ioWrapper, 2);
}
else {
// pb
task = new TranslationTask(source, *ioWrapper,
hypergraphOutput);
task = new TranslationTask(source, *ioWrapper, 1);
}
// execute task

12
moses/BaseManager.h
View File

@ -3,6 +3,7 @@
#include <iostream>
#include <string>
#include "ScoreComponentCollection.h"
#include "InputType.h"
namespace Moses
{
@ -13,6 +14,11 @@ class OutputCollector;
class BaseManager
{
protected:
const InputType &m_source; /**< source sentence to be translated */
BaseManager(const InputType &source)
:m_source(source)
{}
// output
typedef std::vector<std::pair<Moses::Word, Moses::WordsRange> > ApplicationContext;
@ -46,6 +52,11 @@ protected:
}
public:
//! the input sentence being decoded
const InputType& GetSource() const {
return m_source;
}
virtual void Decode() = 0;
// outputs
virtual void OutputNBest(OutputCollector *collector) const = 0;
@ -56,6 +67,7 @@ public:
virtual void OutputWordGraph(OutputCollector *collector) const = 0;
virtual void OutputSearchGraph(OutputCollector *collector) const = 0;
virtual void OutputSearchGraphSLF() const = 0;
virtual void OutputSearchGraphHypergraph() const = 0;
};

12
moses/ChartManager.cpp
View File

@ -33,6 +33,7 @@
#include "moses/FF/WordPenaltyProducer.h"
#include "moses/OutputCollector.h"
#include "moses/ChartKBestExtractor.h"
#include "moses/HypergraphOutput.h"
using namespace std;
@ -45,7 +46,7 @@ extern bool g_mosesDebug;
* \param system which particular set of models to use.
*/
ChartManager::ChartManager(InputType const& source)
:m_source(source)
:BaseManager(source)
,m_hypoStackColl(source, *this)
,m_start(clock())
,m_hypothesisId(0)
@ -797,4 +798,13 @@ void ChartManager::OutputDetailedAllTranslationReport(
collector->Write(translationId, out.str());
}
void ChartManager::OutputSearchGraphHypergraph() const
{
const StaticData &staticData = StaticData::Instance();
if (staticData.GetOutputSearchGraphHypergraph()) {
HypergraphOutput<ChartManager> hypergraphOutputChart(PRECISION);
hypergraphOutputChart.Write(*this);
}
}
} // namespace Moses

9
moses/ChartManager.h
View File

@ -25,7 +25,6 @@
#include <boost/unordered_map.hpp>
#include "ChartCell.h"
#include "ChartCellCollection.h"
#include "InputType.h"
#include "WordsRange.h"
#include "SentenceStats.h"
#include "ChartTranslationOptionList.h"
@ -47,7 +46,6 @@ class ChartSearchGraphWriter;
class ChartManager : public BaseManager
{
private:
InputType const& m_source; /**< source sentence to be translated */
ChartCellCollection m_hypoStackColl;
std::auto_ptr<SentenceStats> m_sentenceStats;
clock_t m_start; /**< starting time, used for logging */
@ -116,12 +114,6 @@ public:
/** Output in (modified) Kenneth hypergraph format */
void OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const;
//! the input sentence being decoded
const InputType& GetSource() const {
return m_source;
}
//! debug data collected when decoding sentence
SentenceStats& GetSentenceStats() const {
return *m_sentenceStats;
@ -163,6 +155,7 @@ public:
void OutputSearchGraph(OutputCollector *collector) const;
void OutputSearchGraphSLF() const
{}
void OutputSearchGraphHypergraph() const;
};

2
moses/Hypothesis.cpp
View File

@ -47,7 +47,7 @@ ObjectPool<Hypothesis> Hypothesis::s_objectPool("Hypothesis", 300000);
Hypothesis::Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt)
: m_prevHypo(NULL)
, m_sourceCompleted(source.GetSize(), manager.m_source.m_sourceCompleted)
, m_sourceCompleted(source.GetSize(), manager.GetSource().m_sourceCompleted)
, m_sourceInput(source)
, m_currSourceWordsRange(
m_sourceCompleted.GetFirstGapPos()>0 ? 0 : NOT_FOUND,

14
moses/Incremental.cpp
View File

@ -204,7 +204,7 @@ struct ChartCellBaseFactory {
} // namespace
Manager::Manager(const InputType &source) :
source_(source),
BaseManager(source),
cells_(source, ChartCellBaseFactory()),
parser_(source, cells_),
n_best_(search::NBestConfig(StaticData::Instance().GetNBestSize())) {}
@ -221,7 +221,7 @@ template <class Model, class Best> search::History Manager::PopulateBest(const M
search::Config config(abstract.GetWeight() * M_LN10, data.GetCubePruningPopLimit(), search::NBestConfig(data.GetNBestSize()));
search::Context<Model> context(config, model);
size_t size = source_.GetSize();
size_t size = m_source.GetSize();
boost::object_pool<search::Vertex> vertex_pool(std::max<size_t>(size * size / 2, 32));
for (int startPos = size-1; startPos >= 0; --startPos) {
@ -289,7 +289,7 @@ void Manager::OutputNBest(OutputCollector *collector) const
return;
}
OutputNBestList(collector, *completed_nbest_, source_.GetTranslationId());
OutputNBestList(collector, *completed_nbest_, m_source.GetTranslationId());
}
void Manager::OutputNBestList(OutputCollector *collector, const std::vector<search::Applied> &nbest, long translationId) const
@ -329,8 +329,8 @@ void Manager::OutputDetailedTranslationReport(OutputCollector *collector) const
const search::Applied &applied = completed_nbest_->at(0);
OutputDetailedTranslationReport(collector,
&applied,
static_cast<const Sentence&>(source_),
source_.GetTranslationId());
static_cast<const Sentence&>(m_source),
m_source.GetTranslationId());
}
}
@ -422,8 +422,8 @@ void Manager::OutputDetailedTreeFragmentsTranslationReport(OutputCollector *coll
}
const search::Applied *applied = &Completed()[0];
const Sentence &sentence = dynamic_cast<const Sentence &>(source_);
const size_t translationId = source_.GetTranslationId();
const Sentence &sentence = dynamic_cast<const Sentence &>(m_source);
const size_t translationId = m_source.GetTranslationId();
std::ostringstream out;
ApplicationContext applicationContext;

3
moses/Incremental.h
View File

@ -54,12 +54,13 @@ public:
{}
void OutputSearchGraphSLF() const
{}
void OutputSearchGraphHypergraph() const
{}
private:
template <class Model, class Best> search::History PopulateBest(const Model &model, const std::vector<lm::WordIndex> &words, Best &out);
const InputType &source_;
ChartCellCollectionBase cells_;
ChartParser parser_;

4
moses/Jamfile
View File

@ -89,7 +89,7 @@ lib moses :
headers FF_Factory.o LM//LM TranslationModel/CompactPT//CompactPT TranslationModel/ProbingPT//ProbingPT synlm ThreadPool
..//search ../util/double-conversion//double-conversion ..//z ../OnDiskPt//OnDiskPt
$(TOP)//boost_iostreams mmlib
$(TOP)//boost_filesystem $(TOP)//boost_iostreams mmlib
:
<threading>single:<source>../util//rt
;
@ -99,5 +99,5 @@ alias headers-to-install : [ glob-tree *.h ] ;
import testing ;
unit-test moses_test : [ glob *Test.cpp Mock*.cpp FF/*Test.cpp ] moses headers ..//z ../OnDiskPt//OnDiskPt ..//boost_unit_test_framework ;
unit-test moses_test : [ glob *Test.cpp Mock*.cpp FF/*Test.cpp ] ..//boost_filesystem moses headers ..//z ../OnDiskPt//OnDiskPt ..//boost_unit_test_framework ;

14
moses/Manager.cpp
View File

@ -44,6 +44,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "moses/LM/Base.h"
#include "moses/TranslationModel/PhraseDictionary.h"
#include "moses/TranslationAnalysis.h"
#include "moses/HypergraphOutput.h"
#ifdef HAVE_PROTOBUF
#include "hypergraph.pb.h"
@ -57,11 +58,11 @@ using namespace std;
namespace Moses
{
Manager::Manager(InputType const& source, SearchAlgorithm searchAlgorithm)
:m_transOptColl(source.CreateTranslationOptionCollection())
:BaseManager(source)
,m_transOptColl(source.CreateTranslationOptionCollection())
,m_search(Search::CreateSearch(*this, source, searchAlgorithm, *m_transOptColl))
,interrupted_flag(0)
,m_hypoId(0)
,m_source(source)
{
StaticData::Instance().InitializeForInput(m_source);
}
@ -1810,4 +1811,13 @@ void Manager::OutputSearchGraphSLF() const
}
void Manager::OutputSearchGraphHypergraph() const
{
const StaticData &staticData = StaticData::Instance();
if (staticData.GetOutputSearchGraphHypergraph()) {
HypergraphOutput<Manager> hypergraphOutput(PRECISION);
hypergraphOutput.Write(*this);
}
}
} // namespace

3
moses/Manager.h
View File

@ -111,7 +111,6 @@ private:
protected:
// data
// InputType const& m_source; /**< source sentence to be translated */
TranslationOptionCollection *m_transOptColl; /**< pre-computed list of translation options for the phrases in this sentence */
Search *m_search;
@ -146,7 +145,6 @@ protected:
void OutputWordGraph(std::ostream &outputWordGraphStream, const Hypothesis *hypo, size_t &linkId) const;
public:
InputType const& m_source; /**< source sentence to be translated */
Manager(InputType const& source, SearchAlgorithm searchAlgorithm);
~Manager();
const TranslationOptionCollection* getSntTranslationOptions();
@ -199,6 +197,7 @@ public:
void OutputWordGraph(OutputCollector *collector) const;
void OutputSearchGraph(OutputCollector *collector) const;
void OutputSearchGraphSLF() const;
void OutputSearchGraphHypergraph() const;
};

2
moses/Syntax/S2T/Manager-inl.h
View File

@ -30,7 +30,7 @@ namespace S2T
template<typename Parser>
Manager<Parser>::Manager(const InputType &source)
: m_source(source)
: BaseManager(source)
, m_pchart(source.GetSize(), Parser::RequiresCompressedChart())
, m_schart(source.GetSize())
{

3
moses/Syntax/S2T/Manager.h
View File

@ -58,6 +58,8 @@ class Manager : public BaseManager
{}
void OutputSearchGraphSLF() const
{}
void OutputSearchGraphHypergraph() const
{}
private:
void FindOovs(const PChart &, std::set<Word> &, std::size_t);
@ -70,7 +72,6 @@ class Manager : public BaseManager
void PrunePChart(const SChart::Cell &, PChart::Cell &);
const InputType &m_source;
PChart m_pchart;
SChart m_schart;
std::set<Word> m_oovs;

9
moses/TranslationModel/UG/Jamfile
View File

@ -3,6 +3,7 @@ try-align.cc
$(TOP)/moses//moses
$(TOP)/moses/TranslationModel/UG/generic//generic
$(TOP)//boost_iostreams
$(TOP)//boost_filesystem
$(TOP)//boost_program_options
$(TOP)/moses/TranslationModel/UG/mm//mm
$(TOP)/moses/TranslationModel/UG//mmsapt
@ -15,6 +16,7 @@ $(TOP)/moses//moses
$(TOP)/moses/TranslationModel/UG/generic//generic
$(TOP)/moses/TranslationModel/UG/generic//stringdist
$(TOP)//boost_iostreams
$(TOP)//boost_filesystem
$(TOP)//boost_program_options
$(TOP)/moses/TranslationModel/UG/mm//mm
$(TOP)/moses/TranslationModel/UG//mmsapt
@ -26,6 +28,7 @@ ptable-describe-features.cc
$(TOP)/moses//moses
$(TOP)/moses/TranslationModel/UG/generic//generic
$(TOP)//boost_iostreams
$(TOP)//boost_filesystem
$(TOP)//boost_program_options
$(TOP)/moses/TranslationModel/UG/mm//mm
$(TOP)/moses/TranslationModel/UG//mmsapt
@ -37,6 +40,7 @@ count-ptable-features.cc
$(TOP)/moses//moses
$(TOP)/moses/TranslationModel/UG/generic//generic
$(TOP)//boost_iostreams
$(TOP)//boost_filesystem
$(TOP)//boost_program_options
$(TOP)/moses/TranslationModel/UG/mm//mm
$(TOP)/moses/TranslationModel/UG//mmsapt
@ -48,6 +52,7 @@ ptable-lookup.cc
$(TOP)/moses//moses
$(TOP)/moses/TranslationModel/UG/generic//generic
$(TOP)//boost_iostreams
$(TOP)//boost_filesystem
$(TOP)//boost_program_options
$(TOP)/moses/TranslationModel/UG/mm//mm
$(TOP)/moses/TranslationModel/UG//mmsapt
@ -59,6 +64,7 @@ sim-pe.cc
$(TOP)/moses//moses
$(TOP)/moses/TranslationModel/UG/generic//generic
$(TOP)//boost_iostreams
$(TOP)//boost_filesystem
$(TOP)//boost_program_options
$(TOP)/moses/TranslationModel/UG/mm//mm
$(TOP)/moses/TranslationModel/UG//mmsapt
@ -70,6 +76,7 @@ spe-check-coverage.cc
$(TOP)/moses//moses
$(TOP)/moses/TranslationModel/UG/generic//generic
$(TOP)//boost_iostreams
$(TOP)//boost_filesystem
$(TOP)//boost_program_options
$(TOP)/moses/TranslationModel/UG/mm//mm
$(TOP)/moses/TranslationModel/UG//mmsapt
@ -81,6 +88,7 @@ spe-check-coverage2.cc
$(TOP)/moses//moses
$(TOP)/moses/TranslationModel/UG/generic//generic
$(TOP)//boost_iostreams
$(TOP)//boost_filesystem
$(TOP)//boost_program_options
$(TOP)/moses/TranslationModel/UG/mm//mm
$(TOP)/moses/TranslationModel/UG//mmsapt
@ -92,6 +100,7 @@ spe-check-coverage3.cc
$(TOP)/moses//moses
$(TOP)/moses/TranslationModel/UG/generic//generic
$(TOP)//boost_iostreams
$(TOP)//boost_filesystem
$(TOP)//boost_program_options
$(TOP)/moses/TranslationModel/UG/mm//mm
$(TOP)/moses/TranslationModel/UG//mmsapt

23
moses/TranslationTask.cpp
View File

@ -20,20 +20,10 @@ using namespace std;
namespace Moses
{
TranslationTask::TranslationTask(InputType* source, Moses::IOWrapper &ioWrapper,
boost::shared_ptr<HypergraphOutput<Manager> > hypergraphOutput)
TranslationTask::TranslationTask(InputType* source, Moses::IOWrapper &ioWrapper, int pbOrChart)
: m_source(source)
, m_ioWrapper(ioWrapper)
, m_hypergraphOutput(hypergraphOutput)
, m_pbOrChart(1)
{}
TranslationTask::TranslationTask(InputType *source, IOWrapper &ioWrapper,
boost::shared_ptr<HypergraphOutput<ChartManager> > hypergraphOutputChart)
: m_source(source)
, m_ioWrapper(ioWrapper)
, m_hypergraphOutputChart(hypergraphOutputChart)
, m_pbOrChart(2)
, m_pbOrChart(pbOrChart)
{}
TranslationTask::~TranslationTask() {
@ -96,9 +86,7 @@ void TranslationTask::RunPb()
manager.OutputSearchGraphSLF();
// Output search graph in hypergraph format for Kenneth Heafield's lazy hypergraph decoder
if (m_hypergraphOutput.get()) {
m_hypergraphOutput->Write(manager);
}
manager.OutputSearchGraphHypergraph();
additionalReportingTime.stop();
@ -302,10 +290,7 @@ void TranslationTask::RunChart()
UTIL_THROW_IF2(staticData.UseMBR(), "Cannot use MBR");
// Output search graph in hypergraph format for Kenneth Heafield's lazy hypergraph decoder
if (m_hypergraphOutputChart.get()) {
m_hypergraphOutputChart->Write(manager);
}
manager.OutputSearchGraphHypergraph();
// 1-best
const ChartHypothesis *bestHypo = manager.GetBestHypothesis();

9
moses/TranslationTask.h
View File

@ -26,11 +26,7 @@ class TranslationTask : public Moses::Task
public:
TranslationTask(Moses::InputType* source, Moses::IOWrapper &ioWrapper,
boost::shared_ptr<Moses::HypergraphOutput<Moses::Manager> > hypergraphOutput);
TranslationTask(Moses::InputType *source, IOWrapper &ioWrapper,
boost::shared_ptr<Moses::HypergraphOutput<Moses::ChartManager> > hypergraphOutputChart);
TranslationTask(Moses::InputType* source, Moses::IOWrapper &ioWrapper, int pbOrChart);
~TranslationTask();
@ -44,9 +40,6 @@ private:
Moses::InputType* m_source;
Moses::IOWrapper &m_ioWrapper;
boost::shared_ptr<Moses::HypergraphOutput<Moses::Manager> > m_hypergraphOutput;
boost::shared_ptr<Moses::HypergraphOutput<Moses::ChartManager> > m_hypergraphOutputChart;
void RunPb();
void RunChart();

2
phrase-extract/Jamfile
View File

@ -4,7 +4,7 @@ for local d in $(most-deps) {
obj $(d:B).o : $(d) ;
}
#and stuff them into an alias.
alias deps : $(most-deps:B).o ..//z ..//boost_iostreams ../moses//moses ../moses//ThreadPool ../moses//Util ../util//kenutil ;
alias deps : $(most-deps:B).o ..//z ..//boost_iostreams ..//boost_filesystem ../moses//moses ../moses//ThreadPool ../moses//Util ../util//kenutil ;
#ExtractionPhrasePair.cpp requires that main define some global variables.
#Build the mains that do not need these global variables.