diff --git a/cruise-control/example.config b/cruise-control/example.config index 203dbb0b0..dd756329a 100644 --- a/cruise-control/example.config +++ b/cruise-control/example.config @@ -1,21 +1,25 @@ # the tmp directory -MCC_TEMPDIR="/tmp/" +MCC_TEMPDIR="/disk4/cruise-control/cruise-control/tmp/" # where logs are saved -MCC_LOGDIR="/home/cruise/" - -# where moses gets checked out, tested etc. -MCC_WORKDIR="/home/cruise/work/" +MCC_LOGDIR="/disk4/cruise-control/cruise-control/" # repository that will be checked out MCC_GITREPO="https://github.com/moses-smt/mosesdecoder.git" # arguments passed to Moses configure -MCC_CONFIGURE_ARGS=" --with-srilm=/path/to/srilm --with-irstlm=/path/to/irstlm " +MCC_CONFIGURE_ARGS=" --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " + +ALTERNATIVE_CONFIGURE_ARGS=( + " --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " + " --with-srilm=/disk4/cruise-control/srilm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " + " --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " + " --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " + " --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --no-xmlrpc-c -j8 " + ) # list of branches to be checked -MCC_SCAN_BRANCHES="master" - -# regression test data archive (optional, this is the default) -MCC_REGTEST_ARCHIVE="http://www.statmt.org/moses/reg-testing/moses-reg-test-data-8.tgz" +MCC_SCAN_BRANCHES="origin/master" +# run full training/eval pipeline using EMS? +MCC_RUN_EMS="yes" diff --git a/cruise-control/test_all_new_commits.sh b/cruise-control/test_all_new_commits.sh index c30afa0d1..11d71c62d 100755 --- a/cruise-control/test_all_new_commits.sh +++ b/cruise-control/test_all_new_commits.sh @@ -116,8 +116,16 @@ function run_single_test () { #regtest_dir=$PWD/$(basename $regtest_file .tgz) cd .. + # test build with different configurations + echo "## test build with different configurations" >> $longlog + for configArgs in "${ALTERNATIVE_CONFIGURE_ARGS[@]}" + do + echo "building with args: $configArgs" >> $longlog + ./bjam clean -a $configArgs >> $longlog 2>&1 || warn "bjam clean failed, suspicious" + done + echo "## ./bjam clean" >> $longlog - ./bjam clean $MCC_CONFIGURE_ARGS --with-regtest=$regtest_dir >> $longlog 2>&1 || warn "bjam clean failed, suspicious" + ./bjam clean -a $MCC_CONFIGURE_ARGS --with-regtest=$regtest_dir >> $longlog 2>&1 || warn "bjam clean failed, suspicious" echo "## ./bjam $MCC_CONFIGURE_ARGS" >> $longlog if [ -z "$err" ]; then @@ -169,7 +177,7 @@ function run_single_test () { echo "## Status: $status" >> $longlog nicedate=$(date +"%Y%m%d-%H%M%S") - echo "$commit $status $configname $ccversion $nicedate" \ + echo "$commit$status$configname$ccversion$nicedate" \ >> "$LOGDIR/brief.log" if [ -z "$err" ]; then diff --git a/moses/BaseManager.cpp b/moses/BaseManager.cpp index b4fdee865..a804081af 100644 --- a/moses/BaseManager.cpp +++ b/moses/BaseManager.cpp @@ -101,7 +101,7 @@ OutputSurface(std::ostream &out, Phrase const& phrase) const std::vector const& factor_order = options()->output.factor_order; bool markUnknown = options()->unk.mark; - std::string const& fd = options()->output.FactorDelimiter; + std::string const& fd = options()->output.factor_delimiter; size_t size = phrase.GetSize(); for (size_t pos = 0 ; pos < size ; pos++) { diff --git a/moses/ChartManager.cpp b/moses/ChartManager.cpp index 374169e30..b7920b614 100644 --- a/moses/ChartManager.cpp +++ b/moses/ChartManager.cpp @@ -34,6 +34,7 @@ #include "moses/OutputCollector.h" #include "moses/ChartKBestExtractor.h" #include "moses/HypergraphOutput.h" +#include "moses/TranslationTask.h" using namespace std; @@ -50,7 +51,7 @@ ChartManager::ChartManager(ttasksptr const& ttask) , m_start(clock()) , m_hypothesisId(0) , m_parser(ttask, m_hypoStackColl) - , m_translationOptionList(StaticData::Instance().GetRuleLimit(), m_source) + , m_translationOptionList(ttask->options()->syntax.rule_limit, m_source) { } ChartManager::~ChartManager() @@ -297,8 +298,8 @@ OutputSearchGraphAsHypergraph(std::ostream& out) const void ChartManager::OutputSearchGraphMoses(std::ostream &outputSearchGraphStream) const { - ChartSearchGraphWriterMoses writer(options(), &outputSearchGraphStream, - m_source.GetTranslationId()); + ChartSearchGraphWriterMoses writer(options(), &outputSearchGraphStream, + m_source.GetTranslationId()); WriteSearchGraph(writer); } diff --git a/moses/ChartParser.cpp b/moses/ChartParser.cpp index 60899c02d..a611835fa 100644 --- a/moses/ChartParser.cpp +++ b/moses/ChartParser.cpp @@ -237,7 +237,10 @@ void ChartParser::CreateInputPaths(const InputType &input) m_inputPathMatrix.resize(size); UTIL_THROW_IF2(input.GetType() != SentenceInput && input.GetType() != TreeInputType, - "Input must be a sentence or a tree, not lattice or confusion networks"); + "Input must be a sentence or a tree, " << + "not lattice or confusion networks"); + + TranslationTask const* ttask = m_ttask.lock().get(); for (size_t phaseSize = 1; phaseSize <= size; ++phaseSize) { for (size_t startPos = 0; startPos < size - phaseSize + 1; ++startPos) { size_t endPos = startPos + phaseSize -1; @@ -249,11 +252,11 @@ void ChartParser::CreateInputPaths(const InputType &input) InputPath *node; if (range.GetNumWordsCovered() == 1) { - node = new InputPath(m_ttask, subphrase, labels, range, NULL, NULL); + node = new InputPath(ttask, subphrase, labels, range, NULL, NULL); vec.push_back(node); } else { const InputPath &prevNode = GetInputPath(startPos, endPos - 1); - node = new InputPath(m_ttask, subphrase, labels, range, &prevNode, NULL); + node = new InputPath(ttask, subphrase, labels, range, &prevNode, NULL); vec.push_back(node); } diff --git a/moses/ChartTranslationOptionList.cpp b/moses/ChartTranslationOptionList.cpp index c9f88e494..169497f40 100644 --- a/moses/ChartTranslationOptionList.cpp +++ b/moses/ChartTranslationOptionList.cpp @@ -158,8 +158,8 @@ void ChartTranslationOptionList::ApplyThreshold(float const threshold) scoreThreshold += threshold; // StaticData::Instance().GetTranslationOptionThreshold(); CollType::iterator bound = std::partition(m_collection.begin(), - m_collection.begin()+m_size, - ScoreThresholdPred(scoreThreshold)); + m_collection.begin()+m_size, + ScoreThresholdPred(scoreThreshold)); m_size = std::distance(m_collection.begin(), bound); } diff --git a/moses/ChartTranslationOptions.cpp b/moses/ChartTranslationOptions.cpp index b02a694ba..2a589999e 100644 --- a/moses/ChartTranslationOptions.cpp +++ b/moses/ChartTranslationOptions.cpp @@ -68,7 +68,7 @@ void ChartTranslationOptions::EvaluateWithSourceContext(const InputType &input, { SetInputPath(&inputPath); // if (StaticData::Instance().GetPlaceholderFactor() != NOT_FOUND) { - if (inputPath.ttask.lock()->options()->input.placeholder_factor != NOT_FOUND) { + if (inputPath.ttask->options()->input.placeholder_factor != NOT_FOUND) { CreateSourceRuleFromInputPath(); } diff --git a/moses/ConfusionNet.cpp b/moses/ConfusionNet.cpp index c8a05ca3c..86ea9b32b 100644 --- a/moses/ConfusionNet.cpp +++ b/moses/ConfusionNet.cpp @@ -283,7 +283,7 @@ CreateTranslationOptionCollection(ttasksptr const& ttask) const // float translationOptionThreshold // = ttask->options()->search.trans_opt_threshold; TranslationOptionCollection *rv - = new TranslationOptionCollectionConfusionNet(ttask, *this); + = new TranslationOptionCollectionConfusionNet(ttask, *this); //, maxNoTransOptPerCoverage, translationOptionThreshold); assert(rv); return rv; diff --git a/moses/ContextScope.h b/moses/ContextScope.h index d98b96086..7c24f585c 100644 --- a/moses/ContextScope.h +++ b/moses/ContextScope.h @@ -36,7 +36,7 @@ protected: #endif SPTR const> m_context_weights; public: - + typedef boost::shared_ptr ptr; template boost::shared_ptr const& set(void const* const key, boost::shared_ptr const& val) { diff --git a/moses/FF/BleuScoreFeature.h b/moses/FF/BleuScoreFeature.h index 50976cd2c..074ad0c45 100644 --- a/moses/FF/BleuScoreFeature.h +++ b/moses/FF/BleuScoreFeature.h @@ -144,7 +144,7 @@ public: return m_avg_input_length; } - void Load(AllOptions::ptr const& opts); + void Load(AllOptions::ptr const& opts); private: static std::vector s_staticColl; diff --git a/moses/FF/ConstrainedDecoding.cpp b/moses/FF/ConstrainedDecoding.cpp index 823b08b89..2eefbc444 100644 --- a/moses/FF/ConstrainedDecoding.cpp +++ b/moses/FF/ConstrainedDecoding.cpp @@ -56,7 +56,7 @@ void ConstrainedDecoding::Load(AllOptions::ptr const& opts) for(size_t i = 0; i < m_paths.size(); ++i) { InputFileStream constraintFile(m_paths[i]); std::string line; - long sentenceID = opts->output.start_translation_id - 1 ; + long sentenceID = opts->output.start_translation_id - 1 ; while (getline(constraintFile, line)) { vector vecStr = Tokenize(line, "\t"); diff --git a/moses/FF/DistortionScoreProducer.cpp b/moses/FF/DistortionScoreProducer.cpp index 3082b83f0..0cc9dfe10 100644 --- a/moses/FF/DistortionScoreProducer.cpp +++ b/moses/FF/DistortionScoreProducer.cpp @@ -49,16 +49,16 @@ const FFState* DistortionScoreProducer::EmptyHypothesisState(const InputType &in NOT_FOUND); } -float +float DistortionScoreProducer:: CalculateDistortionScore(const Hypothesis& hypo, - const Range &prev, const Range &curr, const int FirstGap) + const Range &prev, const Range &curr, const int FirstGap) { // if(!StaticData::Instance().UseEarlyDistortionCost()) { if(!hypo.GetManager().options()->reordering.use_early_distortion_cost) { return - (float) hypo.GetInput().ComputeDistortionDistance(prev, curr); } // else { - + /* Pay distortion score as soon as possible, from Moore and Quirk MT Summit 2007 Definitions: S : current source range diff --git a/moses/FF/GlobalLexicalModel.cpp b/moses/FF/GlobalLexicalModel.cpp index 89f27f4a1..986b427b0 100644 --- a/moses/FF/GlobalLexicalModel.cpp +++ b/moses/FF/GlobalLexicalModel.cpp @@ -55,7 +55,9 @@ void GlobalLexicalModel::Load(AllOptions::ptr const& opts) { m_options = opts; FactorCollection &factorCollection = FactorCollection::Instance(); - const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter(); + const std::string& oFactorDelimiter = opts->output.factor_delimiter; + const std::string& iFactorDelimiter = opts->input.factor_delimiter; + VERBOSE(2, "Loading global lexical model from file " << m_filePath << endl); @@ -76,21 +78,23 @@ void GlobalLexicalModel::Load(AllOptions::ptr const& opts) // create the output word Word *outWord = new Word(); - vector factorString = Tokenize( token[0], factorDelimiter ); + vector factorString = Tokenize( token[0], oFactorDelimiter ); for (size_t i=0 ; i < m_outputFactorsVec.size() ; i++) { const FactorDirection& direction = Output; const FactorType& factorType = m_outputFactorsVec[i]; - const Factor* factor = factorCollection.AddFactor( direction, factorType, factorString[i] ); + const Factor* factor + = factorCollection.AddFactor( direction, factorType, factorString[i] ); outWord->SetFactor( factorType, factor ); } // create the input word Word *inWord = new Word(); - factorString = Tokenize( token[1], factorDelimiter ); + factorString = Tokenize( token[1], iFactorDelimiter ); for (size_t i=0 ; i < m_inputFactorsVec.size() ; i++) { const FactorDirection& direction = Input; const FactorType& factorType = m_inputFactorsVec[i]; - const Factor* factor = factorCollection.AddFactor( direction, factorType, factorString[i] ); + const Factor* factor + = factorCollection.AddFactor( direction, factorType, factorString[i] ); inWord->SetFactor( factorType, factor ); } diff --git a/moses/HypergraphOutput.h b/moses/HypergraphOutput.h index 58467d781..db3bd3496 100644 --- a/moses/HypergraphOutput.h +++ b/moses/HypergraphOutput.h @@ -73,8 +73,8 @@ public: class ChartSearchGraphWriterMoses : public virtual ChartSearchGraphWriter { public: - ChartSearchGraphWriterMoses(AllOptions::ptr const& opts, - std::ostream* out, size_t lineNumber) + ChartSearchGraphWriterMoses(AllOptions::ptr const& opts, + std::ostream* out, size_t lineNumber) : ChartSearchGraphWriter(opts), m_out(out), m_lineNumber(lineNumber) {} virtual void WriteHeader(size_t, size_t) const { /* do nothing */ @@ -91,7 +91,7 @@ private: class ChartSearchGraphWriterHypergraph : public virtual ChartSearchGraphWriter { public: - ChartSearchGraphWriterHypergraph(AllOptions::ptr const& opts, std::ostream* out) + ChartSearchGraphWriterHypergraph(AllOptions::ptr const& opts, std::ostream* out) : ChartSearchGraphWriter(opts), m_out(out), m_nodeId(0) { } virtual void WriteHeader(size_t winners, size_t losers) const; virtual void WriteHypos(const ChartHypothesisCollection& hypos, diff --git a/moses/InputPath.cpp b/moses/InputPath.cpp index 4368e6b8e..28a135b99 100644 --- a/moses/InputPath.cpp +++ b/moses/InputPath.cpp @@ -11,7 +11,7 @@ using namespace std; namespace Moses { InputPath:: -InputPath(ttaskwptr const theTask, +InputPath(TranslationTask const* theTask, Phrase const& phrase, NonTerminalSet const& sourceNonTerms, Range const& range, InputPath const *prevNode, diff --git a/moses/InputPath.h b/moses/InputPath.h index 3a1f5e435..75e05d789 100644 --- a/moses/InputPath.h +++ b/moses/InputPath.h @@ -40,7 +40,8 @@ public: TargetPhrases; public: - ttaskwptr const ttask; + // ttaskwptr const ttask; + TranslationTask const* ttask; protected: const InputPath *m_prevPath; Phrase m_phrase; @@ -65,7 +66,7 @@ public: , m_nextNode(NOT_FOUND) { } - InputPath(ttaskwptr const ttask, + InputPath(TranslationTask const* ttask, // ttaskwptr const ttask, Phrase const& phrase, NonTerminalSet const& sourceNonTerms, Range const& range, diff --git a/moses/LM/BackwardTest.cpp b/moses/LM/BackwardTest.cpp index ef45d6e60..9fb5282fa 100644 --- a/moses/LM/BackwardTest.cpp +++ b/moses/LM/BackwardTest.cpp @@ -123,7 +123,6 @@ public: Input, outputFactorOrder, "the", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 1 ); @@ -150,7 +149,6 @@ public: Input, outputFactorOrder, "the licenses", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 2 ); @@ -179,7 +177,6 @@ public: Input, outputFactorOrder, "the licenses for", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 3 ); @@ -206,7 +203,6 @@ public: Input, outputFactorOrder, "the licenses for most", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 4 ); @@ -252,7 +248,6 @@ public: Input, outputFactorOrder, "the", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 1 ); @@ -280,7 +275,6 @@ public: Input, outputFactorOrder, "licenses", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 1 ); @@ -308,7 +302,6 @@ public: Input, outputFactorOrder, "for", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 1 ); @@ -336,7 +329,6 @@ public: Input, outputFactorOrder, "most", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 1 ); diff --git a/moses/Manager.cpp b/moses/Manager.cpp index f4dc03584..b0a85bea5 100644 --- a/moses/Manager.cpp +++ b/moses/Manager.cpp @@ -1699,7 +1699,7 @@ OutputSurface(std::ostream &out, Hypothesis const& edge, bool const recursive) c } bool markUnknown = options()->unk.mark; - std::string const& fd = options()->output.FactorDelimiter; + std::string const& fd = options()->output.factor_delimiter; TargetPhrase const& phrase = edge.GetCurrTargetPhrase(); size_t size = phrase.GetSize(); diff --git a/moses/PDTAimp.cpp b/moses/PDTAimp.cpp index c4bd2c338..a13344aba 100644 --- a/moses/PDTAimp.cpp +++ b/moses/PDTAimp.cpp @@ -116,6 +116,7 @@ PDTAimp::GetTargetPhraseCollection(Phrase const &src) const // convert into TargetPhrases + std::string fd = m_obj->options()->output.factor_delimiter; for(size_t i=0; ifirst + , m_obj->options()->output.factor_delimiter , scores.transScore , scores.inputScores , NULL @@ -403,6 +406,7 @@ void PDTAimp::CacheSource(ConfusionNet const& src) void PDTAimp::CreateTargetPhrase(TargetPhrase& targetPhrase, StringTgtCand::Tokens const& factorStrings, + std::string const& factorDelimiter, Scores const& transVector, Scores const& inputVector, const std::string *alignmentString, @@ -411,7 +415,8 @@ void PDTAimp::CreateTargetPhrase(TargetPhrase& targetPhrase, FactorCollection &factorCollection = FactorCollection::Instance(); for(size_t k=0; k word(*factorStrings[k], StaticData::Instance().GetFactorDelimiter()); + util::TokenIter + word(*factorStrings[k], factorDelimiter); Word& w=targetPhrase.AddWord(); for(size_t l=0; lSetParameter(m_verboseLevel, "verbose", (size_t) 1); - - - m_parameter->SetParameter(m_outputUnknownsFile, "output-unknowns", ""); - - // m_parameter->SetParameter(m_startTranslationId, - // "start-translation-id", 0); - return true; } @@ -206,16 +198,10 @@ bool StaticData::LoadData(Parameter *parameter) // threading etc. if (!ini_performance_options()) return false; - // Compact phrase table and reordering model - // m_parameter->SetParameter(m_minphrMemory, "minphr-memory", false ); - // m_parameter->SetParameter(m_minlexrMemory, "minlexr-memory", false ); - - // S2T decoder - // FEATURE FUNCTION INITIALIZATION HAPPENS HERE =============================== // set class-specific default parameters -#if !defined WIN32 || defined __MINGW32__ || defined HAVE_CMPH +#if defined HAVE_CMPH LexicalReorderingTableCompact::SetStaticDefaultParameters(*parameter); PhraseDictionaryCompact::SetStaticDefaultParameters(*parameter); #endif @@ -322,8 +308,6 @@ void StaticData::LoadChartDecodingParameters() // source label overlap m_parameter->SetParameter(m_sourceLabelOverlap, "source-label-overlap", SourceLabelOverlapAdd); - m_parameter->SetParameter(m_ruleLimit, "rule-limit", - DEFAULT_MAX_TRANS_OPT_SIZE); } @@ -438,7 +422,7 @@ LoadDecodeGraphsOld(const vector &mappingVector, UTIL_THROW_IF2(decodeStep == NULL, "Null decode step"); if (m_decodeGraphs.size() < decodeGraphInd + 1) { DecodeGraph *decodeGraph; - if (is_syntax(m_options->search.algo)) { + if (is_syntax(m_options->search.algo)) { size_t maxChartSpan = (decodeGraphInd < maxChartSpans.size()) ? maxChartSpans[decodeGraphInd] : DEFAULT_MAX_CHART_SPAN; VERBOSE(1,"max-chart-span: " << maxChartSpans[decodeGraphInd] << endl); decodeGraph = new DecodeGraph(m_decodeGraphs.size(), maxChartSpan); @@ -506,7 +490,7 @@ void StaticData::LoadDecodeGraphsNew(const std::vector &mappingVect UTIL_THROW_IF2(decodeStep == NULL, "Null decode step"); if (m_decodeGraphs.size() < decodeGraphInd + 1) { DecodeGraph *decodeGraph; - if (is_syntax(m_options->search.algo)) { + if (is_syntax(m_options->search.algo)) { size_t maxChartSpan = (decodeGraphInd < maxChartSpans.size()) ? maxChartSpans[decodeGraphInd] : DEFAULT_MAX_CHART_SPAN; VERBOSE(1,"max-chart-span: " << maxChartSpans[decodeGraphInd] << endl); decodeGraph = new DecodeGraph(m_decodeGraphs.size(), maxChartSpan); @@ -619,7 +603,6 @@ void StaticData::LoadFeatureFunctions() m_requireSortingAfterSourceContext = true; } - // if (PhraseDictionary *ffCast = dynamic_cast(ff)) { if (dynamic_cast(ff)) { doLoad = false; } diff --git a/moses/StaticData.h b/moses/StaticData.h index e633d94aa..871b82641 100644 --- a/moses/StaticData.h +++ b/moses/StaticData.h @@ -104,12 +104,6 @@ protected: std::string m_outputUnknownsFile; //! output unknowns in this file - size_t m_ruleLimit; - - // Whether to load compact phrase table and reordering table into memory - bool m_minphrMemory; - bool m_minlexrMemory; - // Initial = 0 = can be used when creating poss trans // Other = 1 = used to calculate LM score once all steps have been processed Word m_inputDefaultNonTerminal, m_outputDefaultNonTerminal; @@ -146,7 +140,6 @@ protected: void NoCache(); - bool m_continuePartialTranslation; std::string m_binPath; // soft NT lookup for chart models @@ -252,17 +245,10 @@ public: return m_unknownLHS; } - size_t GetRuleLimit() const { - return m_ruleLimit; - } float GetRuleCountThreshold() const { return 999999; /* TODO wtf! */ } - bool ContinuePartialTranslation() const { - return m_continuePartialTranslation; - } - void ReLoadBleuScoreFeatureParameter(float weight); Parameter* GetParameter() { @@ -273,10 +259,6 @@ public: return m_threadCount; } - // long GetStartTranslationId() const { - // return m_startTranslationId; - // } - void SetExecPath(const std::string &path); const std::string &GetBinDirectory() const; diff --git a/moses/Syntax/F2S/Manager-inl.h b/moses/Syntax/F2S/Manager-inl.h index 29b1f2ba9..60c0ba25f 100644 --- a/moses/Syntax/F2S/Manager-inl.h +++ b/moses/Syntax/F2S/Manager-inl.h @@ -60,9 +60,9 @@ void Manager::Decode() const StaticData &staticData = StaticData::Instance(); // Get various pruning-related constants. - const std::size_t popLimit = staticData.options()->cube.pop_limit; - const std::size_t ruleLimit = staticData.GetRuleLimit(); - const std::size_t stackLimit = staticData.options()->search.stack_size; + const std::size_t popLimit = options()->cube.pop_limit; + const std::size_t ruleLimit = options()->syntax.rule_limit; + const std::size_t stackLimit = options()->search.stack_size; // Initialize the stacks. InitializeStacks(); diff --git a/moses/Syntax/S2T/Manager-inl.h b/moses/Syntax/S2T/Manager-inl.h index 6bfc4a61c..a21de6a94 100644 --- a/moses/Syntax/S2T/Manager-inl.h +++ b/moses/Syntax/S2T/Manager-inl.h @@ -163,9 +163,9 @@ void Manager::Decode() const StaticData &staticData = StaticData::Instance(); // Get various pruning-related constants. - const std::size_t popLimit = staticData.options()->cube.pop_limit; - const std::size_t ruleLimit = staticData.GetRuleLimit(); - const std::size_t stackLimit = staticData.options()->search.stack_size; + const std::size_t popLimit = options()->cube.pop_limit; + const std::size_t ruleLimit = options()->syntax.rule_limit; + const std::size_t stackLimit = options()->search.stack_size; // Initialise the PChart and SChart. InitializeCharts(); diff --git a/moses/Syntax/T2S/Manager-inl.h b/moses/Syntax/T2S/Manager-inl.h index ec97e76de..ee8b33b15 100644 --- a/moses/Syntax/T2S/Manager-inl.h +++ b/moses/Syntax/T2S/Manager-inl.h @@ -94,11 +94,11 @@ void Manager::InitializeStacks() template void Manager::Decode() { - const StaticData &staticData = StaticData::Instance(); + // const StaticData &staticData = StaticData::Instance(); // Get various pruning-related constants. const std::size_t popLimit = this->options()->cube.pop_limit; - const std::size_t ruleLimit = staticData.GetRuleLimit(); + const std::size_t ruleLimit = this->options()->syntax.rule_limit; const std::size_t stackLimit = this->options()->search.stack_size; // Initialize the stacks. @@ -215,8 +215,8 @@ void Manager::ExtractKBest( // than k. The k-best factor determines how much bigger the limit should be, // with 0 being 'unlimited.' This actually sets a large-ish limit in case // too many translations are identical. - const StaticData &staticData = StaticData::Instance(); - const std::size_t nBestFactor = staticData.options()->nbest.factor; + // const StaticData &staticData = StaticData::Instance(); + const std::size_t nBestFactor = this->options()->nbest.factor; std::size_t numDerivations = (nBestFactor == 0) ? k*1000 : k*nBestFactor; // Extract the derivations. diff --git a/moses/Syntax/T2S/RuleTrieLoader.cpp b/moses/Syntax/T2S/RuleTrieLoader.cpp index 026237128..8006799e0 100644 --- a/moses/Syntax/T2S/RuleTrieLoader.cpp +++ b/moses/Syntax/T2S/RuleTrieLoader.cpp @@ -44,9 +44,6 @@ bool RuleTrieLoader::Load(Moses::AllOptions const& opts, { PrintUserTime(std::string("Start loading text phrase table. Moses format")); - // const StaticData &staticData = StaticData::Instance(); - // const std::string &factorDelimiter = staticData.GetFactorDelimiter(); - std::size_t count = 0; std::ostream *progress = NULL; diff --git a/moses/TargetPhrase.cpp b/moses/TargetPhrase.cpp index 7cd3afc3b..f47684cda 100644 --- a/moses/TargetPhrase.cpp +++ b/moses/TargetPhrase.cpp @@ -53,7 +53,6 @@ TargetPhrase::TargetPhrase( std::string out_string, const PhraseDictionary *pt) const StaticData &staticData = StaticData::Instance(); // XXX should this really be InputFactorOrder??? CreateFromString(Output, staticData.options()->input.factor_order, out_string, - // staticData.GetFactorDelimiter(), // eliminated [UG] NULL); } diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemory.cpp b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemory.cpp index 55db20303..16e6cf668 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemory.cpp +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemory.cpp @@ -45,8 +45,8 @@ ChartRuleLookupManagerMemory::ChartRuleLookupManagerMemory( { size_t sourceSize = parser.GetSize(); - - m_completedRules.resize(sourceSize); + size_t ruleLimit = parser.options()->syntax.rule_limit; + m_completedRules.resize(sourceSize, CompletedRuleCollection(ruleLimit)); m_isSoftMatching = !m_softMatchingMap.empty(); } diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp index 4b1bdb03a..8bd790c03 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp @@ -45,8 +45,8 @@ ChartRuleLookupManagerMemoryPerSentence::ChartRuleLookupManagerMemoryPerSentence { size_t sourceSize = parser.GetSize(); - - m_completedRules.resize(sourceSize); + size_t ruleLimit = parser.options()->syntax.rule_limit; + m_completedRules.resize(sourceSize, CompletedRuleCollection(ruleLimit)); m_isSoftMatching = !m_softMatchingMap.empty(); } diff --git a/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.cpp b/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.cpp index 3a8fc8662..508ebc2d5 100644 --- a/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.cpp +++ b/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.cpp @@ -27,7 +27,8 @@ using namespace std; namespace Moses { -CompletedRuleCollection::CompletedRuleCollection() : m_ruleLimit(StaticData::Instance().GetRuleLimit()) +CompletedRuleCollection::CompletedRuleCollection(size_t rule_limit) + : m_ruleLimit(rule_limit) { m_scoreThreshold = numeric_limits::infinity(); } diff --git a/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.h b/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.h index 95823328f..2a139cd07 100644 --- a/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.h +++ b/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.h @@ -74,7 +74,7 @@ public: struct CompletedRuleCollection { public: - CompletedRuleCollection(); + CompletedRuleCollection(size_t rule_limit); ~CompletedRuleCollection(); CompletedRuleCollection(const CompletedRuleCollection &old) diff --git a/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h b/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h index 3e1e7222e..ce4f5b10e 100644 --- a/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h +++ b/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h @@ -84,9 +84,9 @@ public: void Load(std::string filePath); - static void + static void SetStaticDefaultParameters(Parameter const& param); - + }; } diff --git a/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp b/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp index 0c6277437..5333e9f86 100644 --- a/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp +++ b/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp @@ -70,8 +70,8 @@ void PhraseDictionaryCompact::Load(AllOptions::ptr const& opts) if (!FileExists(tFilePath)) throw runtime_error("Error: File " + tFilePath + " does not exist."); - m_phraseDecoder - = new PhraseDecoder(*this, &m_input, &m_output, m_numScoreComponents); + m_phraseDecoder + = new PhraseDecoder(*this, &m_input, &m_output, m_numScoreComponents); std::FILE* pFile = std::fopen(tFilePath.c_str() , "r"); @@ -155,7 +155,7 @@ PhraseDictionaryCompact:: delete m_phraseDecoder; } -void +void PhraseDictionaryCompact:: CacheForCleanup(TargetPhraseCollection::shared_ptr tpc) { @@ -164,12 +164,12 @@ CacheForCleanup(TargetPhraseCollection::shared_ptr tpc) m_sentenceCache->push_back(tpc); } -void +void PhraseDictionaryCompact:: -AddEquivPhrase(const Phrase &source, const TargetPhrase &targetPhrase) +AddEquivPhrase(const Phrase &source, const TargetPhrase &targetPhrase) { } -void +void PhraseDictionaryCompact:: CleanUpAfterSentenceProcessing(const InputType &source) { diff --git a/moses/TranslationModel/PhraseDictionary.cpp b/moses/TranslationModel/PhraseDictionary.cpp index 4bfe57b03..346934416 100644 --- a/moses/TranslationModel/PhraseDictionary.cpp +++ b/moses/TranslationModel/PhraseDictionary.cpp @@ -35,16 +35,6 @@ namespace Moses { std::vector PhraseDictionary::s_staticColl; -// CacheColl::~CacheColl() -// { -// // not needed any more since the switch to shared pointers -// // for (iterator iter = begin(); iter != end(); ++iter) { -// // std::pair &key = iter->second; -// // TargetPhraseCollection::shared_ptr tps = key.first; -// // delete tps; -// // } -// } - PhraseDictionary::PhraseDictionary(const std::string &line, bool registerNow) : DecodeFeature(line, registerNow) , m_tableLimit(20) // default @@ -82,8 +72,7 @@ GetTargetPhraseCollectionLEGACY(const Phrase& src) const ret.reset(new TargetPhraseCollection(*ret)); } cache[hash] = entry(ret, clock()); - } else { - // in cache. just use it + } else { // in cache. just use it iter->second.second = clock(); ret = iter->second.first; } @@ -175,31 +164,6 @@ GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const } } -// persistent cache handling -// saving presistent cache to disk -//void PhraseDictionary::SaveCache() const -//{ -// CacheColl &cache = GetCache(); -// for( std::map >::iterator iter, -// iter != cache.end(), -// iter++ ) { -// -// } -//} - -// loading persistent cache from disk -//void PhraseDictionary::LoadCache() const -//{ -// CacheColl &cache = GetCache(); -// std::map >::iterator iter; -// iter = cache.begin(); -// while( iter != cache.end() ) { -// std::map >::iterator iterRemove = iter++; -// delete iterRemove->second.first; -// cache.erase(iterRemove); -// } -//} - // reduce presistent cache by half of maximum size void PhraseDictionary::ReduceCache() const { @@ -233,7 +197,9 @@ void PhraseDictionary::ReduceCache() const << reduceCacheTime << " seconds." << std::endl); } -CacheColl &PhraseDictionary::GetCache() const +CacheColl & +PhraseDictionary:: +GetCache() const { CacheColl *cache; cache = m_cache.get(); diff --git a/moses/TranslationModel/PhraseDictionary.h b/moses/TranslationModel/PhraseDictionary.h index 351c6050a..33a128638 100644 --- a/moses/TranslationModel/PhraseDictionary.h +++ b/moses/TranslationModel/PhraseDictionary.h @@ -44,6 +44,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "moses/TargetPhraseCollection.h" #include "moses/InputPath.h" #include "moses/FF/DecodeFeature.h" +#include "moses/ContextScope.h" namespace Moses { @@ -58,15 +59,6 @@ class ChartParser; // typedef std::pair TPCollLastUse; typedef std::pair CacheCollEntry; typedef boost::unordered_map CacheColl; -// class CacheColl : public boost::unordered_map -// { -// // 1st = hash of source phrase/ address of phrase-table node -// // 2nd = all translations -// // 3rd = time of last access - -// public: -// ~CacheColl(); -// }; /** * Abstract base class for phrase dictionaries (tables). diff --git a/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.cpp b/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.cpp index cf051ff36..80bbd5b3d 100644 --- a/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.cpp +++ b/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.cpp @@ -330,7 +330,6 @@ void PhraseDictionaryDynamicCacheBased::ClearEntries(std::string sourcePhraseStr { VERBOSE(3,"PhraseDictionaryDynamicCacheBased::ClearEntries(std::string sourcePhraseString, std::string targetPhraseString)" << std::endl); const StaticData &staticData = StaticData::Instance(); - // const std::string& factorDelimiter = staticData.GetFactorDelimiter(); Phrase sourcePhrase(0); Phrase targetPhrase(0); @@ -428,7 +427,6 @@ void PhraseDictionaryDynamicCacheBased::ClearSource(std::vector ent { VERBOSE(3,"entries.size():|" << entries.size() << "|" << std::endl); const StaticData &staticData = StaticData::Instance(); - // const std::string& factorDelimiter = staticData.GetFactorDelimiter(); Phrase sourcePhrase(0); std::vector::iterator it; @@ -517,7 +515,6 @@ void PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, s { VERBOSE(3,"PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, std::string targetPhraseString, std::string ageString, std::string waString)" << std::endl); const StaticData &staticData = StaticData::Instance(); - // const std::string& factorDelimiter = staticData.GetFactorDelimiter(); Phrase sourcePhrase(0); TargetPhrase targetPhrase(0); diff --git a/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp b/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp index dbc99cbbf..217562f05 100644 --- a/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp +++ b/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp @@ -144,7 +144,10 @@ TargetPhraseCollection::shared_ptr PhraseDictionaryMultiModelCounts::GetTargetPh } -void PhraseDictionaryMultiModelCounts::CollectSufficientStats(const Phrase& src, vector &fs, map* allStats) const +void +PhraseDictionaryMultiModelCounts:: +CollectSufficientStats(const Phrase& src, vector &fs, + map* allStats) const //fill fs and allStats with statistics from models { for(size_t i = 0; i < m_numModels; ++i) { diff --git a/moses/TranslationModel/ProbingPT/ProbingPT.cpp b/moses/TranslationModel/ProbingPT/ProbingPT.cpp index aa7cc1efe..cbfd2c1a4 100644 --- a/moses/TranslationModel/ProbingPT/ProbingPT.cpp +++ b/moses/TranslationModel/ProbingPT/ProbingPT.cpp @@ -77,7 +77,7 @@ void ProbingPT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQue InputPath &inputPath = **iter; const Phrase &sourcePhrase = inputPath.GetPhrase(); - if (sourcePhrase.GetSize() > StaticData::Instance().options()->search.max_phrase_length) { + if (sourcePhrase.GetSize() > m_options->search.max_phrase_length) { continue; } diff --git a/moses/TranslationOptionCollectionConfusionNet.cpp b/moses/TranslationOptionCollectionConfusionNet.cpp index 0fa50b36c..e5be9ecab 100644 --- a/moses/TranslationOptionCollectionConfusionNet.cpp +++ b/moses/TranslationOptionCollectionConfusionNet.cpp @@ -20,11 +20,11 @@ namespace Moses /** constructor; just initialize the base class */ TranslationOptionCollectionConfusionNet:: -TranslationOptionCollectionConfusionNet(ttasksptr const& ttask, +TranslationOptionCollectionConfusionNet(ttasksptr const& ttask, const ConfusionNet &input) - // , size_t maxNoTransOptPerCoverage, float translationOptionThreshold) +// , size_t maxNoTransOptPerCoverage, float translationOptionThreshold) : TranslationOptionCollection(ttask,input)// - // , maxNoTransOptPerCoverage, translationOptionThreshold) + // , maxNoTransOptPerCoverage, translationOptionThreshold) { size_t maxNoTransOptPerCoverage = ttask->options()->search.max_trans_opt_per_cov; float translationOptionThreshold = ttask->options()->search.trans_opt_threshold; @@ -63,7 +63,8 @@ TranslationOptionCollectionConfusionNet(ttasksptr const& ttask, const ScorePair &scores = col[i].second; ScorePair *inputScore = new ScorePair(scores); - InputPath *path = new InputPath(ttask, subphrase, labels, range, NULL, inputScore); + InputPath* path = new InputPath(ttask.get(), subphrase, labels, + range, NULL, inputScore); list.push_back(path); m_inputPathQueue.push_back(path); @@ -114,7 +115,8 @@ TranslationOptionCollectionConfusionNet(ttasksptr const& ttask, ScorePair *inputScore = new ScorePair(*prevInputScore); inputScore->PlusEquals(scores); - InputPath *path = new InputPath(ttask, subphrase, labels, range, &prevPath, inputScore); + InputPath *path = new InputPath(ttask.get(), subphrase, labels, range, + &prevPath, inputScore); list.push_back(path); m_inputPathQueue.push_back(path); diff --git a/moses/TranslationOptionCollectionConfusionNet.h b/moses/TranslationOptionCollectionConfusionNet.h index 5fe09d8cf..3dc605ec7 100644 --- a/moses/TranslationOptionCollectionConfusionNet.h +++ b/moses/TranslationOptionCollectionConfusionNet.h @@ -36,7 +36,7 @@ protected: public: TranslationOptionCollectionConfusionNet - (ttasksptr const& ttask, const ConfusionNet &source); + (ttasksptr const& ttask, const ConfusionNet &source); // , size_t maxNoTransOptPerCoverage, float translationOptionThreshold); void ProcessUnknownWord(size_t sourcePos); diff --git a/moses/TranslationOptionCollectionLattice.cpp b/moses/TranslationOptionCollectionLattice.cpp index b0973aefe..6559ccdf9 100644 --- a/moses/TranslationOptionCollectionLattice.cpp +++ b/moses/TranslationOptionCollectionLattice.cpp @@ -22,9 +22,9 @@ namespace Moses TranslationOptionCollectionLattice ::TranslationOptionCollectionLattice ( ttasksptr const& ttask, const WordLattice &input) - // , size_t maxNoTransOptPerCoverage, float translationOptionThreshold) +// , size_t maxNoTransOptPerCoverage, float translationOptionThreshold) : TranslationOptionCollection(ttask, input)// - // , maxNoTransOptPerCoverage, translationOptionThreshold) + // , maxNoTransOptPerCoverage, translationOptionThreshold) { UTIL_THROW_IF2(StaticData::Instance().GetUseLegacyPT(), "Not for models using the legqacy binary phrase table"); @@ -65,7 +65,7 @@ TranslationOptionCollectionLattice ScorePair *inputScore = new ScorePair(scores); InputPath *path - = new InputPath(ttask, subphrase, labels, range, NULL, inputScore); + = new InputPath(ttask.get(), subphrase, labels, range, NULL, inputScore); path->SetNextNode(nextNode); m_inputPathQueue.push_back(path); diff --git a/moses/TranslationOptionCollectionText.cpp b/moses/TranslationOptionCollectionText.cpp index 27b3353a6..a249de0a1 100644 --- a/moses/TranslationOptionCollectionText.cpp +++ b/moses/TranslationOptionCollectionText.cpp @@ -35,14 +35,14 @@ namespace Moses /** constructor; just initialize the base class */ TranslationOptionCollectionText:: TranslationOptionCollectionText(ttasksptr const& ttask, Sentence const &input) - //, size_t maxNoTransOptPerCoverage, float translationOptionThreshold) +//, size_t maxNoTransOptPerCoverage, float translationOptionThreshold) : TranslationOptionCollection(ttask,input) - // , maxNoTransOptPerCoverage, translationOptionThreshold) + // , maxNoTransOptPerCoverage, translationOptionThreshold) { size_t maxNoTransOptPerCoverage - = ttask->options()->search.max_trans_opt_per_cov; + = ttask->options()->search.max_trans_opt_per_cov; float translationOptionThreshold - = ttask->options()->search.trans_opt_threshold; + = ttask->options()->search.trans_opt_threshold; size_t size = input.GetSize(); m_inputPathMatrix.resize(size); for (size_t phaseSize = 1; phaseSize <= size; ++phaseSize) { @@ -56,11 +56,11 @@ TranslationOptionCollectionText(ttasksptr const& ttask, Sentence const &input) InputPath *path; if (range.GetNumWordsCovered() == 1) { - path = new InputPath(ttask, subphrase, labels, range, NULL, NULL); + path = new InputPath(ttask.get(), subphrase, labels, range, NULL, NULL); vec.push_back(path); } else { const InputPath &prevPath = GetInputPath(startPos, endPos - 1); - path = new InputPath(ttask, subphrase, labels, range, &prevPath, NULL); + path = new InputPath(ttask.get(), subphrase, labels, range, &prevPath, NULL); vec.push_back(path); } diff --git a/moses/Word.cpp b/moses/Word.cpp index a1c852f7c..d826cbd32 100644 --- a/moses/Word.cpp +++ b/moses/Word.cpp @@ -213,7 +213,8 @@ TO_STRING_BODY(Word); ostream& operator<<(ostream& out, const Word& word) { util::StringStream strme; - const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter(); + const std::string& factorDelimiter + = StaticData::Instance().options()->output.factor_delimiter; bool firstPass = true; unsigned int stop = max_fax(); for (unsigned int currFactor = 0 ; currFactor < stop; currFactor++) { diff --git a/moses/parameters/InputOptions.cpp b/moses/parameters/InputOptions.cpp index 98735ea66..dbdb2f60e 100644 --- a/moses/parameters/InputOptions.cpp +++ b/moses/parameters/InputOptions.cpp @@ -17,6 +17,7 @@ namespace Moses { xml_brackets.first = "<"; xml_brackets.second = ">"; factor_order.assign(1,0); + factor_delimiter = "|"; } bool @@ -76,6 +77,7 @@ namespace Moses { if (factor_order.empty()) factor_order.assign(1,0); param.SetParameter(placeholder_factor, "placeholder-factor", NOT_FOUND); + param.SetParameter(factor_delimiter, "factor-delimiter", "|"); param.SetParameter(input_file_path,"input-file",""); return true; diff --git a/moses/parameters/InputOptions.h b/moses/parameters/InputOptions.h index c767d9e98..572ba349a 100644 --- a/moses/parameters/InputOptions.h +++ b/moses/parameters/InputOptions.h @@ -14,10 +14,9 @@ namespace Moses InputTypeEnum input_type; XmlInputType xml_policy; // pass through, ignore, exclusive, inclusive std::vector factor_order; // input factor order - + std::string factor_delimiter; FactorType placeholder_factor; // where to store original text for placeholders std::string input_file_path; - std::pair xml_brackets; // strings to use as XML tags' opening and closing brackets. // Default are "<" and ">" diff --git a/moses/parameters/ReportingOptions.cpp b/moses/parameters/ReportingOptions.cpp index f1cb89b06..d7d3cc64b 100644 --- a/moses/parameters/ReportingOptions.cpp +++ b/moses/parameters/ReportingOptions.cpp @@ -24,6 +24,7 @@ namespace Moses { , lattice_sample_size(0) { factor_order.assign(1,0); + factor_delimiter = "|"; } bool @@ -94,14 +95,14 @@ namespace Moses { params= param.GetParam("output-factors"); if (params) factor_order = Scan(*params); if (factor_order.empty()) factor_order.assign(1,0); - + if (ReportAllFactors) { for (size_t i = 1; i < MAX_NUM_FACTORS; ++i) factor_order.push_back(i); } - param.SetParameter(FactorDelimiter, "factor-delimiter", std::string("|")); - param.SetParameter(FactorDelimiter, "output-factor-delimiter", FactorDelimiter); + param.SetParameter(factor_delimiter, "factor-delimiter", std::string("|")); + param.SetParameter(factor_delimiter, "output-factor-delimiter", factor_delimiter); return true; } @@ -134,12 +135,12 @@ namespace Moses { m = param.find("factor-delimiter"); if (m != param.end()) { - FactorDelimiter = Trim(xmlrpc_c::value_string(m->second)); + factor_delimiter = Trim(xmlrpc_c::value_string(m->second)); } m = param.find("output-factor-delimiter"); if (m != param.end()) { - FactorDelimiter = Trim(xmlrpc_c::value_string(m->second)); + factor_delimiter = Trim(xmlrpc_c::value_string(m->second)); } return true; diff --git a/moses/parameters/ReportingOptions.h b/moses/parameters/ReportingOptions.h index 76f8fdae4..92a0f1489 100644 --- a/moses/parameters/ReportingOptions.h +++ b/moses/parameters/ReportingOptions.h @@ -13,6 +13,7 @@ namespace Moses long start_translation_id; std::vector factor_order; + std::string factor_delimiter; bool ReportAllFactors; // m_reportAllFactors; int ReportSegmentation; // 0: no 1: m_reportSegmentation 2: ..._enriched @@ -23,7 +24,6 @@ namespace Moses WordAlignmentSort WA_SortOrder; // 0: no, 1: target order std::string AlignmentOutputFile; - std::string FactorDelimiter; bool WordGraph;