From 7ae38d8e0c1c13e47e337e1705ca4e4959820127 Mon Sep 17 00:00:00 2001 From: Ales Tamchyna Date: Wed, 26 Oct 2011 20:19:12 +0100 Subject: [PATCH 01/17] use new data for regression tests --- cruise-control/test_all_new_commits.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cruise-control/test_all_new_commits.sh b/cruise-control/test_all_new_commits.sh index 4103563ce..eec6d9d4b 100755 --- a/cruise-control/test_all_new_commits.sh +++ b/cruise-control/test_all_new_commits.sh @@ -44,7 +44,7 @@ GITREPO="$MCC_GITREPO" # location of moses regression test data archive (assumes url at the moment) REGTEST_ARCHIVE="$MCC_REGTEST_ARCHIVE" [ -n "$REGTEST_ARCHIVE" ] \ - || REGTEST_ARCHIVE="http://www.statmt.org/moses/reg-testing/moses-reg-test-data-8.tgz" + || REGTEST_ARCHIVE="http://www.statmt.org/moses/reg-testing/moses-reg-test-data-9.tgz" if [ ! -d "$WORKDIR" ]; then mkdir "$WORKDIR" || die "Failed to create workdir $WORKDIR" From 0b3c7da21b74831afe4c542ce23f92287996b0d5 Mon Sep 17 00:00:00 2001 From: Ales Tamchyna Date: Tue, 11 Sep 2012 09:30:54 +0100 Subject: [PATCH 02/17] update to new regtest location --- cruise-control/test_all_new_commits.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cruise-control/test_all_new_commits.sh b/cruise-control/test_all_new_commits.sh index d064ff0df..93ef30cf1 100755 --- a/cruise-control/test_all_new_commits.sh +++ b/cruise-control/test_all_new_commits.sh @@ -44,7 +44,7 @@ GITREPO="$MCC_GITREPO" # location of moses regression test data archive (assumes url at the moment) REGTEST_ARCHIVE="$MCC_REGTEST_ARCHIVE" [ -n "$REGTEST_ARCHIVE" ] \ - || REGTEST_ARCHIVE="git://github.com/hieuhoang/moses-reg-test-data.git" + || REGTEST_ARCHIVE="git://github.com/moses-smt/moses-regression-tests.git" if [ ! -d "$WORKDIR" ]; then mkdir "$WORKDIR" || die "Failed to create workdir $WORKDIR" From 47c7e1cad12cb5a42d4f2d2b098d58a9d2c83413 Mon Sep 17 00:00:00 2001 From: Ales Tamchyna Date: Tue, 11 Sep 2012 09:32:55 +0100 Subject: [PATCH 03/17] revert --- cruise-control/test_all_new_commits.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cruise-control/test_all_new_commits.sh b/cruise-control/test_all_new_commits.sh index 93ef30cf1..d064ff0df 100755 --- a/cruise-control/test_all_new_commits.sh +++ b/cruise-control/test_all_new_commits.sh @@ -44,7 +44,7 @@ GITREPO="$MCC_GITREPO" # location of moses regression test data archive (assumes url at the moment) REGTEST_ARCHIVE="$MCC_REGTEST_ARCHIVE" [ -n "$REGTEST_ARCHIVE" ] \ - || REGTEST_ARCHIVE="git://github.com/moses-smt/moses-regression-tests.git" + || REGTEST_ARCHIVE="git://github.com/hieuhoang/moses-reg-test-data.git" if [ ! -d "$WORKDIR" ]; then mkdir "$WORKDIR" || die "Failed to create workdir $WORKDIR" From fb667918e50dd22dd81378fb6f5a89440e5a18ba Mon Sep 17 00:00:00 2001 From: MosesAdmin Date: Fri, 27 Nov 2015 00:00:04 +0000 Subject: [PATCH 04/17] daily automatic beautifier --- cruise-control/test_all_new_commits.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/cruise-control/test_all_new_commits.sh b/cruise-control/test_all_new_commits.sh index 93ef30cf1..cf9ec7d01 100755 --- a/cruise-control/test_all_new_commits.sh +++ b/cruise-control/test_all_new_commits.sh @@ -17,6 +17,15 @@ configname=$(basename $configf | sed 's/\.config$//') source "$configf" +# beautifier +git clone git@github.com:moses-smt/mosesdecoder.git /tmp/moses +cd /tmp/moses +./scripts/other/beautify.py --format --skip-perltidy +git commit -am "daily automatic beautifier" +git push +rm -rf /tmp/moses +cd - + [ -z "$MCC_SCAN_BRANCHES" ] \ && die "Bad config $configf; does not define MCC_SCAN_BRANCHES" @@ -107,7 +116,6 @@ function run_single_test () { #regtest_dir=$PWD/$(basename $regtest_file .tgz) cd .. - echo "## ./bjam clean" >> $longlog ./bjam clean $MCC_CONFIGURE_ARGS --with-regtest=$regtest_dir >> $longlog 2>&1 || warn "bjam clean failed, suspicious" @@ -115,7 +123,7 @@ function run_single_test () { if [ -z "$err" ]; then ./bjam $MCC_CONFIGURE_ARGS >> $longlog 2>&1 || err="bjam" fi - + echo "## regression tests" >> $longlog if [ -z "$err" ]; then ./bjam $MCC_CONFIGURE_ARGS --with-regtest=$regtest_dir >> $longlog 2>&1 || err="regression tests" @@ -155,12 +163,13 @@ function run_single_test () { if [ -z "$err" ]; then status="OK" else + git reset --hard HEAD status="FAIL:$err" fi echo "## Status: $status" >> $longlog - + nicedate=$(date +"%Y%m%d-%H%M%S") - echo "$commit $status $configname $ccversion $nicedate" \ + echo "$commit$status$configname$ccversion$nicedate" \ >> "$LOGDIR/brief.log" if [ -z "$err" ]; then @@ -180,14 +189,14 @@ done # create info files for new commits for i in $(git rev-list $MCC_SCAN_BRANCHES); do first_char=$(echo $i | grep -o '^.') - mkdir -p "$LOGDIR/logs/$configname/$first_char" + mkdir -p "$LOGDIR/logs/$configname/$first_char" [ -f "$LOGDIR/logs/$configname/$first_char/$i.info" ] && break; git show $i | $MYDIR/shorten_info.pl > "$LOGDIR/logs/$configname/$first_char/$i.info" done #### Main loop over all commits for i in $MCC_SCAN_BRANCHES; do - warn "On brach $i" + warn "On branch $i" git rev-list $i \ | while read commit; do first_char=$(echo $commit | grep -o '^.') From b0d8c397bb698d411108d3696df774d5cc395a28 Mon Sep 17 00:00:00 2001 From: MosesAdmin Date: Sat, 12 Dec 2015 18:04:13 +0000 Subject: [PATCH 05/17] daily automatic beautifier --- moses/ChartManager.cpp | 4 ++-- moses/ChartTranslationOptionList.cpp | 4 ++-- moses/ConfusionNet.cpp | 2 +- moses/FF/BleuScoreFeature.h | 2 +- moses/FF/ConstrainedDecoding.cpp | 2 +- moses/FF/DistortionScoreProducer.cpp | 6 +++--- moses/HypergraphOutput.h | 6 +++--- moses/Sentence.cpp | 2 +- moses/StaticData.cpp | 4 ++-- .../CompactPT/LexicalReorderingTableCompact.h | 4 ++-- .../CompactPT/PhraseDictionaryCompact.cpp | 12 ++++++------ moses/TranslationOptionCollectionConfusionNet.cpp | 6 +++--- moses/TranslationOptionCollectionConfusionNet.h | 2 +- moses/TranslationOptionCollectionLattice.cpp | 4 ++-- moses/TranslationOptionCollectionText.cpp | 8 ++++---- 15 files changed, 34 insertions(+), 34 deletions(-) diff --git a/moses/ChartManager.cpp b/moses/ChartManager.cpp index ba58a010e..bed257e63 100644 --- a/moses/ChartManager.cpp +++ b/moses/ChartManager.cpp @@ -299,8 +299,8 @@ OutputSearchGraphAsHypergraph(std::ostream& out) const void ChartManager::OutputSearchGraphMoses(std::ostream &outputSearchGraphStream) const { - ChartSearchGraphWriterMoses writer(options(), &outputSearchGraphStream, - m_source.GetTranslationId()); + ChartSearchGraphWriterMoses writer(options(), &outputSearchGraphStream, + m_source.GetTranslationId()); WriteSearchGraph(writer); } diff --git a/moses/ChartTranslationOptionList.cpp b/moses/ChartTranslationOptionList.cpp index c9f88e494..169497f40 100644 --- a/moses/ChartTranslationOptionList.cpp +++ b/moses/ChartTranslationOptionList.cpp @@ -158,8 +158,8 @@ void ChartTranslationOptionList::ApplyThreshold(float const threshold) scoreThreshold += threshold; // StaticData::Instance().GetTranslationOptionThreshold(); CollType::iterator bound = std::partition(m_collection.begin(), - m_collection.begin()+m_size, - ScoreThresholdPred(scoreThreshold)); + m_collection.begin()+m_size, + ScoreThresholdPred(scoreThreshold)); m_size = std::distance(m_collection.begin(), bound); } diff --git a/moses/ConfusionNet.cpp b/moses/ConfusionNet.cpp index c8a05ca3c..86ea9b32b 100644 --- a/moses/ConfusionNet.cpp +++ b/moses/ConfusionNet.cpp @@ -283,7 +283,7 @@ CreateTranslationOptionCollection(ttasksptr const& ttask) const // float translationOptionThreshold // = ttask->options()->search.trans_opt_threshold; TranslationOptionCollection *rv - = new TranslationOptionCollectionConfusionNet(ttask, *this); + = new TranslationOptionCollectionConfusionNet(ttask, *this); //, maxNoTransOptPerCoverage, translationOptionThreshold); assert(rv); return rv; diff --git a/moses/FF/BleuScoreFeature.h b/moses/FF/BleuScoreFeature.h index 50976cd2c..074ad0c45 100644 --- a/moses/FF/BleuScoreFeature.h +++ b/moses/FF/BleuScoreFeature.h @@ -144,7 +144,7 @@ public: return m_avg_input_length; } - void Load(AllOptions::ptr const& opts); + void Load(AllOptions::ptr const& opts); private: static std::vector s_staticColl; diff --git a/moses/FF/ConstrainedDecoding.cpp b/moses/FF/ConstrainedDecoding.cpp index 823b08b89..2eefbc444 100644 --- a/moses/FF/ConstrainedDecoding.cpp +++ b/moses/FF/ConstrainedDecoding.cpp @@ -56,7 +56,7 @@ void ConstrainedDecoding::Load(AllOptions::ptr const& opts) for(size_t i = 0; i < m_paths.size(); ++i) { InputFileStream constraintFile(m_paths[i]); std::string line; - long sentenceID = opts->output.start_translation_id - 1 ; + long sentenceID = opts->output.start_translation_id - 1 ; while (getline(constraintFile, line)) { vector vecStr = Tokenize(line, "\t"); diff --git a/moses/FF/DistortionScoreProducer.cpp b/moses/FF/DistortionScoreProducer.cpp index 3082b83f0..0cc9dfe10 100644 --- a/moses/FF/DistortionScoreProducer.cpp +++ b/moses/FF/DistortionScoreProducer.cpp @@ -49,16 +49,16 @@ const FFState* DistortionScoreProducer::EmptyHypothesisState(const InputType &in NOT_FOUND); } -float +float DistortionScoreProducer:: CalculateDistortionScore(const Hypothesis& hypo, - const Range &prev, const Range &curr, const int FirstGap) + const Range &prev, const Range &curr, const int FirstGap) { // if(!StaticData::Instance().UseEarlyDistortionCost()) { if(!hypo.GetManager().options()->reordering.use_early_distortion_cost) { return - (float) hypo.GetInput().ComputeDistortionDistance(prev, curr); } // else { - + /* Pay distortion score as soon as possible, from Moore and Quirk MT Summit 2007 Definitions: S : current source range diff --git a/moses/HypergraphOutput.h b/moses/HypergraphOutput.h index 58467d781..db3bd3496 100644 --- a/moses/HypergraphOutput.h +++ b/moses/HypergraphOutput.h @@ -73,8 +73,8 @@ public: class ChartSearchGraphWriterMoses : public virtual ChartSearchGraphWriter { public: - ChartSearchGraphWriterMoses(AllOptions::ptr const& opts, - std::ostream* out, size_t lineNumber) + ChartSearchGraphWriterMoses(AllOptions::ptr const& opts, + std::ostream* out, size_t lineNumber) : ChartSearchGraphWriter(opts), m_out(out), m_lineNumber(lineNumber) {} virtual void WriteHeader(size_t, size_t) const { /* do nothing */ @@ -91,7 +91,7 @@ private: class ChartSearchGraphWriterHypergraph : public virtual ChartSearchGraphWriter { public: - ChartSearchGraphWriterHypergraph(AllOptions::ptr const& opts, std::ostream* out) + ChartSearchGraphWriterHypergraph(AllOptions::ptr const& opts, std::ostream* out) : ChartSearchGraphWriter(opts), m_out(out), m_nodeId(0) { } virtual void WriteHeader(size_t winners, size_t losers) const; virtual void WriteHypos(const ChartHypothesisCollection& hypos, diff --git a/moses/Sentence.cpp b/moses/Sentence.cpp index 200f26215..4db022e5e 100644 --- a/moses/Sentence.cpp +++ b/moses/Sentence.cpp @@ -257,7 +257,7 @@ Sentence:: CreateTranslationOptionCollection(ttasksptr const& ttask) const { TranslationOptionCollection *rv - = new TranslationOptionCollectionText(ttask, *this); + = new TranslationOptionCollectionText(ttask, *this); assert(rv); return rv; } diff --git a/moses/StaticData.cpp b/moses/StaticData.cpp index 51448891e..37407dc3a 100644 --- a/moses/StaticData.cpp +++ b/moses/StaticData.cpp @@ -440,7 +440,7 @@ LoadDecodeGraphsOld(const vector &mappingVector, UTIL_THROW_IF2(decodeStep == NULL, "Null decode step"); if (m_decodeGraphs.size() < decodeGraphInd + 1) { DecodeGraph *decodeGraph; - if (is_syntax(m_options->search.algo)) { + if (is_syntax(m_options->search.algo)) { size_t maxChartSpan = (decodeGraphInd < maxChartSpans.size()) ? maxChartSpans[decodeGraphInd] : DEFAULT_MAX_CHART_SPAN; VERBOSE(1,"max-chart-span: " << maxChartSpans[decodeGraphInd] << endl); decodeGraph = new DecodeGraph(m_decodeGraphs.size(), maxChartSpan); @@ -508,7 +508,7 @@ void StaticData::LoadDecodeGraphsNew(const std::vector &mappingVect UTIL_THROW_IF2(decodeStep == NULL, "Null decode step"); if (m_decodeGraphs.size() < decodeGraphInd + 1) { DecodeGraph *decodeGraph; - if (is_syntax(m_options->search.algo)) { + if (is_syntax(m_options->search.algo)) { size_t maxChartSpan = (decodeGraphInd < maxChartSpans.size()) ? maxChartSpans[decodeGraphInd] : DEFAULT_MAX_CHART_SPAN; VERBOSE(1,"max-chart-span: " << maxChartSpans[decodeGraphInd] << endl); decodeGraph = new DecodeGraph(m_decodeGraphs.size(), maxChartSpan); diff --git a/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h b/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h index 3e1e7222e..ce4f5b10e 100644 --- a/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h +++ b/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h @@ -84,9 +84,9 @@ public: void Load(std::string filePath); - static void + static void SetStaticDefaultParameters(Parameter const& param); - + }; } diff --git a/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp b/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp index 0c6277437..5333e9f86 100644 --- a/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp +++ b/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp @@ -70,8 +70,8 @@ void PhraseDictionaryCompact::Load(AllOptions::ptr const& opts) if (!FileExists(tFilePath)) throw runtime_error("Error: File " + tFilePath + " does not exist."); - m_phraseDecoder - = new PhraseDecoder(*this, &m_input, &m_output, m_numScoreComponents); + m_phraseDecoder + = new PhraseDecoder(*this, &m_input, &m_output, m_numScoreComponents); std::FILE* pFile = std::fopen(tFilePath.c_str() , "r"); @@ -155,7 +155,7 @@ PhraseDictionaryCompact:: delete m_phraseDecoder; } -void +void PhraseDictionaryCompact:: CacheForCleanup(TargetPhraseCollection::shared_ptr tpc) { @@ -164,12 +164,12 @@ CacheForCleanup(TargetPhraseCollection::shared_ptr tpc) m_sentenceCache->push_back(tpc); } -void +void PhraseDictionaryCompact:: -AddEquivPhrase(const Phrase &source, const TargetPhrase &targetPhrase) +AddEquivPhrase(const Phrase &source, const TargetPhrase &targetPhrase) { } -void +void PhraseDictionaryCompact:: CleanUpAfterSentenceProcessing(const InputType &source) { diff --git a/moses/TranslationOptionCollectionConfusionNet.cpp b/moses/TranslationOptionCollectionConfusionNet.cpp index 0fa50b36c..0d1eac9aa 100644 --- a/moses/TranslationOptionCollectionConfusionNet.cpp +++ b/moses/TranslationOptionCollectionConfusionNet.cpp @@ -20,11 +20,11 @@ namespace Moses /** constructor; just initialize the base class */ TranslationOptionCollectionConfusionNet:: -TranslationOptionCollectionConfusionNet(ttasksptr const& ttask, +TranslationOptionCollectionConfusionNet(ttasksptr const& ttask, const ConfusionNet &input) - // , size_t maxNoTransOptPerCoverage, float translationOptionThreshold) +// , size_t maxNoTransOptPerCoverage, float translationOptionThreshold) : TranslationOptionCollection(ttask,input)// - // , maxNoTransOptPerCoverage, translationOptionThreshold) + // , maxNoTransOptPerCoverage, translationOptionThreshold) { size_t maxNoTransOptPerCoverage = ttask->options()->search.max_trans_opt_per_cov; float translationOptionThreshold = ttask->options()->search.trans_opt_threshold; diff --git a/moses/TranslationOptionCollectionConfusionNet.h b/moses/TranslationOptionCollectionConfusionNet.h index 5fe09d8cf..3dc605ec7 100644 --- a/moses/TranslationOptionCollectionConfusionNet.h +++ b/moses/TranslationOptionCollectionConfusionNet.h @@ -36,7 +36,7 @@ protected: public: TranslationOptionCollectionConfusionNet - (ttasksptr const& ttask, const ConfusionNet &source); + (ttasksptr const& ttask, const ConfusionNet &source); // , size_t maxNoTransOptPerCoverage, float translationOptionThreshold); void ProcessUnknownWord(size_t sourcePos); diff --git a/moses/TranslationOptionCollectionLattice.cpp b/moses/TranslationOptionCollectionLattice.cpp index b0973aefe..fc2f84aae 100644 --- a/moses/TranslationOptionCollectionLattice.cpp +++ b/moses/TranslationOptionCollectionLattice.cpp @@ -22,9 +22,9 @@ namespace Moses TranslationOptionCollectionLattice ::TranslationOptionCollectionLattice ( ttasksptr const& ttask, const WordLattice &input) - // , size_t maxNoTransOptPerCoverage, float translationOptionThreshold) +// , size_t maxNoTransOptPerCoverage, float translationOptionThreshold) : TranslationOptionCollection(ttask, input)// - // , maxNoTransOptPerCoverage, translationOptionThreshold) + // , maxNoTransOptPerCoverage, translationOptionThreshold) { UTIL_THROW_IF2(StaticData::Instance().GetUseLegacyPT(), "Not for models using the legqacy binary phrase table"); diff --git a/moses/TranslationOptionCollectionText.cpp b/moses/TranslationOptionCollectionText.cpp index 27b3353a6..fd627b6d9 100644 --- a/moses/TranslationOptionCollectionText.cpp +++ b/moses/TranslationOptionCollectionText.cpp @@ -35,14 +35,14 @@ namespace Moses /** constructor; just initialize the base class */ TranslationOptionCollectionText:: TranslationOptionCollectionText(ttasksptr const& ttask, Sentence const &input) - //, size_t maxNoTransOptPerCoverage, float translationOptionThreshold) +//, size_t maxNoTransOptPerCoverage, float translationOptionThreshold) : TranslationOptionCollection(ttask,input) - // , maxNoTransOptPerCoverage, translationOptionThreshold) + // , maxNoTransOptPerCoverage, translationOptionThreshold) { size_t maxNoTransOptPerCoverage - = ttask->options()->search.max_trans_opt_per_cov; + = ttask->options()->search.max_trans_opt_per_cov; float translationOptionThreshold - = ttask->options()->search.trans_opt_threshold; + = ttask->options()->search.trans_opt_threshold; size_t size = input.GetSize(); m_inputPathMatrix.resize(size); for (size_t phaseSize = 1; phaseSize <= size; ++phaseSize) { From 485887528916b4b27432c4fd2edb03aafac76e49 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Sun, 13 Dec 2015 21:36:18 +0000 Subject: [PATCH 06/17] compile error with compact pt reported by Shaimaa Marzouk --- moses/StaticData.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moses/StaticData.cpp b/moses/StaticData.cpp index 51448891e..580d61559 100644 --- a/moses/StaticData.cpp +++ b/moses/StaticData.cpp @@ -53,7 +53,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #ifdef HAVE_CMPH #include "moses/TranslationModel/CompactPT/PhraseDictionaryCompact.h" #endif -#if !defined WIN32 || defined __MINGW32__ || defined HAVE_CMPH +#if defined HAVE_CMPH #include "moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h" #endif @@ -217,7 +217,7 @@ bool StaticData::LoadData(Parameter *parameter) // FEATURE FUNCTION INITIALIZATION HAPPENS HERE =============================== // set class-specific default parameters -#if !defined WIN32 || defined __MINGW32__ || defined HAVE_CMPH +#if defined HAVE_CMPH LexicalReorderingTableCompact::SetStaticDefaultParameters(*parameter); PhraseDictionaryCompact::SetStaticDefaultParameters(*parameter); #endif From bb6e0157aac67e72b4aacbb87405014da552e140 Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Sat, 12 Dec 2015 16:23:37 +0000 Subject: [PATCH 07/17] Code cleanup and refactoring. --- moses/BaseManager.cpp | 2 +- moses/ChartManager.cpp | 3 +- moses/ChartParser.cpp | 9 ++-- moses/ChartTranslationOptions.cpp | 2 +- moses/ContextScope.h | 2 +- moses/FF/GlobalLexicalModel.cpp | 14 ++++--- moses/InputPath.cpp | 2 +- moses/InputPath.h | 5 ++- moses/LM/BackwardTest.cpp | 8 ---- moses/Manager.cpp | 2 +- moses/PDTAimp.cpp | 9 +++- moses/PDTAimp.h | 1 + moses/Phrase.h | 12 ------ moses/StaticData.cpp | 17 -------- moses/StaticData.h | 18 -------- moses/Syntax/F2S/Manager-inl.h | 6 +-- moses/Syntax/S2T/Manager-inl.h | 6 +-- moses/Syntax/T2S/Manager-inl.h | 8 ++-- moses/Syntax/T2S/RuleTrieLoader.cpp | 3 -- moses/TargetPhrase.cpp | 1 - .../ChartRuleLookupManagerMemory.cpp | 4 +- ...hartRuleLookupManagerMemoryPerSentence.cpp | 4 +- .../CYKPlusParser/CompletedRuleCollection.cpp | 3 +- .../CYKPlusParser/CompletedRuleCollection.h | 2 +- moses/TranslationModel/PhraseDictionary.cpp | 42 ++----------------- moses/TranslationModel/PhraseDictionary.h | 10 +---- .../PhraseDictionaryDynamicCacheBased.cpp | 3 -- .../PhraseDictionaryMultiModelCounts.cpp | 5 ++- .../TranslationModel/ProbingPT/ProbingPT.cpp | 2 +- ...ranslationOptionCollectionConfusionNet.cpp | 8 ++-- moses/TranslationOptionCollectionLattice.cpp | 2 +- moses/TranslationOptionCollectionText.cpp | 4 +- moses/Word.cpp | 3 +- moses/parameters/InputOptions.cpp | 2 + moses/parameters/InputOptions.h | 3 +- moses/parameters/ReportingOptions.cpp | 11 ++--- moses/parameters/ReportingOptions.h | 2 +- 37 files changed, 80 insertions(+), 160 deletions(-) diff --git a/moses/BaseManager.cpp b/moses/BaseManager.cpp index b4fdee865..a804081af 100644 --- a/moses/BaseManager.cpp +++ b/moses/BaseManager.cpp @@ -101,7 +101,7 @@ OutputSurface(std::ostream &out, Phrase const& phrase) const std::vector const& factor_order = options()->output.factor_order; bool markUnknown = options()->unk.mark; - std::string const& fd = options()->output.FactorDelimiter; + std::string const& fd = options()->output.factor_delimiter; size_t size = phrase.GetSize(); for (size_t pos = 0 ; pos < size ; pos++) { diff --git a/moses/ChartManager.cpp b/moses/ChartManager.cpp index ba58a010e..55c8a1eeb 100644 --- a/moses/ChartManager.cpp +++ b/moses/ChartManager.cpp @@ -34,6 +34,7 @@ #include "moses/OutputCollector.h" #include "moses/ChartKBestExtractor.h" #include "moses/HypergraphOutput.h" +#include "moses/TranslationTask.h" using namespace std; @@ -52,7 +53,7 @@ ChartManager::ChartManager(ttasksptr const& ttask) , m_start(clock()) , m_hypothesisId(0) , m_parser(ttask, m_hypoStackColl) - , m_translationOptionList(StaticData::Instance().GetRuleLimit(), m_source) + , m_translationOptionList(ttask->options()->syntax.rule_limit, m_source) { } ChartManager::~ChartManager() diff --git a/moses/ChartParser.cpp b/moses/ChartParser.cpp index 60899c02d..15b18a7cc 100644 --- a/moses/ChartParser.cpp +++ b/moses/ChartParser.cpp @@ -237,7 +237,10 @@ void ChartParser::CreateInputPaths(const InputType &input) m_inputPathMatrix.resize(size); UTIL_THROW_IF2(input.GetType() != SentenceInput && input.GetType() != TreeInputType, - "Input must be a sentence or a tree, not lattice or confusion networks"); + "Input must be a sentence or a tree, " << + "not lattice or confusion networks"); + + TranslationTask const* ttask = m_ttask.lock().get(); for (size_t phaseSize = 1; phaseSize <= size; ++phaseSize) { for (size_t startPos = 0; startPos < size - phaseSize + 1; ++startPos) { size_t endPos = startPos + phaseSize -1; @@ -249,11 +252,11 @@ void ChartParser::CreateInputPaths(const InputType &input) InputPath *node; if (range.GetNumWordsCovered() == 1) { - node = new InputPath(m_ttask, subphrase, labels, range, NULL, NULL); + node = new InputPath(ttask, subphrase, labels, range, NULL, NULL); vec.push_back(node); } else { const InputPath &prevNode = GetInputPath(startPos, endPos - 1); - node = new InputPath(m_ttask, subphrase, labels, range, &prevNode, NULL); + node = new InputPath(ttask, subphrase, labels, range, &prevNode, NULL); vec.push_back(node); } diff --git a/moses/ChartTranslationOptions.cpp b/moses/ChartTranslationOptions.cpp index b02a694ba..2a589999e 100644 --- a/moses/ChartTranslationOptions.cpp +++ b/moses/ChartTranslationOptions.cpp @@ -68,7 +68,7 @@ void ChartTranslationOptions::EvaluateWithSourceContext(const InputType &input, { SetInputPath(&inputPath); // if (StaticData::Instance().GetPlaceholderFactor() != NOT_FOUND) { - if (inputPath.ttask.lock()->options()->input.placeholder_factor != NOT_FOUND) { + if (inputPath.ttask->options()->input.placeholder_factor != NOT_FOUND) { CreateSourceRuleFromInputPath(); } diff --git a/moses/ContextScope.h b/moses/ContextScope.h index d98b96086..7c24f585c 100644 --- a/moses/ContextScope.h +++ b/moses/ContextScope.h @@ -36,7 +36,7 @@ protected: #endif SPTR const> m_context_weights; public: - + typedef boost::shared_ptr ptr; template boost::shared_ptr const& set(void const* const key, boost::shared_ptr const& val) { diff --git a/moses/FF/GlobalLexicalModel.cpp b/moses/FF/GlobalLexicalModel.cpp index 89f27f4a1..f1a92eb47 100644 --- a/moses/FF/GlobalLexicalModel.cpp +++ b/moses/FF/GlobalLexicalModel.cpp @@ -55,7 +55,9 @@ void GlobalLexicalModel::Load(AllOptions::ptr const& opts) { m_options = opts; FactorCollection &factorCollection = FactorCollection::Instance(); - const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter(); + const std::string& oFactorDelimiter = opts->output.factor_delimiter; + const std::string& iFactorDelimiter = opts->input.factor_delimiter; + VERBOSE(2, "Loading global lexical model from file " << m_filePath << endl); @@ -76,21 +78,23 @@ void GlobalLexicalModel::Load(AllOptions::ptr const& opts) // create the output word Word *outWord = new Word(); - vector factorString = Tokenize( token[0], factorDelimiter ); + vector factorString = Tokenize( token[0], oFactorDelimiter ); for (size_t i=0 ; i < m_outputFactorsVec.size() ; i++) { const FactorDirection& direction = Output; const FactorType& factorType = m_outputFactorsVec[i]; - const Factor* factor = factorCollection.AddFactor( direction, factorType, factorString[i] ); + const Factor* factor + = factorCollection.AddFactor( direction, factorType, factorString[i] ); outWord->SetFactor( factorType, factor ); } // create the input word Word *inWord = new Word(); - factorString = Tokenize( token[1], factorDelimiter ); + factorString = Tokenize( token[1], iFactorDelimiter ); for (size_t i=0 ; i < m_inputFactorsVec.size() ; i++) { const FactorDirection& direction = Input; const FactorType& factorType = m_inputFactorsVec[i]; - const Factor* factor = factorCollection.AddFactor( direction, factorType, factorString[i] ); + const Factor* factor + = factorCollection.AddFactor( direction, factorType, factorString[i] ); inWord->SetFactor( factorType, factor ); } diff --git a/moses/InputPath.cpp b/moses/InputPath.cpp index 4368e6b8e..28a135b99 100644 --- a/moses/InputPath.cpp +++ b/moses/InputPath.cpp @@ -11,7 +11,7 @@ using namespace std; namespace Moses { InputPath:: -InputPath(ttaskwptr const theTask, +InputPath(TranslationTask const* theTask, Phrase const& phrase, NonTerminalSet const& sourceNonTerms, Range const& range, InputPath const *prevNode, diff --git a/moses/InputPath.h b/moses/InputPath.h index 3a1f5e435..a2cbe0087 100644 --- a/moses/InputPath.h +++ b/moses/InputPath.h @@ -40,7 +40,8 @@ public: TargetPhrases; public: - ttaskwptr const ttask; + // ttaskwptr const ttask; + TranslationTask const* ttask; protected: const InputPath *m_prevPath; Phrase m_phrase; @@ -65,7 +66,7 @@ public: , m_nextNode(NOT_FOUND) { } - InputPath(ttaskwptr const ttask, + InputPath(TranslationTask const* ttask, // ttaskwptr const ttask, Phrase const& phrase, NonTerminalSet const& sourceNonTerms, Range const& range, diff --git a/moses/LM/BackwardTest.cpp b/moses/LM/BackwardTest.cpp index ef45d6e60..9fb5282fa 100644 --- a/moses/LM/BackwardTest.cpp +++ b/moses/LM/BackwardTest.cpp @@ -123,7 +123,6 @@ public: Input, outputFactorOrder, "the", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 1 ); @@ -150,7 +149,6 @@ public: Input, outputFactorOrder, "the licenses", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 2 ); @@ -179,7 +177,6 @@ public: Input, outputFactorOrder, "the licenses for", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 3 ); @@ -206,7 +203,6 @@ public: Input, outputFactorOrder, "the licenses for most", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 4 ); @@ -252,7 +248,6 @@ public: Input, outputFactorOrder, "the", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 1 ); @@ -280,7 +275,6 @@ public: Input, outputFactorOrder, "licenses", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 1 ); @@ -308,7 +302,6 @@ public: Input, outputFactorOrder, "for", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 1 ); @@ -336,7 +329,6 @@ public: Input, outputFactorOrder, "most", - // StaticData::Instance().GetFactorDelimiter(), NULL); BOOST_CHECK( phrase.GetSize() == 1 ); diff --git a/moses/Manager.cpp b/moses/Manager.cpp index f4dc03584..b0a85bea5 100644 --- a/moses/Manager.cpp +++ b/moses/Manager.cpp @@ -1699,7 +1699,7 @@ OutputSurface(std::ostream &out, Hypothesis const& edge, bool const recursive) c } bool markUnknown = options()->unk.mark; - std::string const& fd = options()->output.FactorDelimiter; + std::string const& fd = options()->output.factor_delimiter; TargetPhrase const& phrase = edge.GetCurrTargetPhrase(); size_t size = phrase.GetSize(); diff --git a/moses/PDTAimp.cpp b/moses/PDTAimp.cpp index c4bd2c338..ec1557982 100644 --- a/moses/PDTAimp.cpp +++ b/moses/PDTAimp.cpp @@ -116,6 +116,7 @@ PDTAimp::GetTargetPhraseCollection(Phrase const &src) const // convert into TargetPhrases + std::string fd = m_obj->options()->output.factor_delimiter; for(size_t i=0; ifirst + , m_obj->options()->output.factor_delimiter , scores.transScore , scores.inputScores , NULL @@ -403,6 +406,7 @@ void PDTAimp::CacheSource(ConfusionNet const& src) void PDTAimp::CreateTargetPhrase(TargetPhrase& targetPhrase, StringTgtCand::Tokens const& factorStrings, + std::string const& factorDelimiter, Scores const& transVector, Scores const& inputVector, const std::string *alignmentString, @@ -411,7 +415,8 @@ void PDTAimp::CreateTargetPhrase(TargetPhrase& targetPhrase, FactorCollection &factorCollection = FactorCollection::Instance(); for(size_t k=0; k word(*factorStrings[k], StaticData::Instance().GetFactorDelimiter()); + util::TokenIter + word(*factorStrings[k], factorDelimiter); Word& w=targetPhrase.AddWord(); for(size_t l=0; lSetParameter(m_verboseLevel, "verbose", (size_t) 1); - - - m_parameter->SetParameter(m_outputUnknownsFile, "output-unknowns", ""); - - // m_parameter->SetParameter(m_startTranslationId, - // "start-translation-id", 0); - return true; } @@ -208,12 +200,6 @@ bool StaticData::LoadData(Parameter *parameter) // threading etc. if (!ini_performance_options()) return false; - // Compact phrase table and reordering model - // m_parameter->SetParameter(m_minphrMemory, "minphr-memory", false ); - // m_parameter->SetParameter(m_minlexrMemory, "minlexr-memory", false ); - - // S2T decoder - // FEATURE FUNCTION INITIALIZATION HAPPENS HERE =============================== // set class-specific default parameters @@ -324,8 +310,6 @@ void StaticData::LoadChartDecodingParameters() // source label overlap m_parameter->SetParameter(m_sourceLabelOverlap, "source-label-overlap", SourceLabelOverlapAdd); - m_parameter->SetParameter(m_ruleLimit, "rule-limit", - DEFAULT_MAX_TRANS_OPT_SIZE); } @@ -621,7 +605,6 @@ void StaticData::LoadFeatureFunctions() m_requireSortingAfterSourceContext = true; } - // if (PhraseDictionary *ffCast = dynamic_cast(ff)) { if (dynamic_cast(ff)) { doLoad = false; } diff --git a/moses/StaticData.h b/moses/StaticData.h index e633d94aa..871b82641 100644 --- a/moses/StaticData.h +++ b/moses/StaticData.h @@ -104,12 +104,6 @@ protected: std::string m_outputUnknownsFile; //! output unknowns in this file - size_t m_ruleLimit; - - // Whether to load compact phrase table and reordering table into memory - bool m_minphrMemory; - bool m_minlexrMemory; - // Initial = 0 = can be used when creating poss trans // Other = 1 = used to calculate LM score once all steps have been processed Word m_inputDefaultNonTerminal, m_outputDefaultNonTerminal; @@ -146,7 +140,6 @@ protected: void NoCache(); - bool m_continuePartialTranslation; std::string m_binPath; // soft NT lookup for chart models @@ -252,17 +245,10 @@ public: return m_unknownLHS; } - size_t GetRuleLimit() const { - return m_ruleLimit; - } float GetRuleCountThreshold() const { return 999999; /* TODO wtf! */ } - bool ContinuePartialTranslation() const { - return m_continuePartialTranslation; - } - void ReLoadBleuScoreFeatureParameter(float weight); Parameter* GetParameter() { @@ -273,10 +259,6 @@ public: return m_threadCount; } - // long GetStartTranslationId() const { - // return m_startTranslationId; - // } - void SetExecPath(const std::string &path); const std::string &GetBinDirectory() const; diff --git a/moses/Syntax/F2S/Manager-inl.h b/moses/Syntax/F2S/Manager-inl.h index 29b1f2ba9..60c0ba25f 100644 --- a/moses/Syntax/F2S/Manager-inl.h +++ b/moses/Syntax/F2S/Manager-inl.h @@ -60,9 +60,9 @@ void Manager::Decode() const StaticData &staticData = StaticData::Instance(); // Get various pruning-related constants. - const std::size_t popLimit = staticData.options()->cube.pop_limit; - const std::size_t ruleLimit = staticData.GetRuleLimit(); - const std::size_t stackLimit = staticData.options()->search.stack_size; + const std::size_t popLimit = options()->cube.pop_limit; + const std::size_t ruleLimit = options()->syntax.rule_limit; + const std::size_t stackLimit = options()->search.stack_size; // Initialize the stacks. InitializeStacks(); diff --git a/moses/Syntax/S2T/Manager-inl.h b/moses/Syntax/S2T/Manager-inl.h index 6bfc4a61c..a21de6a94 100644 --- a/moses/Syntax/S2T/Manager-inl.h +++ b/moses/Syntax/S2T/Manager-inl.h @@ -163,9 +163,9 @@ void Manager::Decode() const StaticData &staticData = StaticData::Instance(); // Get various pruning-related constants. - const std::size_t popLimit = staticData.options()->cube.pop_limit; - const std::size_t ruleLimit = staticData.GetRuleLimit(); - const std::size_t stackLimit = staticData.options()->search.stack_size; + const std::size_t popLimit = options()->cube.pop_limit; + const std::size_t ruleLimit = options()->syntax.rule_limit; + const std::size_t stackLimit = options()->search.stack_size; // Initialise the PChart and SChart. InitializeCharts(); diff --git a/moses/Syntax/T2S/Manager-inl.h b/moses/Syntax/T2S/Manager-inl.h index ec97e76de..ee8b33b15 100644 --- a/moses/Syntax/T2S/Manager-inl.h +++ b/moses/Syntax/T2S/Manager-inl.h @@ -94,11 +94,11 @@ void Manager::InitializeStacks() template void Manager::Decode() { - const StaticData &staticData = StaticData::Instance(); + // const StaticData &staticData = StaticData::Instance(); // Get various pruning-related constants. const std::size_t popLimit = this->options()->cube.pop_limit; - const std::size_t ruleLimit = staticData.GetRuleLimit(); + const std::size_t ruleLimit = this->options()->syntax.rule_limit; const std::size_t stackLimit = this->options()->search.stack_size; // Initialize the stacks. @@ -215,8 +215,8 @@ void Manager::ExtractKBest( // than k. The k-best factor determines how much bigger the limit should be, // with 0 being 'unlimited.' This actually sets a large-ish limit in case // too many translations are identical. - const StaticData &staticData = StaticData::Instance(); - const std::size_t nBestFactor = staticData.options()->nbest.factor; + // const StaticData &staticData = StaticData::Instance(); + const std::size_t nBestFactor = this->options()->nbest.factor; std::size_t numDerivations = (nBestFactor == 0) ? k*1000 : k*nBestFactor; // Extract the derivations. diff --git a/moses/Syntax/T2S/RuleTrieLoader.cpp b/moses/Syntax/T2S/RuleTrieLoader.cpp index 026237128..8006799e0 100644 --- a/moses/Syntax/T2S/RuleTrieLoader.cpp +++ b/moses/Syntax/T2S/RuleTrieLoader.cpp @@ -44,9 +44,6 @@ bool RuleTrieLoader::Load(Moses::AllOptions const& opts, { PrintUserTime(std::string("Start loading text phrase table. Moses format")); - // const StaticData &staticData = StaticData::Instance(); - // const std::string &factorDelimiter = staticData.GetFactorDelimiter(); - std::size_t count = 0; std::ostream *progress = NULL; diff --git a/moses/TargetPhrase.cpp b/moses/TargetPhrase.cpp index 7cd3afc3b..f47684cda 100644 --- a/moses/TargetPhrase.cpp +++ b/moses/TargetPhrase.cpp @@ -53,7 +53,6 @@ TargetPhrase::TargetPhrase( std::string out_string, const PhraseDictionary *pt) const StaticData &staticData = StaticData::Instance(); // XXX should this really be InputFactorOrder??? CreateFromString(Output, staticData.options()->input.factor_order, out_string, - // staticData.GetFactorDelimiter(), // eliminated [UG] NULL); } diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemory.cpp b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemory.cpp index 55db20303..16e6cf668 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemory.cpp +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemory.cpp @@ -45,8 +45,8 @@ ChartRuleLookupManagerMemory::ChartRuleLookupManagerMemory( { size_t sourceSize = parser.GetSize(); - - m_completedRules.resize(sourceSize); + size_t ruleLimit = parser.options()->syntax.rule_limit; + m_completedRules.resize(sourceSize, CompletedRuleCollection(ruleLimit)); m_isSoftMatching = !m_softMatchingMap.empty(); } diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp index 4b1bdb03a..8bd790c03 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp @@ -45,8 +45,8 @@ ChartRuleLookupManagerMemoryPerSentence::ChartRuleLookupManagerMemoryPerSentence { size_t sourceSize = parser.GetSize(); - - m_completedRules.resize(sourceSize); + size_t ruleLimit = parser.options()->syntax.rule_limit; + m_completedRules.resize(sourceSize, CompletedRuleCollection(ruleLimit)); m_isSoftMatching = !m_softMatchingMap.empty(); } diff --git a/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.cpp b/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.cpp index 3a8fc8662..6c5b68be5 100644 --- a/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.cpp +++ b/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.cpp @@ -27,7 +27,8 @@ using namespace std; namespace Moses { -CompletedRuleCollection::CompletedRuleCollection() : m_ruleLimit(StaticData::Instance().GetRuleLimit()) +CompletedRuleCollection::CompletedRuleCollection(size_t rule_limit) + : m_ruleLimit(rule_limit) { m_scoreThreshold = numeric_limits::infinity(); } diff --git a/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.h b/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.h index 95823328f..2a139cd07 100644 --- a/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.h +++ b/moses/TranslationModel/CYKPlusParser/CompletedRuleCollection.h @@ -74,7 +74,7 @@ public: struct CompletedRuleCollection { public: - CompletedRuleCollection(); + CompletedRuleCollection(size_t rule_limit); ~CompletedRuleCollection(); CompletedRuleCollection(const CompletedRuleCollection &old) diff --git a/moses/TranslationModel/PhraseDictionary.cpp b/moses/TranslationModel/PhraseDictionary.cpp index 4bfe57b03..346934416 100644 --- a/moses/TranslationModel/PhraseDictionary.cpp +++ b/moses/TranslationModel/PhraseDictionary.cpp @@ -35,16 +35,6 @@ namespace Moses { std::vector PhraseDictionary::s_staticColl; -// CacheColl::~CacheColl() -// { -// // not needed any more since the switch to shared pointers -// // for (iterator iter = begin(); iter != end(); ++iter) { -// // std::pair &key = iter->second; -// // TargetPhraseCollection::shared_ptr tps = key.first; -// // delete tps; -// // } -// } - PhraseDictionary::PhraseDictionary(const std::string &line, bool registerNow) : DecodeFeature(line, registerNow) , m_tableLimit(20) // default @@ -82,8 +72,7 @@ GetTargetPhraseCollectionLEGACY(const Phrase& src) const ret.reset(new TargetPhraseCollection(*ret)); } cache[hash] = entry(ret, clock()); - } else { - // in cache. just use it + } else { // in cache. just use it iter->second.second = clock(); ret = iter->second.first; } @@ -175,31 +164,6 @@ GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const } } -// persistent cache handling -// saving presistent cache to disk -//void PhraseDictionary::SaveCache() const -//{ -// CacheColl &cache = GetCache(); -// for( std::map >::iterator iter, -// iter != cache.end(), -// iter++ ) { -// -// } -//} - -// loading persistent cache from disk -//void PhraseDictionary::LoadCache() const -//{ -// CacheColl &cache = GetCache(); -// std::map >::iterator iter; -// iter = cache.begin(); -// while( iter != cache.end() ) { -// std::map >::iterator iterRemove = iter++; -// delete iterRemove->second.first; -// cache.erase(iterRemove); -// } -//} - // reduce presistent cache by half of maximum size void PhraseDictionary::ReduceCache() const { @@ -233,7 +197,9 @@ void PhraseDictionary::ReduceCache() const << reduceCacheTime << " seconds." << std::endl); } -CacheColl &PhraseDictionary::GetCache() const +CacheColl & +PhraseDictionary:: +GetCache() const { CacheColl *cache; cache = m_cache.get(); diff --git a/moses/TranslationModel/PhraseDictionary.h b/moses/TranslationModel/PhraseDictionary.h index 351c6050a..33a128638 100644 --- a/moses/TranslationModel/PhraseDictionary.h +++ b/moses/TranslationModel/PhraseDictionary.h @@ -44,6 +44,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "moses/TargetPhraseCollection.h" #include "moses/InputPath.h" #include "moses/FF/DecodeFeature.h" +#include "moses/ContextScope.h" namespace Moses { @@ -58,15 +59,6 @@ class ChartParser; // typedef std::pair TPCollLastUse; typedef std::pair CacheCollEntry; typedef boost::unordered_map CacheColl; -// class CacheColl : public boost::unordered_map -// { -// // 1st = hash of source phrase/ address of phrase-table node -// // 2nd = all translations -// // 3rd = time of last access - -// public: -// ~CacheColl(); -// }; /** * Abstract base class for phrase dictionaries (tables). diff --git a/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.cpp b/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.cpp index cf051ff36..80bbd5b3d 100644 --- a/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.cpp +++ b/moses/TranslationModel/PhraseDictionaryDynamicCacheBased.cpp @@ -330,7 +330,6 @@ void PhraseDictionaryDynamicCacheBased::ClearEntries(std::string sourcePhraseStr { VERBOSE(3,"PhraseDictionaryDynamicCacheBased::ClearEntries(std::string sourcePhraseString, std::string targetPhraseString)" << std::endl); const StaticData &staticData = StaticData::Instance(); - // const std::string& factorDelimiter = staticData.GetFactorDelimiter(); Phrase sourcePhrase(0); Phrase targetPhrase(0); @@ -428,7 +427,6 @@ void PhraseDictionaryDynamicCacheBased::ClearSource(std::vector ent { VERBOSE(3,"entries.size():|" << entries.size() << "|" << std::endl); const StaticData &staticData = StaticData::Instance(); - // const std::string& factorDelimiter = staticData.GetFactorDelimiter(); Phrase sourcePhrase(0); std::vector::iterator it; @@ -517,7 +515,6 @@ void PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, s { VERBOSE(3,"PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, std::string targetPhraseString, std::string ageString, std::string waString)" << std::endl); const StaticData &staticData = StaticData::Instance(); - // const std::string& factorDelimiter = staticData.GetFactorDelimiter(); Phrase sourcePhrase(0); TargetPhrase targetPhrase(0); diff --git a/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp b/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp index dbc99cbbf..3dc91e500 100644 --- a/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp +++ b/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp @@ -144,7 +144,10 @@ TargetPhraseCollection::shared_ptr PhraseDictionaryMultiModelCounts::GetTargetPh } -void PhraseDictionaryMultiModelCounts::CollectSufficientStats(const Phrase& src, vector &fs, map* allStats) const +void +PhraseDictionaryMultiModelCounts:: +CollectSufficientStats(const Phrase& src, vector &fs, + map* allStats) const //fill fs and allStats with statistics from models { for(size_t i = 0; i < m_numModels; ++i) { diff --git a/moses/TranslationModel/ProbingPT/ProbingPT.cpp b/moses/TranslationModel/ProbingPT/ProbingPT.cpp index aa7cc1efe..cbfd2c1a4 100644 --- a/moses/TranslationModel/ProbingPT/ProbingPT.cpp +++ b/moses/TranslationModel/ProbingPT/ProbingPT.cpp @@ -77,7 +77,7 @@ void ProbingPT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQue InputPath &inputPath = **iter; const Phrase &sourcePhrase = inputPath.GetPhrase(); - if (sourcePhrase.GetSize() > StaticData::Instance().options()->search.max_phrase_length) { + if (sourcePhrase.GetSize() > m_options->search.max_phrase_length) { continue; } diff --git a/moses/TranslationOptionCollectionConfusionNet.cpp b/moses/TranslationOptionCollectionConfusionNet.cpp index 0fa50b36c..48178a079 100644 --- a/moses/TranslationOptionCollectionConfusionNet.cpp +++ b/moses/TranslationOptionCollectionConfusionNet.cpp @@ -63,9 +63,10 @@ TranslationOptionCollectionConfusionNet(ttasksptr const& ttask, const ScorePair &scores = col[i].second; ScorePair *inputScore = new ScorePair(scores); - InputPath *path = new InputPath(ttask, subphrase, labels, range, NULL, inputScore); + InputPath* path = new InputPath(ttask.get(), subphrase, labels, + range, NULL, inputScore); list.push_back(path); - + m_inputPathQueue.push_back(path); } } @@ -114,7 +115,8 @@ TranslationOptionCollectionConfusionNet(ttasksptr const& ttask, ScorePair *inputScore = new ScorePair(*prevInputScore); inputScore->PlusEquals(scores); - InputPath *path = new InputPath(ttask, subphrase, labels, range, &prevPath, inputScore); + InputPath *path = new InputPath(ttask.get(), subphrase, labels, range, + &prevPath, inputScore); list.push_back(path); m_inputPathQueue.push_back(path); diff --git a/moses/TranslationOptionCollectionLattice.cpp b/moses/TranslationOptionCollectionLattice.cpp index b0973aefe..f11680605 100644 --- a/moses/TranslationOptionCollectionLattice.cpp +++ b/moses/TranslationOptionCollectionLattice.cpp @@ -65,7 +65,7 @@ TranslationOptionCollectionLattice ScorePair *inputScore = new ScorePair(scores); InputPath *path - = new InputPath(ttask, subphrase, labels, range, NULL, inputScore); + = new InputPath(ttask.get(), subphrase, labels, range, NULL, inputScore); path->SetNextNode(nextNode); m_inputPathQueue.push_back(path); diff --git a/moses/TranslationOptionCollectionText.cpp b/moses/TranslationOptionCollectionText.cpp index 27b3353a6..f0fee0763 100644 --- a/moses/TranslationOptionCollectionText.cpp +++ b/moses/TranslationOptionCollectionText.cpp @@ -56,11 +56,11 @@ TranslationOptionCollectionText(ttasksptr const& ttask, Sentence const &input) InputPath *path; if (range.GetNumWordsCovered() == 1) { - path = new InputPath(ttask, subphrase, labels, range, NULL, NULL); + path = new InputPath(ttask.get(), subphrase, labels, range, NULL, NULL); vec.push_back(path); } else { const InputPath &prevPath = GetInputPath(startPos, endPos - 1); - path = new InputPath(ttask, subphrase, labels, range, &prevPath, NULL); + path = new InputPath(ttask.get(), subphrase, labels, range, &prevPath, NULL); vec.push_back(path); } diff --git a/moses/Word.cpp b/moses/Word.cpp index a1c852f7c..d987698cc 100644 --- a/moses/Word.cpp +++ b/moses/Word.cpp @@ -213,7 +213,8 @@ TO_STRING_BODY(Word); ostream& operator<<(ostream& out, const Word& word) { util::StringStream strme; - const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter(); + const std::string& factorDelimiter + = StaticData::Instance().options()->output.factor_delimiter; bool firstPass = true; unsigned int stop = max_fax(); for (unsigned int currFactor = 0 ; currFactor < stop; currFactor++) { diff --git a/moses/parameters/InputOptions.cpp b/moses/parameters/InputOptions.cpp index 98735ea66..dbdb2f60e 100644 --- a/moses/parameters/InputOptions.cpp +++ b/moses/parameters/InputOptions.cpp @@ -17,6 +17,7 @@ namespace Moses { xml_brackets.first = "<"; xml_brackets.second = ">"; factor_order.assign(1,0); + factor_delimiter = "|"; } bool @@ -76,6 +77,7 @@ namespace Moses { if (factor_order.empty()) factor_order.assign(1,0); param.SetParameter(placeholder_factor, "placeholder-factor", NOT_FOUND); + param.SetParameter(factor_delimiter, "factor-delimiter", "|"); param.SetParameter(input_file_path,"input-file",""); return true; diff --git a/moses/parameters/InputOptions.h b/moses/parameters/InputOptions.h index c767d9e98..572ba349a 100644 --- a/moses/parameters/InputOptions.h +++ b/moses/parameters/InputOptions.h @@ -14,10 +14,9 @@ namespace Moses InputTypeEnum input_type; XmlInputType xml_policy; // pass through, ignore, exclusive, inclusive std::vector factor_order; // input factor order - + std::string factor_delimiter; FactorType placeholder_factor; // where to store original text for placeholders std::string input_file_path; - std::pair xml_brackets; // strings to use as XML tags' opening and closing brackets. // Default are "<" and ">" diff --git a/moses/parameters/ReportingOptions.cpp b/moses/parameters/ReportingOptions.cpp index f1cb89b06..d7d3cc64b 100644 --- a/moses/parameters/ReportingOptions.cpp +++ b/moses/parameters/ReportingOptions.cpp @@ -24,6 +24,7 @@ namespace Moses { , lattice_sample_size(0) { factor_order.assign(1,0); + factor_delimiter = "|"; } bool @@ -94,14 +95,14 @@ namespace Moses { params= param.GetParam("output-factors"); if (params) factor_order = Scan(*params); if (factor_order.empty()) factor_order.assign(1,0); - + if (ReportAllFactors) { for (size_t i = 1; i < MAX_NUM_FACTORS; ++i) factor_order.push_back(i); } - param.SetParameter(FactorDelimiter, "factor-delimiter", std::string("|")); - param.SetParameter(FactorDelimiter, "output-factor-delimiter", FactorDelimiter); + param.SetParameter(factor_delimiter, "factor-delimiter", std::string("|")); + param.SetParameter(factor_delimiter, "output-factor-delimiter", factor_delimiter); return true; } @@ -134,12 +135,12 @@ namespace Moses { m = param.find("factor-delimiter"); if (m != param.end()) { - FactorDelimiter = Trim(xmlrpc_c::value_string(m->second)); + factor_delimiter = Trim(xmlrpc_c::value_string(m->second)); } m = param.find("output-factor-delimiter"); if (m != param.end()) { - FactorDelimiter = Trim(xmlrpc_c::value_string(m->second)); + factor_delimiter = Trim(xmlrpc_c::value_string(m->second)); } return true; diff --git a/moses/parameters/ReportingOptions.h b/moses/parameters/ReportingOptions.h index 76f8fdae4..92a0f1489 100644 --- a/moses/parameters/ReportingOptions.h +++ b/moses/parameters/ReportingOptions.h @@ -13,6 +13,7 @@ namespace Moses long start_translation_id; std::vector factor_order; + std::string factor_delimiter; bool ReportAllFactors; // m_reportAllFactors; int ReportSegmentation; // 0: no 1: m_reportSegmentation 2: ..._enriched @@ -23,7 +24,6 @@ namespace Moses WordAlignmentSort WA_SortOrder; // 0: no, 1: target order std::string AlignmentOutputFile; - std::string FactorDelimiter; bool WordGraph; From 37c10121994b3397f761cc128f6bd546a818b116 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Sun, 13 Dec 2015 22:37:10 +0000 Subject: [PATCH 08/17] config from current cruise control --- cruise-control/example.config | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/cruise-control/example.config b/cruise-control/example.config index 203dbb0b0..257a64253 100644 --- a/cruise-control/example.config +++ b/cruise-control/example.config @@ -1,21 +1,17 @@ # the tmp directory -MCC_TEMPDIR="/tmp/" +MCC_TEMPDIR="/disk4/cruise-control/cruise-control/tmp/" # where logs are saved -MCC_LOGDIR="/home/cruise/" - -# where moses gets checked out, tested etc. -MCC_WORKDIR="/home/cruise/work/" +MCC_LOGDIR="/disk4/cruise-control/cruise-control/" # repository that will be checked out MCC_GITREPO="https://github.com/moses-smt/mosesdecoder.git" # arguments passed to Moses configure -MCC_CONFIGURE_ARGS=" --with-srilm=/path/to/srilm --with-irstlm=/path/to/irstlm " +MCC_CONFIGURE_ARGS=" --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " # list of branches to be checked -MCC_SCAN_BRANCHES="master" - -# regression test data archive (optional, this is the default) -MCC_REGTEST_ARCHIVE="http://www.statmt.org/moses/reg-testing/moses-reg-test-data-8.tgz" +MCC_SCAN_BRANCHES="origin/master" +# run full training/eval pipeline using EMS? +MCC_RUN_EMS="yes" From dfcdb43c28b8582a51f1f5ef87af4c4cfe140a78 Mon Sep 17 00:00:00 2001 From: MosesAdmin Date: Sun, 13 Dec 2015 23:07:15 +0000 Subject: [PATCH 09/17] daily automatic beautifier --- moses/ChartParser.cpp | 4 ++-- moses/FF/GlobalLexicalModel.cpp | 12 ++++++------ moses/InputPath.h | 2 +- moses/PDTAimp.cpp | 12 ++++++------ moses/PDTAimp.h | 2 +- .../CYKPlusParser/CompletedRuleCollection.cpp | 2 +- .../PhraseDictionaryMultiModelCounts.cpp | 6 +++--- moses/TranslationOptionCollectionConfusionNet.cpp | 6 +++--- moses/TranslationOptionCollectionLattice.cpp | 2 +- moses/Word.cpp | 4 ++-- 10 files changed, 26 insertions(+), 26 deletions(-) diff --git a/moses/ChartParser.cpp b/moses/ChartParser.cpp index 15b18a7cc..a611835fa 100644 --- a/moses/ChartParser.cpp +++ b/moses/ChartParser.cpp @@ -237,8 +237,8 @@ void ChartParser::CreateInputPaths(const InputType &input) m_inputPathMatrix.resize(size); UTIL_THROW_IF2(input.GetType() != SentenceInput && input.GetType() != TreeInputType, - "Input must be a sentence or a tree, " << - "not lattice or confusion networks"); + "Input must be a sentence or a tree, " << + "not lattice or confusion networks"); TranslationTask const* ttask = m_ttask.lock().get(); for (size_t phaseSize = 1; phaseSize <= size; ++phaseSize) { diff --git a/moses/FF/GlobalLexicalModel.cpp b/moses/FF/GlobalLexicalModel.cpp index f1a92eb47..986b427b0 100644 --- a/moses/FF/GlobalLexicalModel.cpp +++ b/moses/FF/GlobalLexicalModel.cpp @@ -55,8 +55,8 @@ void GlobalLexicalModel::Load(AllOptions::ptr const& opts) { m_options = opts; FactorCollection &factorCollection = FactorCollection::Instance(); - const std::string& oFactorDelimiter = opts->output.factor_delimiter; - const std::string& iFactorDelimiter = opts->input.factor_delimiter; + const std::string& oFactorDelimiter = opts->output.factor_delimiter; + const std::string& iFactorDelimiter = opts->input.factor_delimiter; VERBOSE(2, "Loading global lexical model from file " << m_filePath << endl); @@ -82,8 +82,8 @@ void GlobalLexicalModel::Load(AllOptions::ptr const& opts) for (size_t i=0 ; i < m_outputFactorsVec.size() ; i++) { const FactorDirection& direction = Output; const FactorType& factorType = m_outputFactorsVec[i]; - const Factor* factor - = factorCollection.AddFactor( direction, factorType, factorString[i] ); + const Factor* factor + = factorCollection.AddFactor( direction, factorType, factorString[i] ); outWord->SetFactor( factorType, factor ); } @@ -93,8 +93,8 @@ void GlobalLexicalModel::Load(AllOptions::ptr const& opts) for (size_t i=0 ; i < m_inputFactorsVec.size() ; i++) { const FactorDirection& direction = Input; const FactorType& factorType = m_inputFactorsVec[i]; - const Factor* factor - = factorCollection.AddFactor( direction, factorType, factorString[i] ); + const Factor* factor + = factorCollection.AddFactor( direction, factorType, factorString[i] ); inWord->SetFactor( factorType, factor ); } diff --git a/moses/InputPath.h b/moses/InputPath.h index a2cbe0087..75e05d789 100644 --- a/moses/InputPath.h +++ b/moses/InputPath.h @@ -41,7 +41,7 @@ public: public: // ttaskwptr const ttask; - TranslationTask const* ttask; + TranslationTask const* ttask; protected: const InputPath *m_prevPath; Phrase m_phrase; diff --git a/moses/PDTAimp.cpp b/moses/PDTAimp.cpp index ec1557982..a13344aba 100644 --- a/moses/PDTAimp.cpp +++ b/moses/PDTAimp.cpp @@ -135,8 +135,8 @@ PDTAimp::GetTargetPhraseCollection(Phrase const &src) const targetPhrase.GetScoreBreakdown().Assign(m_obj, *cands[i].fnames[j], cands[i].fvalues[j]); } - CreateTargetPhrase(targetPhrase,factorStrings, fd, scoreVector, Scores(0), - &wacands[i], &src); + CreateTargetPhrase(targetPhrase,factorStrings, fd, scoreVector, Scores(0), + &wacands[i], &src); costs.push_back(std::make_pair(-targetPhrase.GetFutureScore(),tCands.size())); tCands.push_back(targetPhrase); @@ -377,7 +377,7 @@ void PDTAimp::CacheSource(ConfusionNet const& src) TargetPhrase targetPhrase(m_obj); CreateTargetPhrase(targetPhrase , j ->first - , m_obj->options()->output.factor_delimiter + , m_obj->options()->output.factor_delimiter , scores.transScore , scores.inputScores , NULL @@ -406,7 +406,7 @@ void PDTAimp::CacheSource(ConfusionNet const& src) void PDTAimp::CreateTargetPhrase(TargetPhrase& targetPhrase, StringTgtCand::Tokens const& factorStrings, - std::string const& factorDelimiter, + std::string const& factorDelimiter, Scores const& transVector, Scores const& inputVector, const std::string *alignmentString, @@ -415,8 +415,8 @@ void PDTAimp::CreateTargetPhrase(TargetPhrase& targetPhrase, FactorCollection &factorCollection = FactorCollection::Instance(); for(size_t k=0; k - word(*factorStrings[k], factorDelimiter); + util::TokenIter + word(*factorStrings[k], factorDelimiter); Word& w=targetPhrase.AddWord(); for(size_t l=0; l::infinity(); diff --git a/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp b/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp index 3dc91e500..217562f05 100644 --- a/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp +++ b/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp @@ -144,10 +144,10 @@ TargetPhraseCollection::shared_ptr PhraseDictionaryMultiModelCounts::GetTargetPh } -void +void PhraseDictionaryMultiModelCounts:: -CollectSufficientStats(const Phrase& src, vector &fs, - map* allStats) const +CollectSufficientStats(const Phrase& src, vector &fs, + map* allStats) const //fill fs and allStats with statistics from models { for(size_t i = 0; i < m_numModels; ++i) { diff --git a/moses/TranslationOptionCollectionConfusionNet.cpp b/moses/TranslationOptionCollectionConfusionNet.cpp index 55121b569..e5be9ecab 100644 --- a/moses/TranslationOptionCollectionConfusionNet.cpp +++ b/moses/TranslationOptionCollectionConfusionNet.cpp @@ -63,10 +63,10 @@ TranslationOptionCollectionConfusionNet(ttasksptr const& ttask, const ScorePair &scores = col[i].second; ScorePair *inputScore = new ScorePair(scores); - InputPath* path = new InputPath(ttask.get(), subphrase, labels, + InputPath* path = new InputPath(ttask.get(), subphrase, labels, range, NULL, inputScore); list.push_back(path); - + m_inputPathQueue.push_back(path); } } @@ -115,7 +115,7 @@ TranslationOptionCollectionConfusionNet(ttasksptr const& ttask, ScorePair *inputScore = new ScorePair(*prevInputScore); inputScore->PlusEquals(scores); - InputPath *path = new InputPath(ttask.get(), subphrase, labels, range, + InputPath *path = new InputPath(ttask.get(), subphrase, labels, range, &prevPath, inputScore); list.push_back(path); diff --git a/moses/TranslationOptionCollectionLattice.cpp b/moses/TranslationOptionCollectionLattice.cpp index 77593ed90..6559ccdf9 100644 --- a/moses/TranslationOptionCollectionLattice.cpp +++ b/moses/TranslationOptionCollectionLattice.cpp @@ -65,7 +65,7 @@ TranslationOptionCollectionLattice ScorePair *inputScore = new ScorePair(scores); InputPath *path - = new InputPath(ttask.get(), subphrase, labels, range, NULL, inputScore); + = new InputPath(ttask.get(), subphrase, labels, range, NULL, inputScore); path->SetNextNode(nextNode); m_inputPathQueue.push_back(path); diff --git a/moses/Word.cpp b/moses/Word.cpp index d987698cc..d826cbd32 100644 --- a/moses/Word.cpp +++ b/moses/Word.cpp @@ -213,8 +213,8 @@ TO_STRING_BODY(Word); ostream& operator<<(ostream& out, const Word& word) { util::StringStream strme; - const std::string& factorDelimiter - = StaticData::Instance().options()->output.factor_delimiter; + const std::string& factorDelimiter + = StaticData::Instance().options()->output.factor_delimiter; bool firstPass = true; unsigned int stop = max_fax(); for (unsigned int currFactor = 0 ; currFactor < stop; currFactor++) { From b825817f1d57c0f1f0b8c8b20c6def770851b612 Mon Sep 17 00:00:00 2001 From: MosesAdmin Date: Sun, 13 Dec 2015 23:08:37 +0000 Subject: [PATCH 10/17] update cruise config file with config from actual cruise control --- cruise-control/example.config | 16 ++++++---------- cruise-control/test_all_new_commits.sh | 6 ++++++ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cruise-control/example.config b/cruise-control/example.config index 203dbb0b0..257a64253 100644 --- a/cruise-control/example.config +++ b/cruise-control/example.config @@ -1,21 +1,17 @@ # the tmp directory -MCC_TEMPDIR="/tmp/" +MCC_TEMPDIR="/disk4/cruise-control/cruise-control/tmp/" # where logs are saved -MCC_LOGDIR="/home/cruise/" - -# where moses gets checked out, tested etc. -MCC_WORKDIR="/home/cruise/work/" +MCC_LOGDIR="/disk4/cruise-control/cruise-control/" # repository that will be checked out MCC_GITREPO="https://github.com/moses-smt/mosesdecoder.git" # arguments passed to Moses configure -MCC_CONFIGURE_ARGS=" --with-srilm=/path/to/srilm --with-irstlm=/path/to/irstlm " +MCC_CONFIGURE_ARGS=" --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " # list of branches to be checked -MCC_SCAN_BRANCHES="master" - -# regression test data archive (optional, this is the default) -MCC_REGTEST_ARCHIVE="http://www.statmt.org/moses/reg-testing/moses-reg-test-data-8.tgz" +MCC_SCAN_BRANCHES="origin/master" +# run full training/eval pipeline using EMS? +MCC_RUN_EMS="yes" diff --git a/cruise-control/test_all_new_commits.sh b/cruise-control/test_all_new_commits.sh index cf9ec7d01..0bc2c04a0 100755 --- a/cruise-control/test_all_new_commits.sh +++ b/cruise-control/test_all_new_commits.sh @@ -116,6 +116,12 @@ function run_single_test () { #regtest_dir=$PWD/$(basename $regtest_file .tgz) cd .. + # test build with different configurations + for configArgs in "${ALTERNATIVE_CONFIGURE_ARGS[@]}" + do + echo $configArgs + done + echo "## ./bjam clean" >> $longlog ./bjam clean $MCC_CONFIGURE_ARGS --with-regtest=$regtest_dir >> $longlog 2>&1 || warn "bjam clean failed, suspicious" From 4e66da9137d4bc6b6cebb09c52293b2128403ce8 Mon Sep 17 00:00:00 2001 From: MosesAdmin Date: Sun, 13 Dec 2015 23:34:05 +0000 Subject: [PATCH 11/17] added ALTERNATIVE_CONFIGURE_ARGS --- cruise-control/example.config | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cruise-control/example.config b/cruise-control/example.config index 257a64253..c136850e9 100644 --- a/cruise-control/example.config +++ b/cruise-control/example.config @@ -10,6 +10,14 @@ MCC_GITREPO="https://github.com/moses-smt/mosesdecoder.git" # arguments passed to Moses configure MCC_CONFIGURE_ARGS=" --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " +ALTERNATIVE_CONFIGURE_ARGS=( + " --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " + " --with-srilm=/disk4/cruise-control/srilm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " + " --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " + " --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " + " --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 -j8 " + ) + # list of branches to be checked MCC_SCAN_BRANCHES="origin/master" From af1e6c89732426f17920ff89f6c93687b2908559 Mon Sep 17 00:00:00 2001 From: MosesAdmin Date: Sun, 13 Dec 2015 23:59:28 +0000 Subject: [PATCH 12/17] echo args --- cruise-control/test_all_new_commits.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cruise-control/test_all_new_commits.sh b/cruise-control/test_all_new_commits.sh index 29c207091..c685b106e 100755 --- a/cruise-control/test_all_new_commits.sh +++ b/cruise-control/test_all_new_commits.sh @@ -120,7 +120,9 @@ function run_single_test () { echo "## test build with different configurations" >> $longlog for configArgs in "${ALTERNATIVE_CONFIGURE_ARGS[@]}" do - ./bjam clean $configArgs >> $longlog 2>&1 || warn "bjam clean failed, suspicious" + echo "ARGS=" + echo $configArgs + ./bjam clean $configArgs >> $longlog 2>&1 || warn "bjam clean failed, suspicious" done echo "## ./bjam clean" >> $longlog From b8e73c03767aaa98b1bdcf1e0352574d683530c1 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Mon, 14 Dec 2015 00:18:10 +0000 Subject: [PATCH 13/17] collect compile errors into log files --- cruise-control/test_all_new_commits.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cruise-control/test_all_new_commits.sh b/cruise-control/test_all_new_commits.sh index c685b106e..89f0df05f 100755 --- a/cruise-control/test_all_new_commits.sh +++ b/cruise-control/test_all_new_commits.sh @@ -120,9 +120,8 @@ function run_single_test () { echo "## test build with different configurations" >> $longlog for configArgs in "${ALTERNATIVE_CONFIGURE_ARGS[@]}" do - echo "ARGS=" - echo $configArgs - ./bjam clean $configArgs >> $longlog 2>&1 || warn "bjam clean failed, suspicious" + echo "building with args: $configArgs" + ./bjam clean $configArgs &>> $longlog done echo "## ./bjam clean" >> $longlog From 91fd40b9c485ac06496874787580936c61365571 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Mon, 14 Dec 2015 00:32:21 +0000 Subject: [PATCH 14/17] collect compile errors into log files --- cruise-control/test_all_new_commits.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cruise-control/test_all_new_commits.sh b/cruise-control/test_all_new_commits.sh index 89f0df05f..7c29a8212 100755 --- a/cruise-control/test_all_new_commits.sh +++ b/cruise-control/test_all_new_commits.sh @@ -120,8 +120,8 @@ function run_single_test () { echo "## test build with different configurations" >> $longlog for configArgs in "${ALTERNATIVE_CONFIGURE_ARGS[@]}" do - echo "building with args: $configArgs" - ./bjam clean $configArgs &>> $longlog + echo "building with args: $configArgs" >> $longlog + ./bjam clean $configArgs >> $longlog done echo "## ./bjam clean" >> $longlog From 81f2e133e8b477d3be418bc29822aa65a8f994f8 Mon Sep 17 00:00:00 2001 From: MosesAdmin Date: Mon, 14 Dec 2015 00:58:14 +0000 Subject: [PATCH 15/17] rebuild each different configuration --- cruise-control/test_all_new_commits.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cruise-control/test_all_new_commits.sh b/cruise-control/test_all_new_commits.sh index 7c29a8212..fef758744 100755 --- a/cruise-control/test_all_new_commits.sh +++ b/cruise-control/test_all_new_commits.sh @@ -121,11 +121,11 @@ function run_single_test () { for configArgs in "${ALTERNATIVE_CONFIGURE_ARGS[@]}" do echo "building with args: $configArgs" >> $longlog - ./bjam clean $configArgs >> $longlog + ./bjam clean -a $configArgs >> $longlog done echo "## ./bjam clean" >> $longlog - ./bjam clean $MCC_CONFIGURE_ARGS --with-regtest=$regtest_dir >> $longlog 2>&1 || warn "bjam clean failed, suspicious" + ./bjam clean -a $MCC_CONFIGURE_ARGS --with-regtest=$regtest_dir >> $longlog 2>&1 || warn "bjam clean failed, suspicious" echo "## ./bjam $MCC_CONFIGURE_ARGS" >> $longlog if [ -z "$err" ]; then From ad8bb7080659bfe0e8698839b267b70fdc9db610 Mon Sep 17 00:00:00 2001 From: MosesAdmin Date: Mon, 14 Dec 2015 01:04:57 +0000 Subject: [PATCH 16/17] rebuild each different configuration --- cruise-control/test_all_new_commits.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cruise-control/test_all_new_commits.sh b/cruise-control/test_all_new_commits.sh index fef758744..11d71c62d 100755 --- a/cruise-control/test_all_new_commits.sh +++ b/cruise-control/test_all_new_commits.sh @@ -121,7 +121,7 @@ function run_single_test () { for configArgs in "${ALTERNATIVE_CONFIGURE_ARGS[@]}" do echo "building with args: $configArgs" >> $longlog - ./bjam clean -a $configArgs >> $longlog + ./bjam clean -a $configArgs >> $longlog 2>&1 || warn "bjam clean failed, suspicious" done echo "## ./bjam clean" >> $longlog From 375d9f7cc8561ed13f4ffdfa58111686dc961cb4 Mon Sep 17 00:00:00 2001 From: MosesAdmin Date: Mon, 14 Dec 2015 01:22:20 +0000 Subject: [PATCH 17/17] use --no-xmlrpc-c --- cruise-control/example.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cruise-control/example.config b/cruise-control/example.config index c136850e9..dd756329a 100644 --- a/cruise-control/example.config +++ b/cruise-control/example.config @@ -15,7 +15,7 @@ ALTERNATIVE_CONFIGURE_ARGS=( " --with-srilm=/disk4/cruise-control/srilm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " " --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " " --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-boost=/disk4/cruise-control/boost_1_55_0 --with-xmlrpc-c=/disk4/cruise-control/xmlrpc-c -j8 " - " --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 -j8 " + " --with-srilm=/disk4/cruise-control/srilm --with-irstlm=/disk4/cruise-control/irstlm --with-dalm=/disk4/cruise-control/DALM --with-cmph=/disk4/cruise-control/cmph-2.0 --with-boost=/disk4/cruise-control/boost_1_55_0 --no-xmlrpc-c -j8 " ) # list of branches to be checked