From 28f4e3b1a758124c18048560a7332c2a4df07fff Mon Sep 17 00:00:00 2001 From: mjdenkowski Date: Wed, 2 Sep 2015 01:55:06 -0400 Subject: [PATCH 1/8] Compatibility: mmsapt and factored translation with generation models --- moses/TranslationModel/UG/mmsapt.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/moses/TranslationModel/UG/mmsapt.cpp b/moses/TranslationModel/UG/mmsapt.cpp index c0cc2ab7f..30faf0408 100644 --- a/moses/TranslationModel/UG/mmsapt.cpp +++ b/moses/TranslationModel/UG/mmsapt.cpp @@ -74,6 +74,12 @@ namespace Moses { init(line); setup_local_feature_functions(); + // Set features used for scoring extracted phrases: + // * Use all features that can operate on input factors and this model's + // output factor + // * Don't use features that depend on generation steps that won't be run + // yet at extract time + SetFeaturesToApply(); Register(); } @@ -162,6 +168,10 @@ namespace Moses parse_factor_spec(m_ifactor,"input-factor"); parse_factor_spec(m_ofactor,"output-factor"); + // Masks for available factors that inform SetFeaturesToApply + m_inputFactors = FactorMask(m_ifactor); + m_outputFactors = FactorMask(m_ofactor); + pair dflt = pair ("smooth",".01"); m_lbop_conf = atof(param.insert(dflt).first->second.c_str()); @@ -575,7 +585,8 @@ namespace Moses } tp->SetAlignTerm(pool.aln); tp->GetScoreBreakdown().Assign(this, fvals); - tp->EvaluateInIsolation(src); + // Evaluate with all features that can be computed using available factors + tp->EvaluateInIsolation(src, m_featuresToApply); if (m_lr_func) { From 50d99452d5c3b6e772fafdcc74672ac043c0dca7 Mon Sep 17 00:00:00 2001 From: mjdenkowski Date: Wed, 2 Sep 2015 02:01:17 -0400 Subject: [PATCH 2/8] Revert "Compatibility: mmsapt and factored translation with generation models" (Make this a pull request to get at least one more set of eyes on it) This reverts commit 28f4e3b1a758124c18048560a7332c2a4df07fff. --- moses/TranslationModel/UG/mmsapt.cpp | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/moses/TranslationModel/UG/mmsapt.cpp b/moses/TranslationModel/UG/mmsapt.cpp index 30faf0408..c0cc2ab7f 100644 --- a/moses/TranslationModel/UG/mmsapt.cpp +++ b/moses/TranslationModel/UG/mmsapt.cpp @@ -74,12 +74,6 @@ namespace Moses { init(line); setup_local_feature_functions(); - // Set features used for scoring extracted phrases: - // * Use all features that can operate on input factors and this model's - // output factor - // * Don't use features that depend on generation steps that won't be run - // yet at extract time - SetFeaturesToApply(); Register(); } @@ -168,10 +162,6 @@ namespace Moses parse_factor_spec(m_ifactor,"input-factor"); parse_factor_spec(m_ofactor,"output-factor"); - // Masks for available factors that inform SetFeaturesToApply - m_inputFactors = FactorMask(m_ifactor); - m_outputFactors = FactorMask(m_ofactor); - pair dflt = pair ("smooth",".01"); m_lbop_conf = atof(param.insert(dflt).first->second.c_str()); @@ -585,8 +575,7 @@ namespace Moses } tp->SetAlignTerm(pool.aln); tp->GetScoreBreakdown().Assign(this, fvals); - // Evaluate with all features that can be computed using available factors - tp->EvaluateInIsolation(src, m_featuresToApply); + tp->EvaluateInIsolation(src); if (m_lr_func) { From 8aaf89faf52aad324099dc8b8658b53d1305ce47 Mon Sep 17 00:00:00 2001 From: Phil Williams Date: Wed, 2 Sep 2015 10:28:05 +0100 Subject: [PATCH 3/8] Syntax::Manager: output a blank line if there's no translation Should never happen for T2S/F2S, but possible for S2T. --- moses/Syntax/Manager.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/moses/Syntax/Manager.cpp b/moses/Syntax/Manager.cpp index f7df19d16..5dd4c3b64 100644 --- a/moses/Syntax/Manager.cpp +++ b/moses/Syntax/Manager.cpp @@ -29,6 +29,7 @@ void Manager::OutputBest(OutputCollector *collector) const if (StaticData::Instance().GetOutputHypoScore()) { out << "0 "; } + out << '\n'; } else { if (StaticData::Instance().GetOutputHypoScore()) { out << best->label.score << " "; From 8df6cb5bc6715d8f450c8cf86c392a9876a2ee3f Mon Sep 17 00:00:00 2001 From: Rico Sennrich Date: Thu, 3 Sep 2015 18:15:29 +0100 Subject: [PATCH 4/8] synchronize nplm jamfile options between moses/LM/Jamfile and lm/Jamfile --- lm/Jamfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lm/Jamfile b/lm/Jamfile index 0b5bbf259..a479e2d35 100644 --- a/lm/Jamfile +++ b/lm/Jamfile @@ -17,7 +17,7 @@ wrappers = ; local with-nplm = [ option.get "with-nplm" ] ; if $(with-nplm) { lib nplm : : $(with-nplm)/src ; - obj nplm.o : wrappers/nplm.cc : .. $(with-nplm)/src -fopenmp ; + obj nplm.o : wrappers/nplm.cc : .. $(with-nplm)/src -fopenmp $(with-nplm)/3rdparty/eigen NPLM_DOUBLE_PRECISION=0 ; alias nplm-all : nplm.o nplm ..//boost_thread : : : -fopenmp -fopenmp WITH_NPLM ..//boost_thread ; wrappers += nplm-all ; } From 1eabb49ec81c1f432842646effbf4d00677f96bc Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 3 Sep 2015 19:22:16 +0100 Subject: [PATCH 5/8] eclipse --- contrib/other-builds/moses/.cproject | 24 +- contrib/other-builds/moses/.project | 495 +++++++++++++----- .../UG/generic/threading/ug_ref_counter.h | 25 - .../UG/generic/threading/ug_thread_pool.cc | 31 -- .../UG/generic/threading/ug_thread_pool.h | 30 -- moses/TranslationModel/UG/mm/mtt.count.cc | 10 +- .../UG/mm/ug_conll_bottom_up_token.h | 6 +- 7 files changed, 377 insertions(+), 244 deletions(-) delete mode 100644 moses/TranslationModel/UG/generic/threading/ug_ref_counter.h delete mode 100644 moses/TranslationModel/UG/generic/threading/ug_thread_pool.cc delete mode 100644 moses/TranslationModel/UG/generic/threading/ug_thread_pool.h diff --git a/contrib/other-builds/moses/.cproject b/contrib/other-builds/moses/.cproject index 960a13947..60f2f47fd 100644 --- a/contrib/other-builds/moses/.cproject +++ b/contrib/other-builds/moses/.cproject @@ -19,7 +19,7 @@ - + @@ -33,6 +33,7 @@ + + + + - - - - + - + + + - + @@ -88,7 +92,7 @@ - + @@ -141,10 +145,10 @@ - + - + diff --git a/contrib/other-builds/moses/.project b/contrib/other-builds/moses/.project index 65fa053fa..8b1013bf1 100644 --- a/contrib/other-builds/moses/.project +++ b/contrib/other-builds/moses/.project @@ -2135,16 +2135,6 @@ 2 virtual:/virtual - - TranslationModel/BilingualDynSuffixArray.cpp - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/BilingualDynSuffixArray.cpp - - - TranslationModel/BilingualDynSuffixArray.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/BilingualDynSuffixArray.h - TranslationModel/CYKPlusParser 2 @@ -2155,21 +2145,6 @@ 2 virtual:/virtual - - TranslationModel/DynSAInclude - 2 - virtual:/virtual - - - TranslationModel/DynSuffixArray.cpp - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSuffixArray.cpp - - - TranslationModel/DynSuffixArray.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSuffixArray.h - TranslationModel/PhraseDictionary.cpp 1 @@ -2185,16 +2160,6 @@ 1 PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryDynSuffixArray.README - - TranslationModel/PhraseDictionaryDynSuffixArray.cpp - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryDynSuffixArray.cpp - - - TranslationModel/PhraseDictionaryDynSuffixArray.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryDynSuffixArray.h - TranslationModel/PhraseDictionaryDynamicCacheBased.cpp 1 @@ -2315,16 +2280,6 @@ 2 virtual:/virtual - - TranslationModel/WordCoocTable.cpp - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/WordCoocTable.cpp - - - TranslationModel/WordCoocTable.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/WordCoocTable.h - TranslationModel/fuzzy-match 2 @@ -3190,86 +3145,6 @@ 2 virtual:/virtual - - TranslationModel/DynSAInclude/FileHandler.cpp - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/FileHandler.cpp - - - TranslationModel/DynSAInclude/FileHandler.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/FileHandler.h - - - TranslationModel/DynSAInclude/Jamfile - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/Jamfile - - - TranslationModel/DynSAInclude/RandLMCache.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/RandLMCache.h - - - TranslationModel/DynSAInclude/RandLMFilter.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/RandLMFilter.h - - - TranslationModel/DynSAInclude/fdstream.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/fdstream.h - - - TranslationModel/DynSAInclude/hash.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/hash.h - - - TranslationModel/DynSAInclude/onlineRLM.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/onlineRLM.h - - - TranslationModel/DynSAInclude/params.cpp - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/params.cpp - - - TranslationModel/DynSAInclude/params.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/params.h - - - TranslationModel/DynSAInclude/perfectHash.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/perfectHash.h - - - TranslationModel/DynSAInclude/quantizer.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/quantizer.h - - - TranslationModel/DynSAInclude/types.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/types.h - - - TranslationModel/DynSAInclude/utils.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/utils.h - - - TranslationModel/DynSAInclude/vocab.cpp - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/vocab.cpp - - - TranslationModel/DynSAInclude/vocab.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/DynSAInclude/vocab.h - TranslationModel/ProbingPT/Jamfile 1 @@ -3540,6 +3415,16 @@ 1 PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/Makefile + + TranslationModel/UG/TargetPhraseCollectionCache.cc + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/TargetPhraseCollectionCache.cc + + + TranslationModel/UG/TargetPhraseCollectionCache.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/TargetPhraseCollectionCache.h + TranslationModel/UG/bin 2 @@ -3585,11 +3470,6 @@ 1 PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/ptable-lookup.cc - - TranslationModel/UG/sapt_phrase_key.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/sapt_phrase_key.h - TranslationModel/UG/sapt_phrase_scorers.h 1 @@ -3935,6 +3815,16 @@ 1 PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/test-dynamic-im-tsa.cc + + TranslationModel/UG/mm/test-http-client.cc + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/test-http-client.cc + + + TranslationModel/UG/mm/test-xml-escaping.cc + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/test-xml-escaping.cc + TranslationModel/UG/mm/tpt_pickler.cc 1 @@ -3980,6 +3870,56 @@ 1 PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_bitext.h + + TranslationModel/UG/mm/ug_bitext_agenda.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_bitext_agenda.h + + + TranslationModel/UG/mm/ug_bitext_agenda_job.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_bitext_agenda_job.h + + + TranslationModel/UG/mm/ug_bitext_agenda_worker.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_bitext_agenda_worker.h + + + TranslationModel/UG/mm/ug_bitext_jstats.cc + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_bitext_jstats.cc + + + TranslationModel/UG/mm/ug_bitext_jstats.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_bitext_jstats.h + + + TranslationModel/UG/mm/ug_bitext_moses.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_bitext_moses.h + + + TranslationModel/UG/mm/ug_bitext_phrase_extraction_record.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_bitext_phrase_extraction_record.h + + + TranslationModel/UG/mm/ug_bitext_pstats.cc + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_bitext_pstats.cc + + + TranslationModel/UG/mm/ug_bitext_pstats.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_bitext_pstats.h + + + TranslationModel/UG/mm/ug_bitext_sampler.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_bitext_sampler.h + TranslationModel/UG/mm/ug_conll_bottom_up_token.h 1 @@ -4015,6 +3955,26 @@ 1 PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_deptree.h + + TranslationModel/UG/mm/ug_http_client.cc + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_http_client.cc + + + TranslationModel/UG/mm/ug_http_client.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_http_client.h + + + TranslationModel/UG/mm/ug_im_bitext.cc + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_im_bitext.cc + + + TranslationModel/UG/mm/ug_im_bitext.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_im_bitext.h + TranslationModel/UG/mm/ug_im_tsa.h 1 @@ -4035,6 +3995,16 @@ 1 PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_lexical_phrase_scorer2.h + + TranslationModel/UG/mm/ug_lexical_reordering.cc + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_lexical_reordering.cc + + + TranslationModel/UG/mm/ug_lexical_reordering.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_lexical_reordering.h + TranslationModel/UG/mm/ug_load_primer.cc 1 @@ -4055,6 +4025,11 @@ 1 PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_mm_2d_table.h + + TranslationModel/UG/mm/ug_mm_bitext.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_mm_bitext.h + TranslationModel/UG/mm/ug_mm_tsa.h 1 @@ -4070,16 +4045,6 @@ 1 PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_mm_ttrack.h - - TranslationModel/UG/mm/ug_mmbitext.cc - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_mmbitext.cc - - - TranslationModel/UG/mm/ug_mmbitext.h - 1 - PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_mmbitext.h - TranslationModel/UG/mm/ug_phrasepair.cc 1 @@ -4090,6 +4055,21 @@ 1 PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_phrasepair.h + + TranslationModel/UG/mm/ug_prep_phrases.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_prep_phrases.h + + + TranslationModel/UG/mm/ug_sampling_bias.cc + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_sampling_bias.cc + + + TranslationModel/UG/mm/ug_sampling_bias.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/ug_sampling_bias.h + TranslationModel/UG/mm/ug_tsa_array_entry.cc 1 @@ -4315,6 +4295,21 @@ 1 PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/generic/stringdist/ug_stringdist.h + + TranslationModel/UG/generic/threading/ug_ref_counter.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/generic/threading/ug_ref_counter.h + + + TranslationModel/UG/generic/threading/ug_thread_pool.cc + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/generic/threading/ug_thread_pool.cc + + + TranslationModel/UG/generic/threading/ug_thread_pool.h + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/generic/threading/ug_thread_pool.h + TranslationModel/UG/generic/threading/ug_thread_safe_counter.cc 1 @@ -4325,6 +4320,11 @@ 1 PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/generic/threading/ug_thread_safe_counter.h + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1 + 2 + virtual:/virtual + TranslationModel/UG/mm/bin/gcc-4.8 2 @@ -4365,6 +4365,11 @@ 2 virtual:/virtual + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release + 2 + virtual:/virtual + TranslationModel/UG/mm/bin/gcc-4.8/release 2 @@ -4400,6 +4405,11 @@ 2 virtual:/virtual + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on + 2 + virtual:/virtual + TranslationModel/UG/mm/bin/gcc-4.8/release/debug-symbols-on 2 @@ -4975,6 +4985,11 @@ 2 virtual:/virtual + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static + 2 + virtual:/virtual + TranslationModel/UG/mm/bin/gcc-4.8/release/debug-symbols-on/link-static 2 @@ -5570,6 +5585,11 @@ 2 virtual:/virtual + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi + 2 + virtual:/virtual + TranslationModel/UG/mm/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi 2 @@ -5825,6 +5845,201 @@ 1 PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/generic/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/ug_thread_safe_counter.o + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/calc-coverage + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/calc-coverage + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/calc-coverage.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/calc-coverage.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mam2symal + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mam2symal + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mam2symal.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mam2symal.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mam_verify + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mam_verify + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mam_verify.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mam_verify.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mmlex-build + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mmlex-build + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mmlex-build.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mmlex-build.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mmlex-lookup + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mmlex-lookup + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mmlex-lookup.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mmlex-lookup.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-build + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-build + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-build.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-build.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-count-words + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-count-words + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-count-words.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-count-words.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-demo1 + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-demo1 + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-demo1.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-demo1.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-dump + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-dump + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-dump.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/mtt-dump.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/num_read_write.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/num_read_write.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/symal2mam + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/symal2mam + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/symal2mam.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/symal2mam.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/tpt_pickler.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/tpt_pickler.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/tpt_tightindex.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/tpt_tightindex.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/tpt_tokenindex.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/tpt_tokenindex.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_bitext.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_bitext.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_bitext_jstats.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_bitext_jstats.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_bitext_pstats.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_bitext_pstats.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_conll_record.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_conll_record.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_corpus_token.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_corpus_token.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_deptree.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_deptree.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_http_client.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_http_client.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_im_bitext.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_im_bitext.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_lexical_reordering.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_lexical_reordering.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_load_primer.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_load_primer.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_phrasepair.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_phrasepair.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_sampling_bias.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_sampling_bias.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_tsa_array_entry.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_tsa_array_entry.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_ttrack_base.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_ttrack_base.o + + + TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_ttrack_position.o + 1 + PARENT-3-PROJECT_LOC/moses/TranslationModel/UG/mm/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ug_ttrack_position.o + TranslationModel/UG/mm/bin/gcc-4.8/release/debug-symbols-on/link-static/threading-multi/calc-coverage 1 diff --git a/moses/TranslationModel/UG/generic/threading/ug_ref_counter.h b/moses/TranslationModel/UG/generic/threading/ug_ref_counter.h deleted file mode 100644 index d885b54a5..000000000 --- a/moses/TranslationModel/UG/generic/threading/ug_ref_counter.h +++ /dev/null @@ -1,25 +0,0 @@ -#include "ug_thread_safe_counter.h" -#pragma once -// obsolete once intrusive_ref_counter is available everywhere - -namespace Moses { - - class reference_counter - { - public: - friend void intrusive_ptr_add_ref(reference_counter const* p) - { - if (p) ++p->m_refcount; - } - friend void intrusive_ptr_release(reference_counter const* p) - { - if (p && --p->m_refcount == 0) - delete p; - } - protected: - reference_counter() {} - virtual ~reference_counter() {}; - private: - mutable ThreadSafeCounter m_refcount; - }; -} diff --git a/moses/TranslationModel/UG/generic/threading/ug_thread_pool.cc b/moses/TranslationModel/UG/generic/threading/ug_thread_pool.cc deleted file mode 100644 index 20f95a787..000000000 --- a/moses/TranslationModel/UG/generic/threading/ug_thread_pool.cc +++ /dev/null @@ -1,31 +0,0 @@ -#include "ug_thread_pool.h" -namespace ug { - -ThreadPool:: -ThreadPool(size_t const num_workers) - : m_service(), m_busywork(new boost::asio::io_service::work(m_service)) -{ - m_workers.reserve(num_workers); - for (size_t i = 0; i < num_workers; ++i) - { - // boost::shared_ptr t; - // t.reset(new boost::thread(boost::bind(&service_t::run, &m_service))); - boost::thread* t; - t = new boost::thread(boost::bind(&service_t::run, &m_service)); - m_pool.add_thread(t); - // m_workers.push_back(t); - } -} - -ThreadPool:: -~ThreadPool() -{ - m_busywork.reset(); - m_pool.join_all(); - m_service.stop(); -} - - - - -} diff --git a/moses/TranslationModel/UG/generic/threading/ug_thread_pool.h b/moses/TranslationModel/UG/generic/threading/ug_thread_pool.h deleted file mode 100644 index 41b0a71a0..000000000 --- a/moses/TranslationModel/UG/generic/threading/ug_thread_pool.h +++ /dev/null @@ -1,30 +0,0 @@ -// -*- mode: c++; tab-width: 2; indent-tabs-mode: nil -*- -#pragma once -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace ug { -class ThreadPool -{ - typedef boost::asio::io_service service_t; - service_t m_service; - boost::thread_group m_pool; - boost::scoped_ptr m_busywork; - std::vector > m_workers; - -public: - ThreadPool(size_t const num_workers); - ~ThreadPool(); - - template - void add(callable& job) { m_service.post(job); } - -}; // end of class declaration ThreadPool -} // end of namespace ug diff --git a/moses/TranslationModel/UG/mm/mtt.count.cc b/moses/TranslationModel/UG/mm/mtt.count.cc index 1e2382f67..686456581 100644 --- a/moses/TranslationModel/UG/mm/mtt.count.cc +++ b/moses/TranslationModel/UG/mm/mtt.count.cc @@ -20,15 +20,15 @@ using namespace std; using namespace ugdiss; using namespace Moses; -typedef L2R_Token Token; -typedef mmTSA::tree_iterator iter; +typedef sapt::L2R_Token Token; +typedef sapt::mmTSA::tree_iterator iter; typedef boost::unordered_map,size_t> phrase_counter_t; #define CACHING_THRESHOLD 1000 -mmTtrack T; // token tracks -TokenIndex V; // vocabs -mmTSA I; // suffix arrays +sapt::mmTtrack T; // token tracks +sapt::TokenIndex V; // vocabs +sapt::mmTSA I; // suffix arrays void interpret_args(int ac, char* av[]); string bname; diff --git a/moses/TranslationModel/UG/mm/ug_conll_bottom_up_token.h b/moses/TranslationModel/UG/mm/ug_conll_bottom_up_token.h index 3b98c39ca..6f11334a3 100644 --- a/moses/TranslationModel/UG/mm/ug_conll_bottom_up_token.h +++ b/moses/TranslationModel/UG/mm/ug_conll_bottom_up_token.h @@ -2,8 +2,8 @@ // (c) 2007-2012 Ulrich Germann // Token class for dependency trees, where the linear order // of tokens is defined as going up a dependency chain -#ifndef __ug_conll_bottom_up_token_h -#define __ug_conll_bottok_up_token_h +#pragma once + #include "ug_typedefs.h" namespace sapt { @@ -53,4 +53,4 @@ namespace sapt } } // end of namespace ugdiss -#endif + From f0de6e17126035f4894b6782f6a5cf43bf481f8d Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 3 Sep 2015 20:53:12 +0100 Subject: [PATCH 6/8] re-add accidently deleted files --- .../UG/generic/threading/ug_ref_counter.h | 25 +++++++++++++++ .../UG/generic/threading/ug_thread_pool.cc | 31 +++++++++++++++++++ .../UG/generic/threading/ug_thread_pool.h | 30 ++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 moses/TranslationModel/UG/generic/threading/ug_ref_counter.h create mode 100644 moses/TranslationModel/UG/generic/threading/ug_thread_pool.cc create mode 100644 moses/TranslationModel/UG/generic/threading/ug_thread_pool.h diff --git a/moses/TranslationModel/UG/generic/threading/ug_ref_counter.h b/moses/TranslationModel/UG/generic/threading/ug_ref_counter.h new file mode 100644 index 000000000..d885b54a5 --- /dev/null +++ b/moses/TranslationModel/UG/generic/threading/ug_ref_counter.h @@ -0,0 +1,25 @@ +#include "ug_thread_safe_counter.h" +#pragma once +// obsolete once intrusive_ref_counter is available everywhere + +namespace Moses { + + class reference_counter + { + public: + friend void intrusive_ptr_add_ref(reference_counter const* p) + { + if (p) ++p->m_refcount; + } + friend void intrusive_ptr_release(reference_counter const* p) + { + if (p && --p->m_refcount == 0) + delete p; + } + protected: + reference_counter() {} + virtual ~reference_counter() {}; + private: + mutable ThreadSafeCounter m_refcount; + }; +} diff --git a/moses/TranslationModel/UG/generic/threading/ug_thread_pool.cc b/moses/TranslationModel/UG/generic/threading/ug_thread_pool.cc new file mode 100644 index 000000000..20f95a787 --- /dev/null +++ b/moses/TranslationModel/UG/generic/threading/ug_thread_pool.cc @@ -0,0 +1,31 @@ +#include "ug_thread_pool.h" +namespace ug { + +ThreadPool:: +ThreadPool(size_t const num_workers) + : m_service(), m_busywork(new boost::asio::io_service::work(m_service)) +{ + m_workers.reserve(num_workers); + for (size_t i = 0; i < num_workers; ++i) + { + // boost::shared_ptr t; + // t.reset(new boost::thread(boost::bind(&service_t::run, &m_service))); + boost::thread* t; + t = new boost::thread(boost::bind(&service_t::run, &m_service)); + m_pool.add_thread(t); + // m_workers.push_back(t); + } +} + +ThreadPool:: +~ThreadPool() +{ + m_busywork.reset(); + m_pool.join_all(); + m_service.stop(); +} + + + + +} diff --git a/moses/TranslationModel/UG/generic/threading/ug_thread_pool.h b/moses/TranslationModel/UG/generic/threading/ug_thread_pool.h new file mode 100644 index 000000000..41b0a71a0 --- /dev/null +++ b/moses/TranslationModel/UG/generic/threading/ug_thread_pool.h @@ -0,0 +1,30 @@ +// -*- mode: c++; tab-width: 2; indent-tabs-mode: nil -*- +#pragma once +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace ug { +class ThreadPool +{ + typedef boost::asio::io_service service_t; + service_t m_service; + boost::thread_group m_pool; + boost::scoped_ptr m_busywork; + std::vector > m_workers; + +public: + ThreadPool(size_t const num_workers); + ~ThreadPool(); + + template + void add(callable& job) { m_service.post(job); } + +}; // end of class declaration ThreadPool +} // end of namespace ug From e58ddf74e85a0f8b48c55cd9c84dae4fd62cc11f Mon Sep 17 00:00:00 2001 From: Barry Haddow Date: Fri, 4 Sep 2015 10:06:54 +0100 Subject: [PATCH 7/8] parameter changes --- scripts/ems/experiment.meta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ems/experiment.meta b/scripts/ems/experiment.meta index b0c8c9a73..7d60fcaa8 100644 --- a/scripts/ems/experiment.meta +++ b/scripts/ems/experiment.meta @@ -174,7 +174,7 @@ prepare-bilingual-lm in: TRAINING:corpus TRAINING:word-alignment out: numberized_ngrams ignore-unless: bilingual-lm - rerun-on-change: TRAINING:corpus TRAINING:word-alignment + rerun-on-change: TRAINING:corpus TRAINING:word-alignment source-window order epochs default-name: lm/blm train-bilingual-lm in: numberized_ngrams TRAINING:corpus From 90f15cc619d7bb451c09c7bd42f23bb7f9c48633 Mon Sep 17 00:00:00 2001 From: Barry Haddow Date: Fri, 4 Sep 2015 10:07:50 +0100 Subject: [PATCH 8/8] extra nplm settings --- scripts/ems/experiment.perl | 2 +- scripts/training/bilingual-lm/train_nplm.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/ems/experiment.perl b/scripts/ems/experiment.perl index cb56704d8..d2dd281e8 100755 --- a/scripts/ems/experiment.perl +++ b/scripts/ems/experiment.perl @@ -1801,7 +1801,7 @@ sub define_lm_train_bilingual_lm { $cmd .= " -e $epochs" if defined($epochs); my $nplm_settings = backoff_and_get("LM:$set:nplm-settings"); - $cmd .= " $nplm_settings" if defined($nplm_settings); + $cmd .= " --extra-settings \"$nplm_settings\"" if defined($nplm_settings); # Create the ini file $cmd .= "\n"; diff --git a/scripts/training/bilingual-lm/train_nplm.py b/scripts/training/bilingual-lm/train_nplm.py index 572076006..3280d506e 100755 --- a/scripts/training/bilingual-lm/train_nplm.py +++ b/scripts/training/bilingual-lm/train_nplm.py @@ -41,6 +41,8 @@ parser.add_argument("--input_vocab_size", dest="input_vocab_size", type=int) parser.add_argument("--output_vocab_size", dest="output_vocab_size", type=int) parser.add_argument("--mmap", dest="mmap", action="store_true", help="Use memory-mapped file (for lower memory consumption).") +parser.add_argument("--extra-settings", dest="extra_settings", + help="Extra settings to be passed to NPLM") parser.set_defaults( working_dir="working", @@ -136,6 +138,7 @@ def main(options): "--activation_function", options.activation_fn, "--ngram_size", str(options.ngram_size), ] + validations_command + vocab_command + mmap_command + if options.extra_settings: train_args += options.extra_settings.split() print("Train model command: ") print(', '.join(train_args))