moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2025-01-06 19:49:41 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:moses-smt/mosesdecoder

Browse Source
This commit is contained in:
Kenneth Heafield 2012-10-10 18:30:23 +01:00
commit 33b932fcc7
10 changed files with 46 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
contrib/other-builds/OnDiskPt/.cproject
View File

@ -47,6 +47,7 @@
</option>
<option id="gnu.cpp.compiler.option.preprocessor.def.1052680347" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
<listOptionValue builtIn="false" value="TRACE_ENABLE"/>
<listOptionValue builtIn="false" value="WITH_THREADS"/>
</option>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1930757481" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>

4
contrib/other-builds/moses-chart-cmd/.cproject
View File

@ -29,6 +29,9 @@
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../moses/src&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../..&quot;"/>
</option>
<option id="gnu.cpp.compiler.option.preprocessor.def.1785368241" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
<listOptionValue builtIn="false" value="WITH_THREADS"/>
</option>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1402496521" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.c.compiler.exe.debug.827478809" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.exe.debug">
@ -52,6 +55,7 @@
<listOptionValue builtIn="false" value="z"/>
<listOptionValue builtIn="false" value="rt"/>
<listOptionValue builtIn="false" value="boost_system"/>
<listOptionValue builtIn="false" value="boost_thread"/>
<listOptionValue builtIn="false" value="boost_filesystem"/>
<listOptionValue builtIn="false" value="lm"/>
<listOptionValue builtIn="false" value="util"/>

4
contrib/other-builds/moses-cmd/.cproject
View File

@ -29,6 +29,9 @@
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../moses/src&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../..&quot;"/>
</option>
<option id="gnu.cpp.compiler.option.preprocessor.def.849384962" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
<listOptionValue builtIn="false" value="WITH_THREADS"/>
</option>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.363379373" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.c.compiler.exe.debug.504208780" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.exe.debug">
@ -52,6 +55,7 @@
<listOptionValue builtIn="false" value="z"/>
<listOptionValue builtIn="false" value="rt"/>
<listOptionValue builtIn="false" value="boost_system"/>
<listOptionValue builtIn="false" value="boost_thread"/>
<listOptionValue builtIn="false" value="lm"/>
<listOptionValue builtIn="false" value="util"/>
</option>

1
contrib/other-builds/moses/.cproject
View File

@ -38,6 +38,7 @@
</option>
<option id="gnu.cpp.compiler.option.preprocessor.def.752586397" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
<listOptionValue builtIn="false" value="IS_ECLIPSE"/>
<listOptionValue builtIn="false" value="WITH_THREADS"/>
<listOptionValue builtIn="false" value="KENLM_MAX_ORDER=7"/>
<listOptionValue builtIn="false" value="TRACE_ENABLE"/>
<listOptionValue builtIn="false" value="LM_IRST"/>

1
contrib/other-builds/util/.cproject
View File

@ -46,6 +46,7 @@
</option>
<option id="gnu.cpp.compiler.option.preprocessor.def.1952961175" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
<listOptionValue builtIn="false" value="TRACE_ENABLE"/>
<listOptionValue builtIn="false" value="WITH_THREADS"/>
</option>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1420621104" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>

2
moses/src/PhraseDictionary.cpp
View File

@ -72,7 +72,7 @@ PhraseDictionaryFeature::PhraseDictionaryFeature
const StaticData& staticData = StaticData::Instance();
const_cast<ScoreIndexManager&>(staticData.GetScoreIndexManager()).AddScoreProducer(this);
if (implementation == Memory || implementation == SCFG || implementation == SuffixArray
|| implementation == Compact) {
|| implementation == Compact || implementation == OnDisk) {
m_useThreadSafePhraseDictionary = true;
} else {
m_useThreadSafePhraseDictionary = false;

2
moses/src/RuleTable/PhraseDictionaryFuzzyMatch.cpp
View File

@ -54,7 +54,7 @@ namespace Moses
: PhraseDictionary(numScoreComponents, feature)
{
const StaticData &staticData = StaticData::Instance();
CHECK(staticData.ThreadCount() == 1);
//CHECK(staticData.ThreadCount() == 1);
}
bool PhraseDictionaryFuzzyMatch::Load(const std::vector<FactorType> &input

20
moses/src/fuzzy-match/FuzzyMatchWrapper.cpp
View File

@ -19,19 +19,17 @@ using namespace std;
namespace tmmt
{
int basic_flag = false;
int lsed_flag = true;
int refined_flag = true;
int length_filter_flag = true;
int parse_flag = true;
int min_match = 70;
int multiple_flag = false;
int multiple_slack = 0;
int multiple_max = 100;
map< WORD_ID,vector< int > > single_word_index;
FuzzyMatchWrapper::FuzzyMatchWrapper(const std::string &sourcePath, const std::string &targetPath, const std::string &alignmentPath)
:basic_flag(false)
,lsed_flag(true)
,refined_flag(true)
,length_filter_flag(true)
,parse_flag(true)
,min_match(70)
,multiple_flag(false)
,multiple_slack(0)
,multiple_max(100)
{
// create suffix array
//load_corpus(m_config[0], input);

11
moses/src/fuzzy-match/FuzzyMatchWrapper.h
View File

@ -33,6 +33,17 @@ protected:
std::vector< std::vector< tmmt::WORD_ID > > source;
std::vector< std::vector< tmmt::SentenceAlignment > > targetAndAlignment;
tmmt::SuffixArray *suffixArray;
std::map< WORD_ID,std::vector< int > > single_word_index;
int basic_flag;
int lsed_flag;
int refined_flag;
int length_filter_flag;
int parse_flag;
int min_match;
int multiple_flag;
int multiple_slack;
int multiple_max;
// global cache for word pairs
std::map< std::pair< WORD_ID, WORD_ID >, unsigned int > lsed;

14
scripts/training/clone_moses_model.pl
View File

@ -66,7 +66,19 @@ while (<INI>) {
$_ = "$a $b $c $d ./sa.src.$cnt{$section}$src_suffix ./sa.tgt.$cnt{$section}$tgt_suffix ./sa.align.$cnt{$section}$align_suffix\n";
}
else {
elsif ( $a eq '1' ) {
# handle binarized phrase tables
$fn = ensure_relative_from_origin(fixpath($fn));
foreach my $suf (qw( idx srctree srcvoc tgtdata tgtvoc )) {
my $fullname = "$fn.binphr.$suf";
if (-f $fullname) {
clone_file_or_die($fullname, "./$section.$cnt{$section}.binphr.$suf");
} else {
die "Binary format specified but file $fullname not found!\n";
}
}
$_ = "$a $b $c $d ./$section.$cnt{$section}\n";
} else {
$fn = fixpath($fn);
$fn = ensure_relative_from_origin($fn, $ini);
$fn = ensure_exists_or_gzipped_exists($fn);