mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-12-28 14:32:38 +03:00
Merge github.com:moses-smt/mosesdecoder
This commit is contained in:
Hieu Hoang
commit
b8cf0deb65
2
Jamroot
2
Jamroot
@ -104,7 +104,7 @@ build-project scripts ;
|
||||
#Regression tests (only does anything if --with-regtest is passed)
|
||||
build-project regression-testing ;
|
||||
|
||||
alias programs : lm//query lm//build_binary moses-chart-cmd/src//moses_chart moses-cmd/src//programs OnDiskPt//CreateOnDisk mert//programs contrib/server//mosesserver misc//programs ;
|
||||
alias programs : lm//query lm//build_binary moses-chart-cmd/src//moses_chart moses-cmd/src//programs OnDiskPt//CreateOnDisk OnDiskPt//queryOnDiskPt mert//programs contrib/server//mosesserver misc//programs ;
|
||||
|
||||
install-bin-libs programs ;
|
||||
install-headers headers-base : [ glob-tree *.h *.hh : jam-files dist kenlm moses ] : . ;
|
||||
|
||||
@ -1,2 +1,5 @@
|
||||
lib OnDiskPt : OnDiskWrapper.cpp SourcePhrase.cpp TargetPhrase.cpp Word.cpp Phrase.cpp PhraseNode.cpp TargetPhraseCollection.cpp Vocab.cpp ../moses/src//headers ;
|
||||
|
||||
exe CreateOnDisk : Main.cpp ../moses/src//moses OnDiskPt ;
|
||||
exe queryOnDiskPt : queryOnDiskPt.cpp ../moses/src//moses OnDiskPt ;
|
||||
|
||||
|
||||
@ -1,41 +0,0 @@
|
||||
#If you get compilation errors here, make sure you have xmlrpc-c installed properly, including the abyss server option.
|
||||
|
||||
import option ;
|
||||
import path ;
|
||||
|
||||
with-xmlrpc-c = [ option.get "with-xmlrpc-c" ] ;
|
||||
if $(with-xmlrpc-c) {
|
||||
build-moses-server = true ;
|
||||
xmlrpc-command = $(with-xmlrpc-c)/bin/xmlrpc-c-config ;
|
||||
if ! [ path.exists $(xmlrpc-command) ] {
|
||||
exit Could not find $(xmlrpc-command) : 1 ;
|
||||
}
|
||||
} else {
|
||||
xmlrpc-check = [ _shell "xmlrpc-c-config --features 2>/dev/null" : exit-status ] ;
|
||||
if $(xmlrpc-check[2]) = 0 {
|
||||
if [ MATCH "(abyss-server)" : $(xmlrpc-check[1]) ] {
|
||||
build-moses-server = true ;
|
||||
} else {
|
||||
echo "Found xmlrpc-c but it does not have abyss-server. Skipping mosesserver." ;
|
||||
}
|
||||
}
|
||||
xmlrpc-command = "xmlrpc-c-config" ;
|
||||
}
|
||||
|
||||
rule shell_or_die ( cmd ) {
|
||||
local ret = [ _shell $(cmd) : exit-status ] ;
|
||||
if $(ret[2]) != 0 {
|
||||
exit "Failed to run $(cmd)" : 1 ;
|
||||
}
|
||||
return $(ret[1]) ;
|
||||
}
|
||||
|
||||
if $(build-moses-server) = true
|
||||
{
|
||||
xmlrpc-linkflags = [ shell_or_die "$(xmlrpc-command) c++2 abyss-server --libs" ] ;
|
||||
xmlrpc-cxxflags = [ shell_or_die "$(xmlrpc-command) c++2 abyss-server --cflags" ] ;
|
||||
|
||||
exe queryOnDiskPt : queryOnDiskPt.cpp ../../moses/src//moses ../../OnDiskPt//OnDiskPt : <linkflags>$(xmlrpc-linkflags) <cxxflags>$(xmlrpc-cxxflags) ;
|
||||
} else {
|
||||
alias queryOnDiskPt ;
|
||||
}
|
||||
@ -1,6 +0,0 @@
|
||||
SRI=/Users/hieuhoang/workspace/srilm
|
||||
IRST=/Users/hieuhoang/workspace/irstlm/trunk
|
||||
|
||||
g++ -o queryOnDiskPt queryOnDiskPt.cpp ../../moses/src/PhraseDictionary.cpp -I../../moses/src/ -I../../ -L../../dist/lib/ -I../../OnDiskPt -lmert_lib -ldynsa -lz -lmoses_internal -lOnDiskPt -lLM -lkenlm -lkenutil -lRuleTable -lCYKPlusParser -lScope3Parser -L$SRI/lib/macosx/ -ldstruct -lflm -llattice -lmisc -loolm -L/opt/local/lib -lboost_thread-mt -L$IRST/lib -lirstlm
|
||||
|
||||
|
||||
@ -15,18 +15,18 @@ if $(with-regtest) {
|
||||
actions reg_test_decode {
|
||||
$(TOP)/regression-testing/run-single-test.perl --decoder=$(>) --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<)
|
||||
}
|
||||
reg_test phrase : [ glob tests/phrase.* ] : ../moses-cmd/src//moses : @reg_test_decode ;
|
||||
reg_test chart : chart.target-syntax chart.target-syntax.ondisk chart.hierarchical chart.hierarchical-withsrilm chart.hierarchical.ondisk : ../moses-chart-cmd/src//moses_chart : @reg_test_decode ;
|
||||
reg_test phrase : [ glob $(TESTS)/phrase.* ] : ../moses-cmd/src//moses : @reg_test_decode ;
|
||||
reg_test chart : [ glob $(TESTS)/chart.* : $(TESTS)/chart.hierarchical-withkenlm ] : ../moses-chart-cmd/src//moses_chart : @reg_test_decode ;
|
||||
|
||||
actions reg_test_score {
|
||||
$(TOP)/regression-testing/run-test-scorer.perl --scorer=$(>) --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<)
|
||||
}
|
||||
reg_test score : [ glob tests/score.* ] : ../scripts/training/phrase-extract//score : @reg_test_score ;
|
||||
reg_test score : [ glob $(TESTS)/score.* ] : ../scripts/training/phrase-extract//score : @reg_test_score ;
|
||||
|
||||
actions reg_test_mert {
|
||||
$(TOP)/regression-testing/run-test-mert.perl --mert-dir=$(TOP)/mert --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<)
|
||||
}
|
||||
reg_test mert : [ glob tests/mert.* ] : ../mert//legacy : @reg_test_mert ;
|
||||
|
||||
alias all : phrase chart score mert ;
|
||||
reg_test mert : [ glob $(TESTS)/mert.* ] : ../mert//legacy : @reg_test_mert ;
|
||||
|
||||
alias all : phrase chart mert score ;
|
||||
}
|
||||
|
||||
@ -29,7 +29,7 @@ my($_ROOT_DIR, $_CORPUS_DIR, $_GIZA_E2F, $_GIZA_F2E, $_MODEL_DIR, $_TEMP_DIR, $_
|
||||
$_DECODING_GRAPH_BACKOFF,
|
||||
$_DECODING_STEPS, $_PARALLEL, $_FACTOR_DELIMITER, @_PHRASE_TABLE,
|
||||
@_REORDERING_TABLE, @_GENERATION_TABLE, @_GENERATION_TYPE, $_GENERATION_CORPUS,
|
||||
$_DONT_ZIP, $_MGIZA, $_MGIZA_CPUS, $_HMM_ALIGN, $_CONFIG,
|
||||
$_DONT_ZIP, $_MGIZA, $_MGIZA_CPUS, $_SNT2COOC, $_HMM_ALIGN, $_CONFIG,
|
||||
$_HIERARCHICAL,$_XML,$_SOURCE_SYNTAX,$_TARGET_SYNTAX,$_GLUE_GRAMMAR,$_GLUE_GRAMMAR_FILE,$_UNKNOWN_WORD_LABEL_FILE,$_GHKM,$_EXTRACT_OPTIONS,$_SCORE_OPTIONS,
|
||||
$_PHRASE_WORD_ALIGNMENT,$_FORCE_FACTORED_FILENAMES,
|
||||
$_MEMSCORE, $_FINAL_ALIGNMENT_MODEL,
|
||||
@ -75,6 +75,7 @@ $_HELP = 1
|
||||
'help' => \$_HELP,
|
||||
'mgiza' => \$_MGIZA, # multi-thread
|
||||
'mgiza-cpus=i' => \$_MGIZA_CPUS, # multi-thread
|
||||
'snt2cooc=s' => \$_SNT2COOC, # override snt2cooc exe. For when you want to run reduced memory snt2cooc.perl from mgiza
|
||||
'hmm-align' => \$_HMM_ALIGN,
|
||||
'final-alignment-model=s' => \$_FINAL_ALIGNMENT_MODEL, # use word alignment model 1/2/hmm/3/4/5 as final (default is 4); value 'hmm' equivalent to the --hmm-align switch
|
||||
'debug' => \$debug,
|
||||
@ -189,25 +190,28 @@ my $SNT2COOC;
|
||||
if(!defined $_MGIZA ){
|
||||
$GIZA = "$BINDIR/GIZA++";
|
||||
if (-x "$BINDIR/snt2cooc.out") {
|
||||
$SNT2COOC = "$BINDIR/snt2cooc.out";
|
||||
$SNT2COOC = "$BINDIR/snt2cooc.out";
|
||||
} elsif (-x "$BINDIR/snt2cooc") { # Since "snt2cooc.out" and "snt2cooc" work the same
|
||||
$SNT2COOC = "$BINDIR/snt2cooc";
|
||||
}
|
||||
print STDERR "Using single-thread GIZA\n";
|
||||
} else {
|
||||
$GIZA = "$BINDIR/mgiza";
|
||||
$GIZA = "$BINDIR/mgiza";
|
||||
if (-x "$BINDIR/snt2cooc") {
|
||||
$SNT2COOC = "$BINDIR/snt2cooc";
|
||||
} elsif (-x "$BINDIR/snt2cooc.out") { # Important for users that use MGIZA and copy only the "mgiza" file to $BINDIR
|
||||
$SNT2COOC = "$BINDIR/snt2cooc.out";
|
||||
}
|
||||
$SNT2COOC = "$BINDIR/snt2cooc";
|
||||
} elsif (-x "$BINDIR/snt2cooc.out") { # Important for users that use MGIZA and copy only the "mgiza" file to $BINDIR
|
||||
$SNT2COOC = "$BINDIR/snt2cooc.out";
|
||||
}
|
||||
print STDERR "Using multi-thread GIZA\n";
|
||||
if (!defined($_MGIZA_CPUS)) {
|
||||
$_MGIZA_CPUS=4;
|
||||
}
|
||||
die("ERROR: Cannot find $MGIZA_MERGE_ALIGN") unless (-x $MGIZA_MERGE_ALIGN);
|
||||
if (!defined($_MGIZA_CPUS)) {
|
||||
$_MGIZA_CPUS=4;
|
||||
}
|
||||
die("ERROR: Cannot find $MGIZA_MERGE_ALIGN") unless (-x $MGIZA_MERGE_ALIGN);
|
||||
}
|
||||
|
||||
# override
|
||||
$SNT2COOC = "$BINDIR/$_SNT2COOC" if defined($_SNT2COOC);
|
||||
|
||||
my $MKCLS = "$BINDIR/mkcls";
|
||||
|
||||
# parallel extract
|
||||
|
||||
Loading…
Reference in New Issue
Block a user