mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2025-01-02 17:09:36 +03:00
Merge branch 'master' of github.com:moses-smt/mosesdecoder
This commit is contained in:
Hieu Hoang
commit
b207d1329a
@ -18,11 +18,6 @@ http://hlt.fbk.eu/en/irstlm
|
||||
|
||||
Ken's LM is included with the Moses distribution.
|
||||
|
||||
IMPORTANT: These instructions are for building the moses decoder ONLY,
|
||||
the training and tuning SCRIPTS contained in scripts/ must be built
|
||||
and installed separately. Also, they may require modification to
|
||||
work in certain environments.
|
||||
|
||||
--------------------------------------------------------------------------
|
||||
|
||||
1) Instructions for building with SRILM
|
||||
|
||||
4
Jamroot
4
Jamroot
@ -1,6 +1,5 @@
|
||||
#BUILDING MOSES
|
||||
#
|
||||
#
|
||||
#PACKAGES
|
||||
#Language models (optional):
|
||||
#--with-irstlm=/path/to/irstlm
|
||||
@ -41,7 +40,7 @@
|
||||
# debug-symbols=on|off include (default) or exclude debugging
|
||||
# information also known as -g
|
||||
#
|
||||
#--notrace compiles without TRACE macros
|
||||
# --notrace compiles without TRACE macros
|
||||
#
|
||||
#
|
||||
#CONTROLLING THE BUILD
|
||||
@ -145,6 +144,7 @@ project : requirements
|
||||
build-project lm ;
|
||||
build-project util ;
|
||||
#Trigger instllation into legacy paths.
|
||||
build-project mert ;
|
||||
build-project moses-cmd/src ;
|
||||
build-project moses-chart-cmd/src ;
|
||||
#Scripts have their own binaries
|
||||
|
||||
@ -40,3 +40,5 @@ exe evaluator : evaluator.cpp mert_lib ;
|
||||
exe pro : pro.cpp mert_lib ..//boost_program_options ;
|
||||
|
||||
alias programs : mert extractor evaluator pro ;
|
||||
|
||||
install legacy : programs : <location>. ;
|
||||
|
||||
208
scripts/Jamfile
208
scripts/Jamfile
@ -3,17 +3,6 @@ import option ;
|
||||
|
||||
build-project training ;
|
||||
|
||||
location = [ option.get "install-scripts" : $(TOP)/scripts/dist ] ;
|
||||
location = $(location)$(GITTAG) ;
|
||||
|
||||
#These two used to live in a tools directory.
|
||||
install ghkm : training/phrase-extract/extract-ghkm//extract-ghkm : <location>$(location)/training/phrase-extract/extract-ghkm/tools ;
|
||||
install compactify : training/compact-rule-table//compactify : <location>$(location)/training/compact-rule-table/tools ;
|
||||
|
||||
install phrase-extract : training/phrase-extract//released-programs : <location>$(location)/training/phrase-extract ;
|
||||
install lexical-reordering : training/lexical-reordering//score : <location>$(location)/training/lexical-reordering ;
|
||||
install symal : training/symal//symal : <location>$(location)/symal ;
|
||||
|
||||
with-giza = [ option.get "with-giza" ] ;
|
||||
if $(with-giza) {
|
||||
rule check-for-bin ( name ) {
|
||||
@ -30,111 +19,104 @@ if $(with-giza) {
|
||||
check-for-bin GIZA++ ;
|
||||
check-for-bin snt2cooc.out ;
|
||||
check-for-bin mkcls ;
|
||||
|
||||
install train-model : training//train-model.perl : <location>$(location)/training ;
|
||||
} else {
|
||||
echo "If you want scripts/training/train-model.perl, pass --with-giza=/path/to/giza" ;
|
||||
constant WITH-GIZA : "no" ;
|
||||
alias train-model ;
|
||||
}
|
||||
|
||||
install scripts :
|
||||
analysis/README
|
||||
analysis/sentence-by-sentence.pl
|
||||
ems/experiment.machines
|
||||
ems/experiment.meta
|
||||
ems/experiment.perl
|
||||
ems/example/config.basic
|
||||
ems/example/config.factored
|
||||
ems/example/config.hierarchical
|
||||
ems/example/config.syntax
|
||||
ems/example/config.toy
|
||||
ems/example/data/nc-5k.en
|
||||
ems/example/data/nc-5k.fr
|
||||
ems/example/data/test-ref.en.sgm
|
||||
ems/example/data/test-src.fr.sgm
|
||||
ems/support/analysis.perl
|
||||
ems/support/berkeley-process.sh
|
||||
ems/support/berkeley-train.sh
|
||||
ems/support/consolidate-training-data.perl
|
||||
ems/support/generic-multicore-parallelizer.perl
|
||||
ems/support/generic-parallelizer.perl
|
||||
ems/support/input-from-sgm.perl
|
||||
ems/support/interpolate-lm.perl
|
||||
ems/support/reference-from-sgm.perl
|
||||
ems/support/remove-segmenation-markup.perl
|
||||
ems/support/report-experiment-scores.perl
|
||||
ems/support/reuse-weights.perl
|
||||
ems/support/run-command-on-multiple-refsets.perl
|
||||
ems/support/wrap-xml.perl
|
||||
ems/web/analysis.php
|
||||
ems/web/analysis_diff.php
|
||||
ems/web/comment.php
|
||||
ems/web/diff.php
|
||||
ems/web/index.php
|
||||
ems/web/lib.php
|
||||
ems/web/overview.php
|
||||
ems/web/setup
|
||||
ems/web/javascripts/builder.js
|
||||
ems/web/javascripts/controls.js
|
||||
ems/web/javascripts/dragdrop.js
|
||||
ems/web/javascripts/effects.js
|
||||
ems/web/javascripts/prototype.js
|
||||
ems/web/javascripts/scriptaculous.js
|
||||
ems/web/javascripts/slider.js
|
||||
ems/web/javascripts/sound.js
|
||||
ems/web/javascripts/unittest.js
|
||||
generic/compound-splitter.perl
|
||||
generic/extract-factors.pl
|
||||
generic/lopar2pos.pl
|
||||
generic/moses-parallel.pl
|
||||
generic/mteval-v12.pl
|
||||
generic/multi-bleu.perl
|
||||
generic/qsub-wrapper.pl
|
||||
README
|
||||
tokenizer/detokenizer.perl
|
||||
tokenizer/tokenizer.perl
|
||||
tokenizer/lowercase.perl
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.ru
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.ro
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.de
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.fr
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.el
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.is
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.pt
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.nl
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.it
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.sl
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.sk
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.pl
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.sv
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.es
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.en
|
||||
tokenizer/nonbreaking_prefixes/nonbreaking_prefix.ca
|
||||
training/absolutize_moses_model.pl
|
||||
training/build-generation-table.perl
|
||||
training/clean-corpus-n.perl
|
||||
training/clone_moses_model.pl
|
||||
training/filter-model-given-input.pl
|
||||
training/filter-rule-table.py
|
||||
training/zmert-moses.pl
|
||||
training/mert-moses.pl
|
||||
training/mert-moses-multi.pl
|
||||
training/postprocess-lopar.perl
|
||||
training/reduce_combine.pl
|
||||
training/combine_factors.pl
|
||||
training/symal/giza2bal.pl
|
||||
training/wrappers/parse-de-bitpar.perl
|
||||
training/wrappers/parse-en-collins.perl
|
||||
training/wrappers/make-factor-en-pos.mxpost.perl
|
||||
training/wrappers/make-factor-pos.tree-tagger.perl
|
||||
training/wrappers/make-factor-stem.perl
|
||||
recaser/train-recaser.perl
|
||||
recaser/recase.perl
|
||||
recaser/truecase.perl
|
||||
recaser/detruecase.perl
|
||||
recaser/train-truecaser.perl
|
||||
: <install-source-root>. <location>$(location) ;
|
||||
location = [ option.get "install-scripts" ] ;
|
||||
if $(location) {
|
||||
location = $(location)$(GITTAG) ;
|
||||
|
||||
alias install : ghkm compactify phrase-extract lexical-reordering symal scripts train-model ;
|
||||
explicit install ghkm compactify phrase-extract lexical-reordering symal scripts train-model ;
|
||||
#These two used to live in a tools directory.
|
||||
install ghkm : training/phrase-extract/extract-ghkm//extract-ghkm : <location>$(location)/training/phrase-extract/extract-ghkm/tools ;
|
||||
install compactify : training/compact-rule-table//compactify : <location>$(location)/training/compact-rule-table/tools ;
|
||||
|
||||
install phrase-extract : training/phrase-extract//released-programs : <location>$(location)/training/phrase-extract ;
|
||||
install lexical-reordering : training/lexical-reordering//score : <location>$(location)/training/lexical-reordering ;
|
||||
install symal : training/symal//symal : <location>$(location)/symal ;
|
||||
|
||||
if $(WITH-GIZA) != no {
|
||||
install train-model : training//train-model.perl : <location>$(location)/training ;
|
||||
} else {
|
||||
alias train-model ;
|
||||
}
|
||||
|
||||
install scripts :
|
||||
analysis/README
|
||||
analysis/sentence-by-sentence.pl
|
||||
ems/experiment.machines
|
||||
ems/experiment.meta
|
||||
ems/experiment.perl
|
||||
ems/example/config.basic
|
||||
ems/example/config.factored
|
||||
ems/example/config.hierarchical
|
||||
ems/example/config.syntax
|
||||
ems/example/config.toy
|
||||
ems/example/data/nc-5k.en
|
||||
ems/example/data/nc-5k.fr
|
||||
ems/example/data/test-ref.en.sgm
|
||||
ems/example/data/test-src.fr.sgm
|
||||
ems/support/analysis.perl
|
||||
ems/support/berkeley-process.sh
|
||||
ems/support/berkeley-train.sh
|
||||
ems/support/consolidate-training-data.perl
|
||||
ems/support/generic-multicore-parallelizer.perl
|
||||
ems/support/generic-parallelizer.perl
|
||||
ems/support/input-from-sgm.perl
|
||||
ems/support/interpolate-lm.perl
|
||||
ems/support/reference-from-sgm.perl
|
||||
ems/support/remove-segmenation-markup.perl
|
||||
ems/support/report-experiment-scores.perl
|
||||
ems/support/reuse-weights.perl
|
||||
ems/support/run-command-on-multiple-refsets.perl
|
||||
ems/support/wrap-xml.perl
|
||||
ems/web/analysis.php
|
||||
ems/web/analysis_diff.php
|
||||
ems/web/comment.php
|
||||
ems/web/diff.php
|
||||
ems/web/index.php
|
||||
ems/web/lib.php
|
||||
ems/web/overview.php
|
||||
ems/web/setup
|
||||
ems/web/javascripts/builder.js
|
||||
ems/web/javascripts/controls.js
|
||||
ems/web/javascripts/dragdrop.js
|
||||
ems/web/javascripts/effects.js
|
||||
ems/web/javascripts/prototype.js
|
||||
ems/web/javascripts/scriptaculous.js
|
||||
ems/web/javascripts/slider.js
|
||||
ems/web/javascripts/sound.js
|
||||
ems/web/javascripts/unittest.js
|
||||
generic/compound-splitter.perl
|
||||
generic/extract-factors.pl
|
||||
generic/lopar2pos.pl
|
||||
generic/moses-parallel.pl
|
||||
generic/mteval-v12.pl
|
||||
generic/multi-bleu.perl
|
||||
generic/qsub-wrapper.pl
|
||||
README
|
||||
[ glob tokenizer/*.perl tokenizer/nonbreaking_prefixes/* ]
|
||||
training/absolutize_moses_model.pl
|
||||
training/build-generation-table.perl
|
||||
training/clean-corpus-n.perl
|
||||
training/clone_moses_model.pl
|
||||
training/filter-model-given-input.pl
|
||||
training/filter-rule-table.py
|
||||
training/zmert-moses.pl
|
||||
training/mert-moses.pl
|
||||
training/mert-moses-multi.pl
|
||||
training/postprocess-lopar.perl
|
||||
training/reduce_combine.pl
|
||||
training/combine_factors.pl
|
||||
training/symal/giza2bal.pl
|
||||
training/wrappers/parse-de-bitpar.perl
|
||||
training/wrappers/parse-en-collins.perl
|
||||
training/wrappers/make-factor-en-pos.mxpost.perl
|
||||
training/wrappers/make-factor-pos.tree-tagger.perl
|
||||
training/wrappers/make-factor-stem.perl
|
||||
[ glob recaser/*.perl ]
|
||||
: <install-source-root>. <location>$(location) ;
|
||||
|
||||
alias install : ghkm compactify phrase-extract lexical-reordering symal scripts train-model ;
|
||||
}
|
||||
|
||||
@ -1,40 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
from=$1
|
||||
to=$2
|
||||
|
||||
logdir=./revision_status_log
|
||||
|
||||
if [ "$from" == "" ] || [ "$to" == "" ]; then
|
||||
cat <<KONEC
|
||||
./validate_more_revisions.sh <fromrev> <torev>
|
||||
will run ./validate_revision.sh for a sequence of revisions and
|
||||
collect logfiles to "$logdir"
|
||||
KONEC
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
mkdir -p $logdir
|
||||
|
||||
tmpdir=/tmp/validate-more-revisions-tmp
|
||||
if [ -e $tmpdir ]; then
|
||||
echo "$0 seems to be already running!"
|
||||
echo "If this is a false alarm, remove our temp directory:"
|
||||
echo " rm -rf $tmpdir"
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
mkdir -p $tmpdir
|
||||
|
||||
# need to save a copy of the helper script validate_revision.sh,
|
||||
# because previous releases might have missed it
|
||||
cp ./validate_revision.sh $tmpdir/
|
||||
|
||||
for i in `seq $from $to`; do
|
||||
echo "Validating $i...";
|
||||
$tmpdir/validate_revision.sh $i > $logdir/$i.log 2>&1
|
||||
tail -1 $logdir/$i.log | sed 's/^/ /'
|
||||
done
|
||||
rm -rf $tmpdir
|
||||
echo "Finished validating, now at revision $to"
|
||||
@ -1,50 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
rev="$1"
|
||||
|
||||
if [ "$rev" == "" ]; then
|
||||
cat << KONEC
|
||||
./validate_revision.sh <revnumber>
|
||||
This will check, if the given revision was compilable (using irstlm).
|
||||
These tasks will be performed:
|
||||
svn update -r <revnumber>
|
||||
compile and install irstlm to a temp directory
|
||||
compile moses with irstlm
|
||||
delete the temp directory
|
||||
KONEC
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
tempdir=/tmp/validatemoses
|
||||
|
||||
function die() {
|
||||
rm -rf $tempdir
|
||||
echo "$@"
|
||||
exit 1
|
||||
}
|
||||
|
||||
if svn status | grep '^[^\?]'; then
|
||||
die "Will not go to a different revision, please synchronize with a revision in repository first"
|
||||
fi
|
||||
|
||||
svn up -r $rev || die "Failed to update to rev. $rev"
|
||||
# dump the information
|
||||
svn info
|
||||
|
||||
./regenerate-makefiles.sh || die "Failed to regenerate makefiles in mosesdecoder"
|
||||
|
||||
|
||||
cd irstlm || die "Failed to chdir to irstlm"
|
||||
./regenerate-makefiles.sh || die "Failed to regenerate makefiles in irstlm"
|
||||
./configure --prefix=$tempdir/irstlm || die "Failed to configure irstlm"
|
||||
make clean || die "Failed to clean irstlm"
|
||||
make || die "Failed to compile irstlm"
|
||||
make install || die "Failed to install irstlm"
|
||||
cd ..
|
||||
|
||||
./configure --with-irstlm=$tempdir/irstlm || die "Failed to configure moses"
|
||||
make clean || die "Failed to clean moses"
|
||||
make || die "Failed to compile moses"
|
||||
|
||||
rm -rf $tempdir || die "Failed to remove tempdir $tempdir"
|
||||
echo "Moses successfully compiled"
|
||||
Loading…
Reference in New Issue
Block a user