mirror of
https://github.com/Helsinki-NLP/OPUS-MT-train.git
synced 2024-09-19 08:07:21 +03:00
103 lines
3.7 KiB
Makefile
103 lines
3.7 KiB
Makefile
# -*-makefile-*-
|
|
|
|
#------------------------------------------------------------------------
|
|
# translate and evaluate all test sets in testsets/
|
|
#------------------------------------------------------------------------
|
|
|
|
## testset dir for all test sets in this language pair
|
|
## and all trokenized test sets that can be found in that directory
|
|
TESTSET_HOME = ${PWD}/testsets
|
|
TESTSET_DIR = ${TESTSET_HOME}/${SRC}-${TRG}
|
|
TESTSETS = $(sort $(patsubst ${TESTSET_DIR}/%.${SRCEXT}.gz,%,${wildcard ${TESTSET_DIR}/*.${SRCEXT}.gz}))
|
|
TESTSETS_PRESRC = $(patsubst %,${TESTSET_DIR}/%.${SRCEXT}.${PRE}.gz,${TESTSETS})
|
|
TESTSETS_PRETRG = $(patsubst %,${TESTSET_DIR}/%.${TRGEXT}.${PRE}.gz,${TESTSETS})
|
|
|
|
# TESTSETS_PRESRC = $(patsubst %.gz,%.${PRE}.gz,${sort $(subst .${PRE},,${wildcard ${TESTSET_DIR}/*.${SRC}.gz})})
|
|
# TESTSETS_PRETRG = $(patsubst %.gz,%.${PRE}.gz,${sort $(subst .${PRE},,${wildcard ${TESTSET_DIR}/*.${TRG}.gz})})
|
|
|
|
## eval all available test sets
|
|
eval-testsets:
|
|
for s in ${SRCLANGS}; do \
|
|
for t in ${TRGLANGS}; do \
|
|
${MAKE} SRC=$$s TRG=$$t compare-testsets-langpair; \
|
|
done \
|
|
done
|
|
|
|
%-testsets-langpair: ${TESTSETS_PRESRC} ${TESTSETS_PRETRG}
|
|
@echo "testsets: ${TESTSET_DIR}/*.${SRCEXT}.gz"
|
|
for t in ${TESTSETS}; do \
|
|
${MAKE} TESTSET=$$t TESTSET_NAME=$$t-${SRC}${TRG} ${@:-testsets-langpair=}; \
|
|
done
|
|
|
|
|
|
|
|
#------------------------------------------------------------------------
|
|
# translate with an ensemble of several models
|
|
#------------------------------------------------------------------------
|
|
|
|
ENSEMBLE = ${wildcard ${WORKDIR}/${MODEL}.${MODELTYPE}.model*.npz.best-perplexity.npz}
|
|
|
|
${WORKDIR}/${TESTSET_NAME}.${MODEL}${NR}.${MODELTYPE}.ensemble.${SRC}.${TRG}: ${TEST_SRC}.${PRE_SRC} ${ENSEMBLE}
|
|
mkdir -p ${dir $@}
|
|
grep . $< > $@.input
|
|
${LOAD_ENV} && ${MARIAN_DECODER} -i $@.input \
|
|
--models ${ENSEMBLE} \
|
|
--vocabs ${WORKDIR}/${MODEL}.vocab.yml \
|
|
${WORKDIR}/${MODEL}.vocab.yml \
|
|
${WORKDIR}/${MODEL}.vocab.yml \
|
|
${MARIAN_DECODER_FLAGS} > $@.output
|
|
ifeq (${PRE_TRG},spm${TRGBPESIZE:000=}k)
|
|
sed 's/ //g;s/▁/ /g' < $@.output | sed 's/^ *//;s/ *$$//' > $@
|
|
else
|
|
sed 's/\@\@ //g;s/ \@\@//g;s/ \@\-\@ /-/g' < $@.output |\
|
|
$(TOKENIZER)/detokenizer.perl -l ${TRG} > $@
|
|
endif
|
|
rm -f $@.input $@.output
|
|
|
|
|
|
#------------------------------------------------------------------------
|
|
# translate, evaluate and generate a file
|
|
# for comparing system to reference translations
|
|
#------------------------------------------------------------------------
|
|
|
|
${WORKDIR}/${TESTSET_NAME}.${MODEL}${NR}.${MODELTYPE}.${SRC}.${TRG}: ${TEST_SRC}.${PRE_SRC} ${MODEL_FINAL}
|
|
mkdir -p ${dir $@}
|
|
grep . $< > $@.input
|
|
${LOAD_ENV} && ${MARIAN_DECODER} -i $@.input \
|
|
-c ${word 2,$^}.decoder.yml \
|
|
-d ${MARIAN_GPUS} \
|
|
${MARIAN_DECODER_FLAGS} > $@.output
|
|
ifeq (${PRE_TRG},spm${TRGBPESIZE:000=}k)
|
|
sed 's/ //g;s/▁/ /g' < $@.output | sed 's/^ *//;s/ *$$//' > $@
|
|
else
|
|
sed 's/\@\@ //g;s/ \@\@//g;s/ \@\-\@ /-/g' < $@.output |\
|
|
$(TOKENIZER)/detokenizer.perl -l ${TRG} > $@
|
|
endif
|
|
rm -f $@.input $@.output
|
|
|
|
## adjust tokenisation to non-space-separated languages
|
|
## TODO: is it correct to simply use 'zh' or should we use 'intl'?
|
|
ifneq ($(filter zh zho jp jpn cmn,${TRGLANGS}),)
|
|
SACREBLEU_PARAMS = --tokenize zh
|
|
endif
|
|
|
|
%.eval: % ${TEST_TRG}
|
|
paste ${TEST_SRC}.${PRE_SRC} ${TEST_TRG} | grep $$'.\t' | cut -f2 > $@.ref
|
|
cat $< | sacrebleu ${SACREBLEU_PARAMS} $@.ref > $@
|
|
cat $< | sacrebleu ${SACREBLEU_PARAMS} --metrics=chrf --width=3 $@.ref >> $@
|
|
rm -f $@.ref
|
|
|
|
|
|
%.compare: %.eval
|
|
grep . ${TEST_SRC} > $@.1
|
|
grep . ${TEST_TRG} > $@.2
|
|
grep . ${<:.eval=} > $@.3
|
|
paste -d "\n" $@.1 $@.2 $@.3 |\
|
|
sed -e "s/'/'/g" \
|
|
-e 's/"/"/g' \
|
|
-e 's/</</g' \
|
|
-e 's/>/>/g' \
|
|
-e 's/&/&/g' |\
|
|
sed 'n;n;G;' > $@
|
|
rm -f $@.1 $@.2 $@.3
|