Helsinki-NLP/OPUS-MT-train
1
1
Fork 0
mirror of https://github.com/Helsinki-NLP/OPUS-MT-train.git synced 2024-11-27 11:03:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

fixes with names

Browse Source
This commit is contained in:
Joerg Tiedemann 2022-02-26 17:57:49 +02:00
parent e94c43062a
commit c421fbdb15
3 changed files with 50 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
lib/dist.mk
View File

@ -197,7 +197,8 @@ best-dist best_dist:
## old: only accept models with a certain evaluation score:
# if [ `grep BLEU $(TEST_EVALUATION) | cut -f3 -d ' ' | cut -f1 -d '.'` -ge ${MIN_BLEU_SCORE} ]; then \
MODELS_URL = https://object.pouta.csc.fi/${DEV_MODEL_CONTAINER}
# MODELS_URL ?= https://object.pouta.csc.fi/${DEV_MODEL_CONTAINER}
MODELS_URL ?= https://object.pouta.csc.fi/${MODEL_CONTAINER}
SKIP_DIST_EVAL = 0
@ -498,15 +499,15 @@ endif
@if [ -e ${RELEASE_PACKAGE} ]; then \
mkdir -p models-backup/${LANGPAIRSTR}/${DATE}; \
mv -f ${RELEASE_PACKAGE} models-backup/${LANGPAIRSTR}/${DATE}/; \
mv -f ${@:.zip=}-${DATE}.eval.txt models-backup/${LANGPAIRSTR}/${DATE}/; \
mv -f ${@:.zip=}-${DATE}.test.txt models-backup/${LANGPAIRSTR}/${DATE}/; \
mv -f ${@:.zip=}_${DATE}.eval.txt models-backup/${LANGPAIRSTR}/${DATE}/; \
mv -f ${@:.zip=}_${DATE}.test.txt models-backup/${LANGPAIRSTR}/${DATE}/; \
fi
@mv -f ${WORKDIR}/${notdir $@} ${RELEASE_PACKAGE}
@${MAKE} ${RELEASE_YML}
@${MAKE} ${RELEASE_README}
ifneq ($(wildcard ${TEST_EVALUATION}),)
@cp $(TEST_EVALUATION) ${@:.zip=}-${DATE}.eval.txt
@cp $(TEST_COMPARISON) ${@:.zip=}-${DATE}.test.txt
@cp $(TEST_EVALUATION) ${@:.zip=}_${DATE}.eval.txt
@cp $(TEST_COMPARISON) ${@:.zip=}_${DATE}.test.txt
endif
@rm -f $@
@cd ${dir $@} && ln -s $(notdir ${RELEASE_PACKAGE}) ${notdir $@}
@ -521,10 +522,12 @@ endif
## --> this is kind of dangerous as we may overwrite existing newer ones with older ones
## --> the reason for doing this is to update yml files and evaluation scores
# d=`realpath ${DIST_PACKAGE} | xargs basename | sed 's/^[^\-]*\-//;s/\.zip$$//'`; \
refresh-release:
if [[ ${DIST_PACKAGE} -nt ${MODEL_FINAL} ]]; then \
echo "updating ${shell realpath ${DIST_PACKAGE}}"; \
d=`realpath ${DIST_PACKAGE} | xargs basename | sed 's/^[^\-]*\-//;s/\.zip$$//'`; \
d=`realpath ${DIST_PACKAGE} | sed 's/^.*[\-\_]\(....\-..\-..\)\.zip$$/\1/'`; \
mkdir -p models-backup/${LANGPAIRSTR}/${DATE}; \
mv -f ${shell realpath ${DIST_PACKAGE}} models-backup/${LANGPAIRSTR}/${DATE}/; \
make DATE="$$d" release; \
@ -534,7 +537,7 @@ refresh-release-yml:
ifneq ("$(wildcard ${TEST_EVALUATION})","")
if [[ ${DIST_PACKAGE} -nt ${MODEL_FINAL} ]]; then \
echo "updating ${patsubst %.zip,%.yml,${shell realpath ${DIST_PACKAGE}}}"; \
d=`realpath ${DIST_PACKAGE} | xargs basename | sed 's/^[^\-]*\-//;s/\.zip$$//'`; \
d=`realpath ${DIST_PACKAGE} | sed 's/^.*[\-\_]\(....\-..\-..\)\.zip$$/\1/'`; \
if [ -e ${MODEL_YML} ]; then \
mv ${MODEL_YML} ${MODEL_YML}.${DATE}; \
fi; \
@ -549,7 +552,7 @@ refresh-release-readme:
ifneq ("$(wildcard ${TEST_EVALUATION})","")
if [[ ${DIST_PACKAGE} -nt ${MODEL_FINAL} ]]; then \
echo "updating ${LANGPAIRSTR}/README.md for ${notdir ${shell realpath ${DIST_PACKAGE}}}"; \
d=`realpath ${DIST_PACKAGE} | xargs basename | sed 's/^[^\-]*\-//;s/\.zip$$//'`; \
d=`realpath ${DIST_PACKAGE} | sed 's/^.*[\-\_]\(....\-..\-..\)\.zip$$/\1/'`; \
if [ -e ${MODEL_README} ]; then \
mv ${MODEL_README} ${MODEL_README}.${DATE}; \
fi; \

13
lib/projects/elg.mk
View File

@ -141,6 +141,12 @@ elg-zle2eng:
DATA_PREPARE_HPCPARAMS='CPUJOB_HPC_CORES=2 CPUJOB_HPC_MEM=16g CPUJOB_HPC_DISK=1000' \
tatoeba-zle2eng-trainjob-bt
elg-zls2eng:
${MAKE} MODELTYPE=transformer-big \
MARIAN_EXTRA=--no-restore-corpus \
DATA_PREPARE_HPCPARAMS='CPUJOB_HPC_CORES=2 CPUJOB_HPC_MEM=16g CPUJOB_HPC_DISK=1000' \
tatoeba-zls2eng-trainjob-bt
elg-multi2eng:
${MAKE} MODELTYPE=transformer-big \
MARIAN_EXTRA=--no-restore-corpus \
@ -191,3 +197,10 @@ elg-eng2fra:
DATA_PREPARE_HPCPARAMS='CPUJOB_HPC_CORES=2 CPUJOB_HPC_MEM=16g CPUJOB_HPC_DISK=1000' \
tatoeba-job-bt
elg-eng2fin:
${MAKE} MODELTYPE=transformer-big \
MARIAN_EXTRA=--no-restore-corpus \
tatoeba-eng2fin-trainjob-bt

26
tatoeba/Makefile
View File

@ -1105,3 +1105,29 @@ tatoeba-trainsize-%.txt: tatoeba-%.md
## fix names of eval/test files
# NEWMODELS = ${wildcard models/*/*_20*zip}
# EVAL_NEWMODELS = $(patsubst %.zip,%.eval.txt,${NEWMODELS})
# TEST_NEWMODELS = $(patsubst %.zip,%.test.txt,${NEWMODELS})
# fix-eval-files: ${EVAL_NEWMODELS} ${TEST_NEWMODELS}
# ${EVAL_NEWMODELS} ${TEST_NEWMODELS}:
# @( f=`echo $@ | sed 's/\_\(....\-..\-..\.[a-z]*.txt\)/-\1/'`; \
# if [ -e $$f ]; then \
# echo "mv $$f $@"; \
# mv $$f $@; \
# else \
# echo "$$f does not exist"; \
# fi )
# fix-refresh-readmes:
# for s in ell eus hbs ita lit; do \
# make MODELTYPE=transformer-big SRCLANGS=$$s TRGLANGS=eng refresh-release-readme-bt-tatoeba; \
# done
# for t in ara bul hun lit mlt ron tur; do \
# make MODELTYPE=transformer-big SRCLANGS=eng TRGLANGS=$$t refresh-release-readme-bt-tatoeba; \
# done