Helsinki-NLP/OPUS-MT-train
1
1
Fork 0
mirror of https://github.com/Helsinki-NLP/OPUS-MT-train.git synced 2024-11-27 11:03:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

changes on mahti

Browse Source
This commit is contained in:
Joerg Tiedemann 2022-04-27 10:59:39 +03:00
parent d45e7aaf7c
commit 220b2aefbe
4 changed files with 53 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
lib/projects/elg.mk
View File

@ -153,6 +153,13 @@ elg-gmq2ukr-tiny11:
STUDENT_DATA=pft-pbt-bt SRCLANGS="dan isl nno nob nor swe" TRGLANGS=ukr \ STUDENT_DATA=pft-pbt-bt SRCLANGS="dan isl nno nob nor swe" TRGLANGS=ukr \
LANGPAIRSTR="gmq-ukr" train-tiny11-student LANGPAIRSTR="gmq-ukr" train-tiny11-student
elg-gmq2ukr-small:
${MAKE} MARIAN_EXTRA=--no-restore-corpus \
MARIAN_EARLY_STOPPING=15 \
STUDENT_DATA=ftbest-ftmono-nopar \
SRCLANGS="dan nob swe" TRGLANGS=ukr \
LANGPAIRSTR="gmq-ukr" train-small-student
## tiny11 transformer model for finnish with pivot data (reuse student recipes) ## tiny11 transformer model for finnish with pivot data (reuse student recipes)
@ -240,6 +247,32 @@ elg-ukr2deu-student3:
${MAKE} MODELZIP=https://object.pouta.csc.fi/Tatoeba-MT-models/ukr-deu/opusTCv20210807_transformer-big_2022-03-14.zip \ SRCLANGS=ukr TRGLANGS=deu STUDENT_DATA=ftbest-bt-nopar train-tiny11-student ${MAKE} MODELZIP=https://object.pouta.csc.fi/Tatoeba-MT-models/ukr-deu/opusTCv20210807_transformer-big_2022-03-14.zip \ SRCLANGS=ukr TRGLANGS=deu STUDENT_DATA=ftbest-bt-nopar train-tiny11-student
elg-deu2ukr-student4:
${MAKE} MARIAN_EARLY_STOPPING=15 STUDENT_DATA=ftbest-ftmono-nopar SRCLANGS=deu TRGLANGS=ukr train-small-student
elg-ukr2deu-student4:
${MAKE} MARIAN_EARLY_STOPPING=15 SRCLANGS=ukr TRGLANGS=deu STUDENT_DATA=ftbest-ftmono-nopar train-small-student
elg-ukr2gmq-small:
${MAKE} MARIAN_EARLY_STOPPING=15 SRCLANGS=ukr TRGLANGS=swe STUDENT_DATA=ftbest-ftmono-nopar train-small-student
${MAKE} MARIAN_EARLY_STOPPING=15 SRCLANGS=ukr TRGLANGS=dan STUDENT_DATA=ftbest-ftmono-nopar train-small-student
${MAKE} MARIAN_EARLY_STOPPING=15 SRCLANGS=ukr TRGLANGS=nob STUDENT_DATA=ftbest-ftmono-nopar train-small-student
elg-dan2ukr-small:
${MAKE} MARIAN_EARLY_STOPPING=15 SRCLANGS=dan TRGLANGS=ukr STUDENT_DATA=ftbest-ftmono-nopar train-small-student
elg-swe2ukr-small:
${MAKE} MARIAN_EARLY_STOPPING=15 SRCLANGS=swe TRGLANGS=ukr STUDENT_DATA=ftbest-ftmono-nopar train-small-student
elg-nob2ukr-small:
${MAKE} MARIAN_EARLY_STOPPING=15 SRCLANGS=nob TRGLANGS=ukr STUDENT_DATA=ftbest-ftmono-nopar train-small-student
elg-fin2ukr-student2: elg-fin2ukr-student2:
${MAKE} SUBWORD_VOCAB_SIZE=16000 MARIAN_EARLY_STOPPING=15 SRCLANGS=fin TRGLANGS=ukr CHECK_TRAINDATA_SIZE=1 CLEAN_CORPUS_TRAINING_DATA=1 STUDENT_DATA=ftbest-ftmono-nopar train-tiny11-student ${MAKE} SUBWORD_VOCAB_SIZE=16000 MARIAN_EARLY_STOPPING=15 SRCLANGS=fin TRGLANGS=ukr CHECK_TRAINDATA_SIZE=1 CLEAN_CORPUS_TRAINING_DATA=1 STUDENT_DATA=ftbest-ftmono-nopar train-tiny11-student
@ -290,13 +323,21 @@ elg-ukr2eng-student:
elg-eng2ukr-student2: elg-eng2ukr-student2:
${MAKE} MARIAN_EARLY_STOPPING=15 STUDENT_DATA=ftbest-bt-nopar SRCLANGS=eng TRGLANGS=ukr train-tiny11-student ${MAKE} MARIAN_EARLY_STOPPING=15 STUDENT_DATA=ftbest-ftmono-nopar SRCLANGS=eng TRGLANGS=ukr train-tiny11-student
elg-ukr2eng-student2: elg-ukr2eng-student2:
${MAKE} CONTINUE_EXISTING=1 MARIAN_EARLY_STOPPING=15 SRCLANGS=ukr TRGLANGS=eng STUDENT_DATA=ftbest-ftmono-nopar train-tiny11-student ${MAKE} CONTINUE_EXISTING=1 MARIAN_EARLY_STOPPING=15 SRCLANGS=ukr TRGLANGS=eng STUDENT_DATA=ftbest-ftmono-nopar train-tiny11-student
elg-eng2ukr-student3:
${MAKE} MARIAN_EARLY_STOPPING=15 STUDENT_DATA=ftbest-ftmono-nopar SRCLANGS=eng TRGLANGS=ukr train-small-student
elg-ukr2eng-student3:
${MAKE} MARIAN_EARLY_STOPPING=15 SRCLANGS=ukr TRGLANGS=eng STUDENT_DATA=ftbest-ftmono-nopar train-small-student
## missing evaluations and dist packages ## missing evaluations and dist packages
## TODO: should probabubly also restart them! ## TODO: should probabubly also restart them!
## (also zls-zle and zle-zls) ## (also zls-zle and zle-zls)
@ -332,6 +373,9 @@ elg-fin2zle-pivot:
${MAKE} MARIAN_EXTRA=--no-restore-corpus MODELTYPE=transformer-big tatoeba-fin2zle-trainjob-pbt-pft-bt ${MAKE} MARIAN_EXTRA=--no-restore-corpus MODELTYPE=transformer-big tatoeba-fin2zle-trainjob-pbt-pft-bt
elg-zho:
${MAKE} MODELTYPE=transformer-big tatoeba-zho2eng-trainjob
elg-continue-missing: elg-continue-missing:
for l in deu fra ita por spa; do \ for l in deu fra ita por spa; do \
@ -431,6 +475,12 @@ elg-zle2eng-xb:
${MAKE} MARIAN_EARLY_STOPPING=25 MARIAN_EXTRA=--no-restore-corpus MODELTYPE=transformer-big CONTINUE_EXISTING=1 tatoeba-zle2eng-trainjob-bt-xb ${MAKE} MARIAN_EARLY_STOPPING=25 MARIAN_EXTRA=--no-restore-corpus MODELTYPE=transformer-big CONTINUE_EXISTING=1 tatoeba-zle2eng-trainjob-bt-xb
elg-fin2zle-xb:
${MAKE} MARIAN_EXTRA=--no-restore-corpus MODELTYPE=transformer-big CONTINUE_EXISTING=1 tatoeba-fin2zle-trainjob-pbt-pft-bt-xb
${MAKE} MARIAN_EXTRA=--no-restore-corpus MODELTYPE=transformer-big CONTINUE_EXISTING=1 tatoeba-zle2fin-trainjob-pbt-pft-bt-xb
elg-pivot-eval: elg-pivot-eval:
for l in dan swe fin deu ron tur; do \ for l in dan swe fin deu ron tur; do \
if [ -e work/$$l-ukr ]; then \ if [ -e work/$$l-ukr ]; then \

4
tatoeba/forward-translate/Makefile
View File

@ -150,8 +150,8 @@ translate: ${BITEXT_LATEST_README} ${BITEXT_LATEST_TRG}
${MAKE} ${BITEXT_LATEST_SRC} ${MAKE} ${BITEXT_LATEST_SRC}
## translate all parts ## translate all parts
.PHONY: translate-all-parts .PHONY: translate-all-parts translate-all
translate-all-parts: ${ALL_BITEXT_LATEST_TRG} translate-all translate-all-parts: ${ALL_BITEXT_LATEST_TRG}
${MAKE} source-all-parts ${MAKE} source-all-parts
.PHONY: source-all-parts .PHONY: source-all-parts

1
testsets/nor-ukr/flores101.nor.gz
View File

@ -1 +0,0 @@
../flores101/nor.devtest.gz

1
testsets/ukr-nor/flores101.nor.gz
View File

@ -1 +0,0 @@
../flores101/nor.devtest.gz