mirror of
https://github.com/Helsinki-NLP/OPUS-MT-train.git
synced 2024-09-11 20:27:19 +03:00
fixed model names
This commit is contained in:
parent
2887762198
commit
f97bc1895c
11
Makefile
11
Makefile
@ -181,6 +181,17 @@ finished:
|
|||||||
echo "${WORKDIR}/${MODEL_BASENAME} unfinished"; \
|
echo "${WORKDIR}/${MODEL_BASENAME} unfinished"; \
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
## remove job files if no trained file exists
|
||||||
|
delete-broken-submit:
|
||||||
|
for l in ${ALL_LANG_PAIRS}; do \
|
||||||
|
if [ -e ${WORKHOME}/$$l/train.submit ]; then \
|
||||||
|
if [ ! `find ${WORKHOME}/$$l -name '*.${PRE_SRC}-${PRE_TRG}.*.best-perplexity.npz' | wc -l` -gt 0 ]; then \
|
||||||
|
echo "rm -f ${WORKHOME}/$$l/train.submit"; \
|
||||||
|
rm -f ${WORKHOME}/$$l/train.submit; \
|
||||||
|
fi \
|
||||||
|
fi \
|
||||||
|
done
|
||||||
|
|
||||||
|
|
||||||
## resume training on an existing model
|
## resume training on an existing model
|
||||||
resume:
|
resume:
|
||||||
|
@ -380,3 +380,29 @@ link-old-models:
|
|||||||
rm -f ${MODEL_TRANSLATE}.eval
|
rm -f ${MODEL_TRANSLATE}.eval
|
||||||
rm -f ${MODEL_TRANSLATE}.compare
|
rm -f ${MODEL_TRANSLATE}.compare
|
||||||
|
|
||||||
|
|
||||||
|
ifneq (${DATASET},${OLDDATASET})
|
||||||
|
TRAINFILES = ${wildcard ${WORKDIR}/train/*${OLDDATASET}*.*}
|
||||||
|
MODELFILES = ${wildcard ${WORKDIR}/*${OLDDATASET}*.*}
|
||||||
|
DECODERFILES = ${wildcard ${WORKDIR}/*${OLDDATASET}*.decoder.yml}
|
||||||
|
endif
|
||||||
|
|
||||||
|
|
||||||
|
## fix model names from old style
|
||||||
|
## where models trained on a single corpus got the name
|
||||||
|
## of that corpus
|
||||||
|
## Now: always use 'opus' as the name of the default dataset
|
||||||
|
|
||||||
|
fix-model-names:
|
||||||
|
ifneq (${DATASET},${OLDDATASET})
|
||||||
|
for f in ${DECODERFILES}; do \
|
||||||
|
perl -i.bak -pe 's/${OLDDATASET}/${DATASET}/' $$f; \
|
||||||
|
done
|
||||||
|
for f in ${TRAINFILES}; do \
|
||||||
|
mv -f $$f `echo $$f | sed 's/${OLDDATASET}/${DATASET}/'`; \
|
||||||
|
ln -s `echo $$f | sed 's/${OLDDATASET}/${DATASET}/'` $$f; \
|
||||||
|
done
|
||||||
|
for f in ${MODELFILES}; do \
|
||||||
|
mv -f $$f `echo $$f | sed 's/${OLDDATASET}/${DATASET}/'`; \
|
||||||
|
done
|
||||||
|
endif
|
||||||
|
@ -20,7 +20,7 @@
|
|||||||
## - make dist-allmodels
|
## - make dist-allmodels
|
||||||
%-allmodels:
|
%-allmodels:
|
||||||
for l in ${ALL_LANG_PAIRS}; do \
|
for l in ${ALL_LANG_PAIRS}; do \
|
||||||
if [ `find ${WORKHOME}/$$l -name '*.${PRE_SRC}-${PRE_TRG}.*.npz' | wc -l` -gt 0 ]; then \
|
if [ `find ${WORKHOME}/$$l -name '*.${PRE_SRC}-${PRE_TRG}.*.best-perplexity.npz' | wc -l` -gt 0 ]; then \
|
||||||
${MAKE} SRCLANGS="`echo $$l | cut -f1 -d'-' | sed 's/\\+/ /g'`" \
|
${MAKE} SRCLANGS="`echo $$l | cut -f1 -d'-' | sed 's/\\+/ /g'`" \
|
||||||
TRGLANGS="`echo $$l | cut -f2 -d'-' | sed 's/\\+/ /g'`" ${@:-allmodels=}; \
|
TRGLANGS="`echo $$l | cut -f2 -d'-' | sed 's/\\+/ /g'`" ${@:-allmodels=}; \
|
||||||
fi \
|
fi \
|
||||||
@ -29,7 +29,7 @@
|
|||||||
## only bilingual models
|
## only bilingual models
|
||||||
%-allbilingual:
|
%-allbilingual:
|
||||||
for l in ${ALL_BILINGUAL_MODELS}; do \
|
for l in ${ALL_BILINGUAL_MODELS}; do \
|
||||||
if [ `find ${WORKHOME}/$$l -name '*.${PRE_SRC}-${PRE_TRG}.*.npz' | wc -l` -gt 0 ]; then \
|
if [ `find ${WORKHOME}/$$l -name '*.${PRE_SRC}-${PRE_TRG}.*.best-perplexity.npz' | wc -l` -gt 0 ]; then \
|
||||||
${MAKE} SRCLANGS="`echo $$l | cut -f1 -d'-' | sed 's/\\+/ /g'`" \
|
${MAKE} SRCLANGS="`echo $$l | cut -f1 -d'-' | sed 's/\\+/ /g'`" \
|
||||||
TRGLANGS="`echo $$l | cut -f2 -d'-' | sed 's/\\+/ /g'`" ${@:-allbilingual=}; \
|
TRGLANGS="`echo $$l | cut -f2 -d'-' | sed 's/\\+/ /g'`" ${@:-allbilingual=}; \
|
||||||
fi \
|
fi \
|
||||||
@ -38,7 +38,7 @@
|
|||||||
## only bilingual models
|
## only bilingual models
|
||||||
%-allmultilingual:
|
%-allmultilingual:
|
||||||
for l in ${ALL_MULTILINGUAL_MODELS}; do \
|
for l in ${ALL_MULTILINGUAL_MODELS}; do \
|
||||||
if [ `find ${WORKHOME}/$$l -name '*.${PRE_SRC}-${PRE_TRG}.*.npz' | wc -l` -gt 0 ]; then \
|
if [ `find ${WORKHOME}/$$l -name '*.${PRE_SRC}-${PRE_TRG}.*.best-perplexity.npz' | wc -l` -gt 0 ]; then \
|
||||||
${MAKE} SRCLANGS="`echo $$l | cut -f1 -d'-' | sed 's/\\+/ /g'`" \
|
${MAKE} SRCLANGS="`echo $$l | cut -f1 -d'-' | sed 's/\\+/ /g'`" \
|
||||||
TRGLANGS="`echo $$l | cut -f2 -d'-' | sed 's/\\+/ /g'`" ${@:-allmultilingual=}; \
|
TRGLANGS="`echo $$l | cut -f2 -d'-' | sed 's/\\+/ /g'`" ${@:-allmultilingual=}; \
|
||||||
fi \
|
fi \
|
||||||
@ -74,10 +74,12 @@
|
|||||||
BT_MODEL = ${MODEL_SUBDIR}opus+bt${TRAINSIZE}.${PRE_SRC}-${PRE_TRG}
|
BT_MODEL = ${MODEL_SUBDIR}opus+bt${TRAINSIZE}.${PRE_SRC}-${PRE_TRG}
|
||||||
BT_MODEL_BASE = ${BT_MODEL}.${MODELTYPE}.model${NR}
|
BT_MODEL_BASE = ${BT_MODEL}.${MODELTYPE}.model${NR}
|
||||||
BT_MODEL_START = ${WORKDIR}/${BT_MODEL_BASE}.npz
|
BT_MODEL_START = ${WORKDIR}/${BT_MODEL_BASE}.npz
|
||||||
|
BT_MODEL_VOCAB = ${WORKDIR}/${BT_MODEL}.vocab.${MODEL_VOCABTYPE}
|
||||||
|
|
||||||
%-add-backtranslations:
|
%-add-backtranslations:
|
||||||
ifneq (${wildcard ${MODEL_FINAL}},)
|
ifneq (${wildcard ${MODEL_FINAL}},)
|
||||||
cp ${MODEL_FINAL} ${BT_MODEL_START}
|
cp ${MODEL_FINAL} ${BT_MODEL_START}
|
||||||
|
cp ${MODEL_VOCAB} ${BT_MODEL_VOCAB}
|
||||||
endif
|
endif
|
||||||
${MAKE} DATASET=opus+bt \
|
${MAKE} DATASET=opus+bt \
|
||||||
CLEAN_TRAIN_SRC="${CLEAN_TRAIN_SRC} ${BACKTRANS_SRC}" \
|
CLEAN_TRAIN_SRC="${CLEAN_TRAIN_SRC} ${BACKTRANS_SRC}" \
|
||||||
|
@ -24,7 +24,7 @@ endif
|
|||||||
echo '#SBATCH -o ${DATASET}-${@:.submit=}.out.%j' >> $@
|
echo '#SBATCH -o ${DATASET}-${@:.submit=}.out.%j' >> $@
|
||||||
echo '#SBATCH -e ${DATASET}-${@:.submit=}.err.%j' >> $@
|
echo '#SBATCH -e ${DATASET}-${@:.submit=}.err.%j' >> $@
|
||||||
echo '#SBATCH --mem=${HPC_MEM}' >> $@
|
echo '#SBATCH --mem=${HPC_MEM}' >> $@
|
||||||
# echo '#SBATCH --exclude=r18g05' >> $@
|
echo '#SBATCH --exclude=r18g08' >> $@
|
||||||
ifdef EMAIL
|
ifdef EMAIL
|
||||||
echo '#SBATCH --mail-type=END' >> $@
|
echo '#SBATCH --mail-type=END' >> $@
|
||||||
echo '#SBATCH --mail-user=${EMAIL}' >> $@
|
echo '#SBATCH --mail-user=${EMAIL}' >> $@
|
||||||
|
Loading…
Reference in New Issue
Block a user