mirror of
https://github.com/Helsinki-NLP/OPUS-MT-train.git
synced 2024-09-19 08:07:21 +03:00
704 lines
23 KiB
Makefile
704 lines
23 KiB
Makefile
# -*-makefile-*-
|
|
#
|
|
# model configurations
|
|
#
|
|
|
|
|
|
## name of the model-specific configuration file
|
|
MODELCONFIG ?= config.mk
|
|
|
|
|
|
## various ways of setting the model languages
|
|
|
|
## (1) explicitly set source and target languages, for example:
|
|
## SRCLANGS="da no sv" TRGLANGS="fi da"
|
|
##
|
|
## (2) specify language pairs, for example:
|
|
## LANGPAIRS="de-en fi-sv da-es"
|
|
## this will set SRCLANGS="de fi da" TRGLANGS="en sv es"
|
|
##
|
|
|
|
|
|
## if LANGPAIRS are set and the model is not supposed to be SYMMETRIC
|
|
## then set SRCLANGS and TRGLANGS to the languages in LANGPAIRS
|
|
ifdef LANGPAIRS
|
|
SRCLANGS ?= ${sort ${shell echo "${LANGPAIRS}" | tr ' ' "\n" | cut -f1 -d '-'}}
|
|
TRGLANGS ?= ${sort ${shell echo "${LANGPAIRS}" | tr ' ' "\n" | cut -f2 -d '-'}}
|
|
endif
|
|
|
|
|
|
## set SRC and TRG unless they are specified already
|
|
ifneq (${words ${SRCLANGS}},1)
|
|
SRC ?= multi
|
|
else
|
|
SRC = ${SRCLANGS}
|
|
endif
|
|
ifneq (${words ${TRGLANGS}},1)
|
|
TRG ?= multi
|
|
else
|
|
TRG = ${TRGLANGS}
|
|
endif
|
|
|
|
|
|
## OLD: set to first and last lang
|
|
## --> this makes the evaluation look like it is one lang-pair
|
|
##
|
|
# SRC ?= ${firstword ${SRCLANGS}}
|
|
# TRG ?= ${lastword ${TRGLANGS}}
|
|
|
|
|
|
##----------------------------------------------------------------------
|
|
## SKIP_LANGPAIRS can be used to skip certain language pairs
|
|
## in data preparation for multilingual models
|
|
## ---> this can be good to skip BIG language pairs
|
|
## that would very much dominate all the data
|
|
## must be a pattern that can be matched by egrep
|
|
## e.g. en-de|en-fr
|
|
##
|
|
## SKIP_SAME_LANG - set to 1 to skip data with the same language
|
|
## on both sides
|
|
##----------------------------------------------------------------------
|
|
|
|
SKIP_LANGPAIRS ?= "nothing"
|
|
SKIP_SAME_LANG ?= 0
|
|
|
|
|
|
##----------------------------------------------------------------------
|
|
## set SHUFFLE_DATA if you want to shuffle data for
|
|
## each language pair to be added to the training data
|
|
## --> especially useful in connection with FIT_DATA_SIZE
|
|
##----------------------------------------------------------------------
|
|
|
|
# SHUFFLE_DATA = 1
|
|
|
|
## devtest data is shuffled by default
|
|
SHUFFLE_DEVDATA = 1
|
|
|
|
|
|
##----------------------------------------------------------------------
|
|
## set FIT_DATA_SIZE to a specific value to fit the training data
|
|
## to a certain number of lines for each language pair in the collection
|
|
## --> especially useful for multilingual models for balancing the
|
|
## the size for each language pair
|
|
## the script does both, over- and undersampling
|
|
##----------------------------------------------------------------------
|
|
|
|
# FIT_DATA_SIZE = 100000
|
|
|
|
## similar for the dev data: set FIT_DEVDATA_SIZE to
|
|
## balance the size of the devdata for each language pair
|
|
##
|
|
# FIT_DEVDATA_SIZE =
|
|
|
|
## define a default dev size fit for multilingual models
|
|
## TODO: is 1000 too small? or too big?
|
|
## TODO: should this depend on the number of languages involved?
|
|
|
|
ifneq (${words ${TRGLANGS}},1)
|
|
FIT_DEVDATA_SIZE ?= 1000
|
|
endif
|
|
ifneq (${words ${SRCLANGS}},1)
|
|
FIT_DEVDATA_SIZE ?= 1000
|
|
endif
|
|
|
|
## maximum number of repeating the same data set
|
|
## in oversampling
|
|
MAX_OVER_SAMPLING ?= 50
|
|
|
|
|
|
##----------------------------------------------------------------------
|
|
## set CHECK_TRAINDATA_SIZE if you want to check that each
|
|
## bitext has equal number of lines in source and target
|
|
## ---> this only prints a warning if not
|
|
##----------------------------------------------------------------------
|
|
# CHECK_TRAINDATA_SIZE
|
|
|
|
|
|
# sorted languages and langpair used to match resources in OPUS
|
|
SORTLANGS = $(sort ${SRC} ${TRG})
|
|
SORTSRC = ${firstword ${SORTLANGS}}
|
|
SORTTRG = ${lastword ${SORTLANGS}}
|
|
LANGPAIR = ${SORTSRC}-${SORTTRG}
|
|
SPACE = $(empty) $(empty)
|
|
LANGSRCSTR = ${subst ${SPACE},+,$(SRCLANGS)}
|
|
LANGTRGSTR = ${subst ${SPACE},+,$(TRGLANGS)}
|
|
LANGPAIRSTR = ${LANGSRCSTR}-${LANGTRGSTR}
|
|
|
|
|
|
## for monolingual things
|
|
LANGS ?= ${SRCLANGS}
|
|
LANGID ?= ${firstword ${LANGS}}
|
|
LANGSTR ?= ${subst ${SPACE},+,$(LANGS)}
|
|
|
|
|
|
## for same language pairs: add numeric extension
|
|
## (this is neccessary to keep source and target files separate)
|
|
ifeq (${SRC},$(TRG))
|
|
SRCEXT = ${SRC}1
|
|
TRGEXT = ${SRC}2
|
|
SORTSRCEXT = ${SORTSRC}1
|
|
SORTTRGEXT = ${SORTSRC}2
|
|
else
|
|
SRCEXT = ${SRC}
|
|
TRGEXT = ${TRG}
|
|
SORTSRCEXT = ${SORTSRC}
|
|
SORTTRGEXT = ${SORTTRG}
|
|
endif
|
|
|
|
## set a flag to use target language labels
|
|
## in multi-target models
|
|
ifneq (${words ${TRGLANGS}},1)
|
|
USE_TARGET_LABELS = 1
|
|
TARGET_LABELS ?= $(patsubst %,>>%<<,${TRGLANGS})
|
|
endif
|
|
|
|
|
|
## size of dev data, test data and BPE merge operations
|
|
## NEW default size = 2500 (keep more for training for small languages)
|
|
## NOTE: size will be increased to 5000 for Tatoeba
|
|
|
|
DEVSIZE = 2500
|
|
TESTSIZE = 2500
|
|
|
|
## set some additional thresholds for
|
|
## the size of test and dev data
|
|
## DEVMINSIZE is the absolute minimum we require
|
|
## to run any training procedures
|
|
|
|
DEVSMALLSIZE = 1000
|
|
TESTSMALLSIZE = 1000
|
|
DEVMINSIZE = 250
|
|
|
|
|
|
## set additional argument options for opus_read (if it is used)
|
|
## e.g. OPUSREAD_ARGS = -a certainty -tr 0.3
|
|
OPUSREAD_ARGS =
|
|
|
|
|
|
##----------------------------------------------------------------------------
|
|
## resources in OPUS
|
|
##----------------------------------------------------------------------------
|
|
|
|
|
|
## get available data from the OPUS-API
|
|
|
|
OPUSAPI = http://opus.nlpl.eu/opusapi/
|
|
OPUSAPI_WGET = wget -qq --no-check-certificate -O - ${OPUSAPI}?
|
|
|
|
get-opus-mono = ${shell ${OPUSAPI_WGET}source=${1}\&corpora=True | ${JQ} '.corpora[]' | tr '"' ' '}
|
|
get-opus-bitexts = ${shell ${OPUSAPI_WGET}source=${1}\&target=${2}\&corpora=True | ${JQ} '.corpora[]' | tr '"' ' '}
|
|
get-bigger-bitexts = ${shell ${OPUSAPI_WGET}source=${1}\&target=${2}\&preprocessing=xml\&version=latest | \
|
|
${JQ} -r '.corpora[1:] | .[] | select(.source!="") | select(.target!="") | select(.alignment_pairs>${3}) | .corpus' }
|
|
get-opus-langs = ${shell ${OPUSAPI_WGET}languages=True | ${JQ} '.languages[]' | tr '"' ' '}
|
|
get-opus-version = ${shell ${OPUSAPI_WGET}source=${1}\&target=${2}\&corpus=${3}\&preprocessing=xml\&version=latest | ${JQ} '.corpora[] | .version' | sed 's/"//g' | head -1}
|
|
get-elra-bitexts = ${shell ${OPUSAPI_WGET}source=${1}\&target=${2}\&corpora=True | \
|
|
${JQ} '.corpora[]' | tr '"' ' ' | grep '^ *ELR[CA][-_]'}
|
|
|
|
|
|
## start of some functions to check whether there is a resource for downloading
|
|
## open question: links to the latest release do not exist in the storage
|
|
## --> would it be better to get that done via the OPUS API?
|
|
|
|
OPUS_STORE = https://object.pouta.csc.fi/OPUS-
|
|
url-status = ${shell curl -Is -K HEAD ${1} | head -1}
|
|
url-exists = ${shell if [ "${call url-status,${1}}" == "HTTP/1.1 200 OK" ]; then echo 1; else echo 0; fi}
|
|
resource-url = ${shell echo "${OPUS_STORE}${3}/${call get-opus-version,${1},${2},${3}}/moses/${1}-${2}.txt.zip"}
|
|
|
|
|
|
## exclude certain data sets
|
|
# EXCLUDE_CORPORA ?= WMT-News MPC1 ${call get-elra-bitexts,${SRC},${TRG}}
|
|
EXCLUDE_CORPORA ?= WMT-News MPC1
|
|
|
|
# all matching corpora in OPUS except for some that we want to exclude
|
|
OPUSCORPORA = $(filter-out ${EXCLUDE_CORPORA},${call get-opus-bitexts,${SRC},${TRG}})
|
|
|
|
## monolingual data
|
|
OPUSMONOCORPORA = $(filter-out ${EXCLUDE_CORPORA},${call get-opus-mono,${LANGID}})
|
|
|
|
## all languages in OPUS
|
|
## TODO: do we need this?
|
|
OPUSLANGS := ${call get-opus-langs}
|
|
|
|
|
|
|
|
|
|
##----------------------------------------------------------------------------
|
|
## train/dev/test data
|
|
##----------------------------------------------------------------------------
|
|
|
|
|
|
## select a suitable DEVSET
|
|
## - POTENTIAL_DEVSETS lists more or less reliable corpora (in order of priority)
|
|
## - BIGGER_BITEXTS lists all bitext with more than DEVSMALLSIZE sentence pairs
|
|
## - SMALLER_BITEXTS lists potentially smaller bitexts but at least DEVMINSIZE big
|
|
## - DEVSET is the first of the potential devset that exists with sufficient size
|
|
|
|
POTENTIAL_DEVSETS = Tatoeba GlobalVoices infopankki JW300 bible-uedin
|
|
BIGGER_BITEXTS := ${call get-bigger-bitexts,${SRC},${TRG},${DEVSMALLSIZE}}
|
|
SMALLER_BITEXTS := ${call get-bigger-bitexts,${SRC},${TRG},${DEVMINSIZE}}
|
|
DEVSET ?= ${firstword ${filter ${POTENTIAL_DEVSETS},${BIGGER_BITEXTS}} \
|
|
${filter ${POTENTIAL_DEVSETS},${SMALLER_BITEXTS}}}
|
|
|
|
## why would we need foreach?
|
|
#DEVSET ?= ${firstword ${foreach c,${POTENTIAL_DEVSETS},${filter ${c},${BIGGER_BITEXTS}}} \
|
|
# ${foreach c,${POTENTIAL_DEVSETS},${filter ${c},${SMALLER_BITEXTS}}}}
|
|
|
|
|
|
|
|
## increase dev/test sets for Tatoeba (very short sentences!)
|
|
ifeq (${DEVSET},Tatoeba)
|
|
DEVSIZE = 5000
|
|
TESTSIZE = 5000
|
|
endif
|
|
|
|
|
|
## in case we want to use some additional data sets
|
|
# EXTRA_TRAINSET =
|
|
|
|
## TESTSET= DEVSET, TRAINSET = OPUS - WMT-News,DEVSET.TESTSET
|
|
TESTSET ?= ${DEVSET}
|
|
TRAINSET ?= $(filter-out ${EXCLUDE_CORPORA} ${DEVSET} ${TESTSET},${OPUSCORPORA} ${EXTRA_TRAINSET})
|
|
MONOSET ?= $(filter-out ${EXCLUDE_CORPORA} ${DEVSET} ${TESTSET},${OPUSMONOCORPORA} ${EXTRA_TRAINSET})
|
|
|
|
## 1 = use remaining data from dev/test data for training
|
|
USE_REST_DEVDATA ?= 1
|
|
|
|
|
|
|
|
## for model fine-tuning
|
|
|
|
TUNE_SRC ?= ${SRC}
|
|
TUNE_TRG ?= ${TRG}
|
|
|
|
TUNE_DOMAIN ?= OpenSubtitles
|
|
TUNE_FIT_DATA_SIZE ?= 1000000
|
|
|
|
TUNE_VALID_FREQ ?= 1000
|
|
TUNE_DISP_FREQ ?= 1000
|
|
TUNE_SAVE_FREQ ?= 1000
|
|
TUNE_EARLY_STOPPING ?= 5
|
|
TUNE_GPUJOB_SUBMIT ?=
|
|
|
|
|
|
|
|
## existing projects in WORKHOME
|
|
ALL_LANG_PAIRS := ${shell ls ${WORKHOME} 2>/dev/null | grep -- '-' | grep -v old}
|
|
ALL_BILINGUAL_MODELS := ${shell echo '${ALL_LANG_PAIRS}' | tr ' ' "\n" | grep -v -- '\+'}
|
|
ALL_MULTILINGUAL_MODELS := ${shell echo '${ALL_LANG_PAIRS}' | tr ' ' "\n" | grep -- '\+'}
|
|
|
|
|
|
##----------------------------------------------------------------------------
|
|
## pre-processing and vocabulary
|
|
##----------------------------------------------------------------------------
|
|
|
|
## type of subword segmentation (bpe|spm)
|
|
## model size (NOTE: BPESIZE is also used for sentencepiece!)
|
|
SUBWORDS ?= spm
|
|
BPESIZE ?= 32000
|
|
SRCBPESIZE ?= ${BPESIZE}
|
|
TRGBPESIZE ?= ${BPESIZE}
|
|
|
|
BPEMODELNAME ?= opus
|
|
|
|
BPESRCMODEL ?= ${WORKDIR}/train/${BPEMODELNAME}.src.bpe${SRCBPESIZE:000=}k-model
|
|
BPETRGMODEL ?= ${WORKDIR}/train/${BPEMODELNAME}.trg.bpe${TRGBPESIZE:000=}k-model
|
|
|
|
SPMSRCMODEL ?= ${WORKDIR}/train/${BPEMODELNAME}.src.spm${SRCBPESIZE:000=}k-model
|
|
SPMTRGMODEL ?= ${WORKDIR}/train/${BPEMODELNAME}.trg.spm${TRGBPESIZE:000=}k-model
|
|
|
|
## don't delete BPE/sentencepiece models!
|
|
.PRECIOUS: ${BPESRCMODEL} ${BPETRGMODEL}
|
|
.PRECIOUS: ${SPMSRCMODEL} ${SPMTRGMODEL}
|
|
|
|
## size of the joined vocabulary
|
|
## TODO: heuristically add 1,000 to cover language labels is a bit ad-hoc
|
|
VOCABSIZE ?= $$((${SRCBPESIZE} + ${TRGBPESIZE} + 1000))
|
|
|
|
## for document-level models
|
|
CONTEXT_SIZE = 100
|
|
|
|
## pre-processing type
|
|
# PRE = norm
|
|
PRE = simple
|
|
PRE_SRC = ${SUBWORDS}${SRCBPESIZE:000=}k
|
|
PRE_TRG = ${SUBWORDS}${TRGBPESIZE:000=}k
|
|
|
|
|
|
##-------------------------------------
|
|
## default name of the data set (and the model)
|
|
##-------------------------------------
|
|
|
|
TRAINSET_NAME ?= opus
|
|
DATASET ?= ${TRAINSET_NAME}
|
|
|
|
## dev and test data come from one specific data set
|
|
## if we have a bilingual model
|
|
|
|
ifeq (${words ${SRCLANGS}},1)
|
|
ifeq (${words ${TRGLANGS}},1)
|
|
DEVSET_NAME ?= ${DEVSET}
|
|
TESTSET_NAME ?= ${TESTSET}
|
|
endif
|
|
endif
|
|
|
|
## otherwise we give them a generic name
|
|
|
|
DEVSET_NAME ?= opus-dev
|
|
TESTSET_NAME ?= opus-test
|
|
|
|
|
|
## DATADIR = directory where the train/dev/test data are
|
|
## WORKDIR = directory used for training
|
|
|
|
DATADIR = ${WORKHOME}/data
|
|
WORKDIR = ${WORKHOME}/${LANGPAIRSTR}
|
|
MODELDIR = ${WORKHOME}/models/${LANGPAIRSTR}
|
|
SPMDIR = ${WORKHOME}/SentencePieceModels
|
|
|
|
## train data sets (word alignment for the guided alignment option)
|
|
TRAIN_BASE = ${WORKDIR}/train/${DATASET}
|
|
TRAIN_SRC = ${TRAIN_BASE}.src
|
|
TRAIN_TRG = ${TRAIN_BASE}.trg
|
|
TRAIN_ALG = ${TRAIN_BASE}${TRAINSIZE}.${PRE_SRC}-${PRE_TRG}.src-trg.alg.gz
|
|
|
|
## data sets that are pre-processed and ready to be used
|
|
TRAINDATA_SRC = ${TRAIN_SRC}.clean.${PRE_SRC}.gz
|
|
TRAINDATA_TRG = ${TRAIN_TRG}.clean.${PRE_TRG}.gz
|
|
DEVDATA_SRC = ${DEV_SRC}.${PRE_SRC}
|
|
DEVDATA_TRG = ${DEV_TRG}.${PRE_TRG}
|
|
TESTDATA_SRC = ${TEST_SRC}.${PRE_SRC}
|
|
TESTDATA_TRG = ${TEST_TRG}
|
|
|
|
## training data in local space
|
|
LOCAL_TRAIN_SRC = ${TMPDIR}/${LANGPAIRSTR}/train/${DATASET}.src
|
|
LOCAL_TRAIN_TRG = ${TMPDIR}/${LANGPAIRSTR}/train/${DATASET}.trg
|
|
LOCAL_MONO_DATA = ${TMPDIR}/${LANGSTR}/train/${DATASET}.mono
|
|
|
|
## dev and test data
|
|
DEV_SRC ?= ${WORKDIR}/val/${DEVSET_NAME}.src
|
|
DEV_TRG ?= ${WORKDIR}/val/${DEVSET_NAME}.trg
|
|
|
|
TEST_SRC ?= ${WORKDIR}/test/${TESTSET_NAME}.src
|
|
TEST_TRG ?= ${WORKDIR}/test/${TESTSET_NAME}.trg
|
|
|
|
|
|
## model basename and optional sub-dir
|
|
|
|
MODEL_SUBDIR =
|
|
MODEL = ${MODEL_SUBDIR}${DATASET}${TRAINSIZE}.${PRE_SRC}-${PRE_TRG}
|
|
|
|
|
|
## supported model types
|
|
## configuration for each type is in lib/train.mk
|
|
|
|
MODELTYPES = transformer \
|
|
transformer-big \
|
|
transformer-align \
|
|
transformer-big-align \
|
|
transformer-small-align \
|
|
transformer-tiny-align \
|
|
transformer-tiny
|
|
MODELTYPE = transformer-align
|
|
NR = 1
|
|
|
|
MODEL_BASENAME = ${MODEL}.${MODELTYPE}.model${NR}
|
|
MODEL_VALIDLOG = ${MODEL}.${MODELTYPE}.valid${NR}.log
|
|
MODEL_TRAINLOG = ${MODEL}.${MODELTYPE}.train${NR}.log
|
|
MODEL_START = ${WORKDIR}/${MODEL_BASENAME}.npz
|
|
MODEL_FINAL = ${WORKDIR}/${MODEL_BASENAME}.npz.best-perplexity.npz
|
|
MODEL_DECODER = ${MODEL_FINAL}.decoder.yml
|
|
|
|
## for sentence-piece models: get plain text vocabularies
|
|
## for others: extract vocabulary from training data with MarianNMT
|
|
## backwards compatibility: if there is already a vocab-file then use it
|
|
|
|
# ifeq (${SUBWORDS},spm)
|
|
# ifeq ($(wildcard ${WORKDIR}/${MODEL}.vocab.yml),)
|
|
# USE_SPM_VOCAB ?= 1
|
|
# endif
|
|
# endif
|
|
|
|
## use vocab from sentence piece instead of
|
|
## marian_vocab from training data
|
|
|
|
ifeq ($(USE_SPM_VOCAB),1)
|
|
MODEL_VOCAB = ${WORKDIR}/${MODEL}.vocab.yml
|
|
MODEL_SRCVOCAB = ${WORKDIR}/${MODEL}.src.vocab
|
|
MODEL_TRGVOCAB = ${WORKDIR}/${MODEL}.trg.vocab
|
|
else
|
|
MODEL_VOCAB = ${WORKDIR}/${MODEL}.vocab.yml
|
|
MODEL_SRCVOCAB = ${MODEL_VOCAB}
|
|
MODEL_TRGVOCAB = ${MODEL_VOCAB}
|
|
endif
|
|
|
|
|
|
|
|
## latest model with the same pre-processing but any data or modeltype
|
|
## except for models that include the string 'tuned4' (fine-tuned models)
|
|
## also allow models that are of the same type but with/without guided alignment
|
|
## --> this will be used if the flag CONTINUE_EXISTING is set on
|
|
|
|
# ifdef CONTINUE_EXISTING
|
|
ifeq (${CONTINUE_EXISTING},1)
|
|
MODEL_LATEST = $(firstword \
|
|
${shell ls -t ${WORKDIR}/*.${PRE_SRC}-${PRE_TRG}.*.model[0-9].npz \
|
|
${WORKDIR}/*.${PRE_SRC}-${PRE_TRG}.*.best-perplexity.npz \
|
|
2>/dev/null | grep -v 'tuned4' | \
|
|
egrep '${MODELTYPE}|${MODELTYPE}-align|${subst -align,,${MODELTYPE}}' })
|
|
MODEL_LATEST_VOCAB = $(shell echo "${MODEL_LATEST}" | \
|
|
sed 's|\.${PRE_SRC}-${PRE_TRG}\..*$$|.${PRE_SRC}-${PRE_TRG}.vocab.yml|')
|
|
MODEL_LATEST_OPTIMIZER = $(shell echo "${MODEL_LATEST}" | \
|
|
sed 's|.best-perplexity.npz|.optimizer.npz|')
|
|
endif
|
|
|
|
## old:
|
|
# ${shell ls -t ${WORKDIR}/*.${PRE_SRC}-${PRE_TRG}.*.best-perplexity.npz \
|
|
|
|
|
|
|
|
## test set translation and scores
|
|
|
|
TEST_TRANSLATION = ${WORKDIR}/${TESTSET_NAME}.${MODEL}${NR}.${MODELTYPE}.${SRC}.${TRG}
|
|
TEST_EVALUATION = ${TEST_TRANSLATION}.eval
|
|
TEST_COMPARISON = ${TEST_TRANSLATION}.compare
|
|
|
|
|
|
|
|
## parameters for running Marian NMT
|
|
|
|
MARIAN_GPUS ?= 0
|
|
MARIAN_EXTRA =
|
|
MARIAN_VALID_FREQ ?= 10000
|
|
MARIAN_SAVE_FREQ ?= ${MARIAN_VALID_FREQ}
|
|
MARIAN_DISP_FREQ ?= ${MARIAN_VALID_FREQ}
|
|
MARIAN_EARLY_STOPPING ?= 10
|
|
MARIAN_VALID_MINI_BATCH ?= 16
|
|
MARIAN_MAXI_BATCH ?= 500
|
|
MARIAN_DROPOUT ?= 0.1
|
|
MARIAN_MAX_LENGTH ?= 500
|
|
MARIAN_ENC_DEPTH ?= 6
|
|
MARIAN_DEC_DEPTH ?= 6
|
|
MARIAN_ATT_HEADS ?= 8
|
|
MARIAN_DIM_EMB ?= 512
|
|
|
|
|
|
|
|
## TODO: currently marianNMT crashes with workspace > 26000
|
|
## TODO: move this to individual env settings?
|
|
ifeq (${GPU},p100)
|
|
MARIAN_WORKSPACE = 13000
|
|
else ifeq (${GPU},a100)
|
|
MARIAN_WORKSPACE = 30000
|
|
else ifeq (${GPU},v100)
|
|
ifeq ($(subst -align,,${MODELTYPE}),transformer-big)
|
|
MARIAN_WORKSPACE = 20000
|
|
else ifeq ($(subst -align,,${MODELTYPE}),transformer-small)
|
|
MARIAN_WORKSPACE = 10000
|
|
else ifeq ($(subst -align,,${MODELTYPE}),transformer-tiny)
|
|
MARIAN_WORKSPACE = 10000
|
|
else
|
|
# MARIAN_WORKSPACE = 30000
|
|
# MARIAN_WORKSPACE = 26000
|
|
MARIAN_WORKSPACE = 24000
|
|
endif
|
|
else
|
|
MARIAN_WORKSPACE = 10000
|
|
endif
|
|
|
|
## check whether we have GPUs available
|
|
## if not: use CPU mode for decoding
|
|
ifneq ($(wildcard ${NVIDIA_SMI}),)
|
|
ifeq (${shell nvidia-smi | grep failed | wc -l},1)
|
|
MARIAN_DECODER_FLAGS = ${MARIAN_DECODER_CPU}
|
|
MARIAN_EXTRA = --cpu-threads ${HPC_CORES}
|
|
endif
|
|
else
|
|
MARIAN_DECODER_FLAGS = ${MARIAN_DECODER_CPU}
|
|
MARIAN_EXTRA = --cpu-threads ${HPC_CORES}
|
|
endif
|
|
|
|
## weights associated with training examples
|
|
ifneq ("$(wildcard ${TRAIN_WEIGHTS})","")
|
|
MARIAN_TRAIN_WEIGHTS = --data-weighting ${TRAIN_WEIGHTS}
|
|
endif
|
|
|
|
### training a model with Marian NMT
|
|
##
|
|
## NR allows to train several models for proper ensembling
|
|
## (with shared vocab)
|
|
##
|
|
## DANGER: if several models are started at the same time
|
|
## then there is some racing issue with creating the vocab!
|
|
|
|
ifdef NR
|
|
SEED=${NR}${NR}${NR}${NR}
|
|
else
|
|
SEED=1234
|
|
endif
|
|
|
|
|
|
## decoder flags (CPU and GPU variants)
|
|
|
|
MARIAN_DECODER_GPU = -b 4 -n1 -d ${MARIAN_GPUS} --quiet-translation -w ${MARIAN_WORKSPACE} \
|
|
--mini-batch 768 --maxi-batch 2048 --maxi-batch-sort src \
|
|
--max-length ${MARIAN_MAX_LENGTH} --max-length-crop --fp16
|
|
MARIAN_DECODER_CPU = -b 4 -n1 --cpu-threads ${HPC_CORES} --quiet-translation \
|
|
--mini-batch ${HPC_CORES} --maxi-batch 100 --maxi-batch-sort src \
|
|
--max-length ${MARIAN_MAX_LENGTH} --max-length-crop --fp16
|
|
MARIAN_DECODER_FLAGS = ${MARIAN_DECODER_GPU}
|
|
|
|
|
|
|
|
## make some data size-specific configuration parameters
|
|
## TODO: is it OK to delete LOCAL_TRAIN data?
|
|
|
|
.PHONY: config local-config
|
|
config local-config: ${WORKDIR}/${MODELCONFIG}
|
|
|
|
SMALLEST_TRAINSIZE = 10000
|
|
SMALL_TRAINSIZE = 100000
|
|
MEDIUM_TRAINSIZE = 500000
|
|
LARGE_TRAINSIZE = 1000000
|
|
LARGEST_TRAINSIZE = 10000000
|
|
|
|
${WORKDIR}/${MODELCONFIG}:
|
|
mkdir -p ${dir $@}
|
|
if [ -e ${TRAIN_SRC}.clean.${PRE_SRC}${TRAINSIZE}.gz ]; then \
|
|
${MAKE} ${TRAIN_SRC}.clean.${PRE_SRC}${TRAINSIZE}.charfreq \
|
|
${TRAIN_TRG}.clean.${PRE_TRG}${TRAINSIZE}.charfreq; \
|
|
s=`${ZCAT} ${TRAIN_SRC}.clean.${PRE_SRC}${TRAINSIZE}.gz | head -10000001 | wc -l`; \
|
|
S=`cat ${TRAIN_SRC}.clean.${PRE_SRC}${TRAINSIZE}.charfreq | wc -l`; \
|
|
T=`cat ${TRAIN_TRG}.clean.${PRE_TRG}${TRAINSIZE}.charfreq | wc -l`; \
|
|
else \
|
|
${MAKE} ${LOCAL_TRAIN_SRC}; \
|
|
${MAKE} ${LOCAL_TRAIN_SRC}.charfreq ${LOCAL_TRAIN_TRG}.charfreq; \
|
|
s=`head -10000001 ${LOCAL_TRAIN_SRC} | wc -l`; \
|
|
S=`cat ${LOCAL_TRAIN_SRC}.charfreq | wc -l`; \
|
|
T=`cat ${LOCAL_TRAIN_TRG}.charfreq | wc -l`; \
|
|
fi; \
|
|
if [ $$s -gt ${LARGEST_TRAINSIZE} ]; then \
|
|
echo "# ${LANGPAIRSTR} training data bigger than ${LARGEST_TRAINSIZE}" > $@; \
|
|
echo "GPUJOB_HPC_MEM = 8g" >> $@; \
|
|
echo "GPUJOB_SUBMIT = -multigpu" >> $@; \
|
|
echo "BPESIZE = ${BPESIZE}" >> $@; \
|
|
echo "DEVSIZE = ${DEVSIZE}" >> $@; \
|
|
echo "TESTSIZE = ${TESTSIZE}" >> $@; \
|
|
echo "DEVMINSIZE = ${DEVMINSIZE}" >> $@; \
|
|
elif [ $$s -gt ${LARGE_TRAINSIZE} ]; then \
|
|
echo "# ${LANGPAIRSTR} training data bigger than ${LARGE_TRAINSIZE}" > $@; \
|
|
echo "GPUJOB_HPC_MEM = 8g" >> $@; \
|
|
echo "GPUJOB_SUBMIT = " >> $@; \
|
|
echo "MARIAN_VALID_FREQ = 2500" >> $@; \
|
|
echo "BPESIZE = ${BPESIZE}" >> $@; \
|
|
echo "DEVSIZE = ${DEVSIZE}" >> $@; \
|
|
echo "TESTSIZE = ${TESTSIZE}" >> $@; \
|
|
echo "DEVMINSIZE = ${DEVMINSIZE}" >> $@; \
|
|
elif [ $$s -gt ${MEDIUM_TRAINSIZE} ]; then \
|
|
echo "# ${LANGPAIRSTR} training data bigger than ${MEDIUM_TRAINSIZE}" > $@; \
|
|
echo "GPUJOB_HPC_MEM = 4g" >> $@; \
|
|
echo "GPUJOB_SUBMIT = " >> $@; \
|
|
echo "MARIAN_VALID_FREQ = 2500" >> $@; \
|
|
echo "MARIAN_WORKSPACE = 10000" >> $@; \
|
|
echo "BPESIZE = 12000" >> $@; \
|
|
echo "DEVSIZE = ${DEVSIZE}" >> $@; \
|
|
echo "TESTSIZE = ${TESTSIZE}" >> $@; \
|
|
echo "DEVMINSIZE = ${DEVMINSIZE}" >> $@; \
|
|
elif [ $$s -gt ${SMALL_TRAINSIZE} ]; then \
|
|
echo "# ${LANGPAIRSTR} training data bigger than ${SMALL_TRAINSIZE}" > $@; \
|
|
echo "GPUJOB_HPC_MEM = 4g" >> $@; \
|
|
echo "GPUJOB_SUBMIT = " >> $@; \
|
|
echo "MARIAN_VALID_FREQ = 1000" >> $@; \
|
|
echo "MARIAN_WORKSPACE = 5000" >> $@; \
|
|
echo "MARIAN_VALID_MINI_BATCH = 8" >> $@; \
|
|
echo "BPESIZE = 4000" >> $@; \
|
|
echo "DEVSIZE = 1000" >> $@; \
|
|
echo "TESTSIZE = 1000" >> $@; \
|
|
echo "DEVMINSIZE = 250" >> $@; \
|
|
elif [ $$s -gt ${SMALLEST_TRAINSIZE} ]; then \
|
|
echo "# ${LANGPAIRSTR} training data less than ${SMALLEST_TRAINSIZE}" > $@; \
|
|
echo "GPUJOB_HPC_MEM = 4g" >> $@; \
|
|
echo "GPUJOB_SUBMIT = " >> $@; \
|
|
echo "MARIAN_VALID_FREQ = 1000" >> $@; \
|
|
echo "MARIAN_WORKSPACE = 3500" >> $@; \
|
|
echo "MARIAN_DROPOUT = 0.5" >> $@; \
|
|
echo "MARIAN_VALID_MINI_BATCH = 4" >> $@; \
|
|
echo "BPESIZE = 1000" >> $@; \
|
|
echo "DEVSIZE = 500" >> $@; \
|
|
echo "TESTSIZE = 1000" >> $@; \
|
|
echo "DEVMINSIZE = 100" >> $@; \
|
|
else \
|
|
echo "${LANGPAIRSTR} too small"; \
|
|
fi; \
|
|
if [ -e $@ ]; then \
|
|
if [ $$S -gt 1000 ]; then \
|
|
echo "SRCBPESIZE = 32000" >> $@; \
|
|
fi; \
|
|
if [ $$T -gt 1000 ]; then \
|
|
echo "TRGBPESIZE = 32000" >> $@; \
|
|
fi; \
|
|
fi
|
|
echo "SRCLANGS = ${SRCLANGS}" >> $@
|
|
echo "TRGLANGS = ${TRGLANGS}" >> $@
|
|
echo "SKIPLANGS = ${SKIPLANGS}" >> $@
|
|
echo "LANGPAIRSTR = ${LANGPAIRSTR}" >> $@
|
|
echo "DATASET = ${DATASET}" >> $@
|
|
echo "TRAINSET = ${TRAINSET}" >> $@
|
|
echo "DEVSET = ${DEVSET}" >> $@
|
|
echo "TESTSET = ${TESTSET}" >> $@
|
|
echo "PRE = ${PRE}" >> $@
|
|
echo "SUBWORDS = ${SUBWORDS}" >> $@
|
|
ifdef SHUFFLE_DATA
|
|
echo "SHUFFLE_DATA = ${SHUFFLE_DATA}" >> $@
|
|
endif
|
|
ifdef FIT_DATA_SIZE
|
|
echo "FIT_DATA_SIZE = ${FIT_DATA_SIZE}" >> $@
|
|
endif
|
|
ifdef FIT_DEVDATA_SIZE
|
|
echo "FIT_DEVDATA_SIZE = ${FIT_DEVDATA_SIZE}" >> $@
|
|
endif
|
|
echo "MAX_OVER_SAMPLING = ${MAX_OVER_SAMPLING}" >> $@
|
|
echo "USE_REST_DEVDATA = ${USE_REST_DEVDATA}" >> $@
|
|
ifdef USE_TARGET_LABELS
|
|
echo "USE_TARGET_LABELS = ${USE_TARGET_LABELS}" >> $@
|
|
endif
|
|
ifdef USE_SPM_VOCAB
|
|
echo "USE_SPM_VOCAB = ${USE_SPM_VOCAB}" >> $@
|
|
endif
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
### DEPRECATED? ################################################
|
|
################################################################
|
|
|
|
## list of all languages in OPUS
|
|
## TODO: do we still need this?
|
|
## --> see OPUSLANGS which is directly taken from the API
|
|
opus-langs.txt:
|
|
wget -O $@.tmp ${OPUSAPI}?languages=true
|
|
grep '",' $@.tmp | tr '",' ' ' | sort | tr "\n" ' ' | sed 's/ */ /g' > $@
|
|
rm -f $@.tmp
|
|
|
|
## all language pairs in opus in one file
|
|
## TODO: do we need this file?
|
|
opus-langpairs.txt:
|
|
for l in ${OPUS_LANGS}; do \
|
|
wget -O $@.tmp ${OPUSAPI}?source=$$l\&languages=true; \
|
|
grep '",' $@.tmp | tr '",' ' ' | sort | tr "\n" ' ' | sed 's/ */ /g' > $@.tmp2; \
|
|
for t in `cat $@.tmp2`; do \
|
|
if [ $$t \< $$l ]; then \
|
|
echo "$$t-$$l" >> $@.all; \
|
|
else \
|
|
echo "$$l-$$t" >> $@.all; \
|
|
fi \
|
|
done; \
|
|
rm -f $@.tmp $@.tmp2; \
|
|
done
|
|
tr ' ' "\n" < $@.all |\
|
|
sed 's/ //g' | sort -u | tr "\n" ' ' > $@
|
|
rm -f $@.all
|
|
|
|
|
|
|