mirror of
https://github.com/Helsinki-NLP/OPUS-MT-train.git
synced 2024-09-11 20:27:19 +03:00
116 lines
3.2 KiB
Makefile
116 lines
3.2 KiB
Makefile
# -*-makefile-*-
|
|
|
|
# enable e-mail notification by setting EMAIL
|
|
|
|
WHOAMI = $(shell whoami)
|
|
ifeq ("$(WHOAMI)","tiedeman")
|
|
EMAIL = jorg.tiedemann@helsinki.fi
|
|
endif
|
|
|
|
# job-specific settings (overwrite if necessary)
|
|
# HPC_EXTRA: additional SBATCH commands
|
|
|
|
CPU_MODULES = gcc/6.2.0 mkl
|
|
GPU_MODULES = cuda-env/8 mkl
|
|
# GPU_MODULES = python-env/3.5.3-ml cuda-env/8 mkl
|
|
|
|
|
|
|
|
# GPU = k80
|
|
GPU = p100
|
|
NR_GPUS = 1
|
|
HPC_MEM = 8g
|
|
HPC_NODES = 1
|
|
HPC_CORES = 1
|
|
HPC_DISK = 500
|
|
HPC_QUEUE = serial
|
|
# HPC_MODULES = nlpl-opus python-env/3.4.1 efmaral moses
|
|
# HPC_MODULES = nlpl-opus moses cuda-env marian python-3.5.3-ml
|
|
HPC_MODULES = ${GPU_MODULES}
|
|
HPC_EXTRA =
|
|
WALLTIME = 72
|
|
|
|
|
|
DEVICE = cuda
|
|
LOADCPU = module load ${CPU_MODULES}
|
|
LOADGPU = module load ${GPU_MODULES}
|
|
|
|
MARIAN_WORKSPACE = 13000
|
|
|
|
ifeq (${shell hostname},dx6-ibs-p2)
|
|
APPLHOME = /opt/tools
|
|
WORKHOME = ${shell realpath ${PWD}/work}
|
|
OPUSHOME = tiedeman@taito.csc.fi:/proj/nlpl/data/OPUS/
|
|
MOSESHOME = ${APPLHOME}/mosesdecoder
|
|
MARIAN = ${APPLHOME}/marian/build
|
|
LOADMODS = echo "nothing to load"
|
|
MARIAN_WORKSPACE = 10000
|
|
else ifeq (${shell hostname},dx7-nkiel-4gpu)
|
|
APPLHOME = /opt/tools
|
|
WORKHOME = ${shell realpath ${PWD}/work}
|
|
OPUSHOME = tiedeman@taito.csc.fi:/proj/nlpl/data/OPUS/
|
|
MOSESHOME = ${APPLHOME}/mosesdecoder
|
|
MARIAN = ${APPLHOME}/marian/build
|
|
LOADMODS = echo "nothing to load"
|
|
MARIAN_WORKSPACE = 10000
|
|
else ifneq ($(wildcard /wrk/tiedeman/research/),)
|
|
DATAHOME = /proj/OPUS/WMT19/data/${LANGPAIR}
|
|
# APPLHOME = ${USERAPPL}/tools
|
|
APPLHOME = /proj/memad/tools
|
|
WORKHOME = /wrk/tiedeman/research/marian/${SRC}-${TRG}
|
|
OPUSHOME = /proj/nlpl/data/OPUS
|
|
MOSESHOME = /proj/nlpl/software/moses/4.0-65c75ff/moses
|
|
# MARIAN = /proj/nlpl/software/marian/1.2.0
|
|
# MARIAN = /appl/ling/marian
|
|
MARIAN = ${HOME}/appl_taito/tools/marian/build-gpu
|
|
MARIANCPU = ${HOME}/appl_taito/tools/marian/build-cpu
|
|
LOADMODS = ${LOADGPU}
|
|
else
|
|
# CSCPROJECT = project_2001194
|
|
CSCPROJECT = project_2000309
|
|
DATAHOME = ${HOME}/work/opentrans/data/${LANGPAIR}
|
|
WORKHOME = ${shell realpath ${PWD}/work}
|
|
APPLHOME = ${HOME}/projappl
|
|
OPUSHOME = /scratch/project_2000661/nlpl/data/OPUS
|
|
MOSESHOME = ${APPLHOME}/mosesdecoder
|
|
EFLOMAL_HOME = ${APPLHOME}/eflomal/
|
|
MARIAN = ${APPLHOME}/marian/build
|
|
MARIANCPU = ${APPLHOME}/marian/build
|
|
# GPU_MODULES = cuda intel-mkl
|
|
GPU = v100
|
|
GPU_MODULES = python-env
|
|
CPU_MODULES = python-env
|
|
LOADMODS = echo "nothing to load"
|
|
HPC_QUEUE = small
|
|
MARIAN_WORKSPACE = 30000
|
|
endif
|
|
|
|
|
|
ifdef LOCAL_SCRATCH
|
|
TMPDIR = ${LOCAL_SCRATCH}
|
|
endif
|
|
|
|
|
|
WORDALIGN = ${EFLOMAL_HOME}align.py
|
|
ATOOLS = ${FASTALIGN_HOME}atools
|
|
|
|
MULTEVALHOME = ${APPLHOME}/multeval
|
|
MOSESSCRIPTS = ${MOSESHOME}/scripts
|
|
TOKENIZER = ${MOSESSCRIPTS}/tokenizer
|
|
SNMTPATH = ${APPLHOME}/subword-nmt/subword_nmt
|
|
|
|
|
|
|
|
# sorted languages and langpair used to match resources in OPUS
|
|
SORTLANGS = $(sort ${SRC} ${TRG})
|
|
LANGPAIR = ${firstword ${SORTLANGS}}-${lastword ${SORTLANGS}}
|
|
|
|
## for same language pairs: add numeric extension
|
|
ifeq (${SRC},$(TRG))
|
|
SRCEXT = ${SRC}1
|
|
TRGEXT = ${SRC}2
|
|
else
|
|
SRCEXT = ${SRC}
|
|
TRGEXT = ${TRG}
|
|
endif
|