mirror of
https://github.com/Helsinki-NLP/OPUS-MT-train.git
synced 2024-08-17 16:20:50 +03:00
some fixes to the installation process
This commit is contained in:
Joerg Tiedemann
parent
0e51cffbc9
commit
e68e5cd81d
59
lib/env.mk
59
lib/env.mk
@ -5,8 +5,9 @@
|
||||
# - system-specific settings
|
||||
#
|
||||
|
||||
SHELL := /bin/bash
|
||||
PWD ?= ${shell pwd}
|
||||
SHELL := /bin/bash
|
||||
PWD ?= ${shell pwd}
|
||||
REPOHOME ?= ${PWD}/
|
||||
|
||||
# job-specific settings (overwrite if necessary)
|
||||
# HPC_EXTRA: additional SBATCH commands
|
||||
@ -39,8 +40,8 @@ WORKHOME ?= ${PWD}/work
|
||||
## the build environment for specific software
|
||||
|
||||
LOAD_BUILD_ENV = echo "nothing to load"
|
||||
LOAD_MARIAN_BUILD_ENV = echo "nothing to load"
|
||||
LOAD_EXTRACTLEX_BUILD_ENV = echo "nothing to load"
|
||||
LOAD_MARIAN_BUILD_ENV = ${LOAD_BUILD_ENV}
|
||||
LOAD_EXTRACTLEX_BUILD_ENV = ${LOAD_BUILD_ENV}
|
||||
|
||||
|
||||
## load system-specific environments
|
||||
@ -151,7 +152,7 @@ BROWSERMT_CONVERT = ${BROWSERMT_HOME}/marian-dev/build/marian-conv
|
||||
## BPE
|
||||
SUBWORD_BPE ?= ${shell which subword-nmt 2>/dev/null || echo ${TOOLSDIR}/subword-nmt/subword_nmt/subword_nmt.py}
|
||||
SUBWORD_HOME ?= ${dir ${SUBWORD_BPE}}
|
||||
ifeq (${shell which subword-nmt},)
|
||||
ifeq (${shell which subword-nmt 2>/dev/null},)
|
||||
BPE_LEARN ?= python3 ${SUBWORD_HOME}/learn_bpe.py
|
||||
BPE_APPLY ?= python3 ${SUBWORD_HOME}/apply_bpe.py
|
||||
else
|
||||
@ -203,12 +204,11 @@ endif
|
||||
## TODO: add EXTRACT_LEX, BROWSERMT_TRAIN, ..?
|
||||
## TODO: add OpusFilter?
|
||||
|
||||
PREREQ_TOOLS := $(lastword ${ISO639}) ${ATOOLS} ${PIGZ} ${TERASHUF} ${JQ} ${MARIAN} ${EFLOMAL} ${TMX2MOSES}
|
||||
PREREQ_TOOLS := $(lastword ${ISO639}) ${ATOOLS} ${PIGZ} ${TERASHUF} ${MARIAN} ${EFLOMAL} ${TMX2MOSES} ${JQ}
|
||||
PREREQ_PERL := ISO::639::3 ISO::639::5 OPUS::Tools XML::Parser
|
||||
|
||||
PIP := ${shell which pip3 2>/dev/null || echo pip}
|
||||
CPAN := ${shell which cpanm 2>/dev/null || echo cpan}
|
||||
|
||||
PIP := ${shell ${LOAD_BUILD_ENV} >/dev/null 2>/dev/null && which pip3 2>/dev/null || echo pip}
|
||||
CPAN := ${shell ${LOAD_BUILD_ENV} >/dev/null 2>/dev/null && which cpanm 2>/dev/null || echo cpan}
|
||||
|
||||
## setup local Perl environment
|
||||
## better install local::lib and put this into your .bashrc:
|
||||
@ -221,12 +221,32 @@ export PERL_LOCAL_LIB_ROOT := ${HOME}/perl5:${PERL_LOCAL_LIB_ROOT}}
|
||||
export PERL_MB_OPT := --install_base "${HOME}/perl5"
|
||||
export PERL_MM_OPT := INSTALL_BASE=${HOME}/perl5
|
||||
|
||||
## quick hack to fix a problem in marian-dev submodule fbgemm
|
||||
## --> googletest changed to 'main' from 'master' (stupid)
|
||||
## TODO: remove this again once it is not needed anymore!
|
||||
|
||||
PHONY: install install-prerequisites install-prereq install-requirements
|
||||
install install-prerequisites install-prereq install-requirements:
|
||||
${PIP} install --user -r requirements.txt
|
||||
${MAKE} install-perl-modules
|
||||
${MAKE} ${PREREQ_TOOLS}
|
||||
-git submodule update --init --recursive --remote
|
||||
cp tools/marian-dev/src/3rd_party/fbgemm/.gitmodules \
|
||||
tools/marian-dev/src/3rd_party/fbgemm/.gitmodules.backup
|
||||
cat tools/marian-dev/src/3rd_party/fbgemm/.gitmodules.backup |\
|
||||
sed 's#google/googletest#google/googletest| branch = main#' | tr '|' "\n" | uniq \
|
||||
> tools/marian-dev/src/3rd_party/fbgemm/.gitmodules
|
||||
cp tools/browsermt/marian-dev/src/3rd_party/fbgemm/.gitmodules \
|
||||
tools/browsermt/marian-dev/src/3rd_party/fbgemm/.gitmodules.backup
|
||||
cat tools/browsermt/marian-dev/src/3rd_party/fbgemm/.gitmodules.backup |\
|
||||
sed 's#google/googletest#google/googletest| branch = main#' | tr '|' "\n" | uniq \
|
||||
> tools/browsermt/marian-dev/src/3rd_party/fbgemm/.gitmodules
|
||||
git submodule update --init --recursive --remote
|
||||
${LOAD_BUILD_ENV} && ${PIP} install --user -r requirements.txt
|
||||
${LOAD_BUILD_ENV} && ${MAKE} install-perl-modules
|
||||
${LOAD_BUILD_ENV} && ${MAKE} ${PREREQ_TOOLS}
|
||||
|
||||
.PHONY: install-prereq-tools
|
||||
install-prereq-tools:
|
||||
${LOAD_BUILD_ENV} && ${MAKE} ${PREREQ_TOOLS}
|
||||
|
||||
|
||||
.PHONY: install-perl-modules
|
||||
install-perl-modules:
|
||||
@ -248,9 +268,12 @@ ${TOOLSDIR}/fast_align/build/atools:
|
||||
${TOOLSDIR}/pigz/pigz:
|
||||
${MAKE} -C ${dir $@}
|
||||
|
||||
|
||||
## Don't need this anymore - it's a submodule
|
||||
# mkdir -p ${TOOLSDIR}
|
||||
# cd ${TOOLSDIR} && git clone https://github.com/alexandres/terashuf.git
|
||||
|
||||
${TOOLSDIR}/terashuf/terashuf:
|
||||
mkdir -p ${TOOLSDIR}
|
||||
cd ${TOOLSDIR} && git clone https://github.com/alexandres/terashuf.git
|
||||
${MAKE} -C ${dir $@}
|
||||
|
||||
${TOOLSDIR}/jq/jq:
|
||||
@ -263,17 +286,19 @@ ${TOOLSDIR}/jq/jq:
|
||||
## - install protobuf: sudo port install protobuf3-cpp
|
||||
## - install MKL (especially for cpu use):
|
||||
## file:///opt/intel/documentation_2020/en/mkl/ps2020/get_started.htm
|
||||
## mkdir -p ${TOOLSDIR}
|
||||
## cd ${TOOLSDIR} && git clone https://github.com/marian-nmt/marian-dev.git
|
||||
|
||||
${TOOLSDIR}/marian-dev/build/marian: ${PROTOC}
|
||||
mkdir -p ${TOOLSDIR}
|
||||
cd ${TOOLSDIR} && git clone https://github.com/marian-nmt/marian-dev.git
|
||||
mkdir -p ${dir $@}
|
||||
cd ${dir $@} && ${LOAD_MARIAN_BUILD_ENV} && cmake -DUSE_SENTENCEPIECE=on ${MARIAN_BUILD_OPTIONS} ..
|
||||
${LOAD_MARIAN_BUILD_ENV} && ${MAKE} -C ${dir $@} -j8
|
||||
|
||||
${TOOLSDIR}/protobuf/bin/protoc:
|
||||
mkdir -p ${TOOLSDIR}
|
||||
cd ${TOOLSDIR} && git clone https://github.com/protocolbuffers/protobuf.git
|
||||
if [ -e ${dir $@} ]; then \
|
||||
cd ${TOOLSDIR} && git clone https://github.com/protocolbuffers/protobuf.git; \
|
||||
fi
|
||||
cd ${TOOLSDIR}/protobuf && git submodule update --init --recursive
|
||||
cd ${TOOLSDIR}/protobuf && ./autogen.sh
|
||||
cd ${TOOLSDIR}/protobuf && ./configure --prefix=${TOOLSDIR}/protobuf
|
||||
|
||||
6
lib/env/puhti.mk
vendored
6
lib/env/puhti.mk
vendored
@ -69,11 +69,11 @@ endif
|
||||
HPC_EXTRA1 = \#SBATCH --account=${CSCPROJECT}
|
||||
|
||||
|
||||
BUILD_MODULES = cmake perl/5.30.0
|
||||
LOAD_BUILD_ENV = module load ${BUILD_MODULES} && module list
|
||||
BUILD_MODULES = StdEnv python-env cmake perl/5.30.0
|
||||
LOAD_BUILD_ENV = module purge && module load ${BUILD_MODULES} && module list
|
||||
|
||||
MARIAN_BUILD_MODULES = gcc/8.3.0 cuda/10.1.168 cudnn/7.6.1.34-10.1 intel-mkl/2019.0.4 cmake/3.18.2
|
||||
LOAD_MARIAN_BUILD_ENV = module purge && module load ${MARIAN_BUILD_MODULES}
|
||||
LOAD_MARIAN_BUILD_ENV = module purge && module load ${MARIAN_BUILD_MODULES} && module list
|
||||
MARIAN_BUILD_OPTIONS = -DTcmalloc_INCLUDE_DIR=/appl/spack/install-tree/gcc-8.3.0/gperftools-2.7-5w7w2c/include \
|
||||
-DTcmalloc_LIBRARY=/appl/spack/install-tree/gcc-8.3.0/gperftools-2.7-5w7w2c/lib/libtcmalloc.so \
|
||||
-DTCMALLOC_LIB=/appl/spack/install-tree/gcc-8.3.0/gperftools-2.7-5w7w2c/lib/libtcmalloc.so \
|
||||
|
||||
@ -1 +1 @@
|
||||
Subproject commit 6575f72aac445e42ff490852161e066588208bc3
|
||||
Subproject commit 844800efccba6e670250caac1735ca2c8c8e508e
|
||||
Loading…
Reference in New Issue
Block a user