OPUS-MT-train/lib/data.mk
2022-07-12 23:08:00 +03:00

989 lines
39 KiB
Makefile

# -*-makefile-*-
#
# create data files for taining, validation and testing
#
# - combine all bitexts in TRAINSET
# - add backtranslation, pivoted data if necessary
# - add language labels if necessary (multi-target models)
# - over/under-sampling of training data if necessary (multilingual models)
# - shuffle dev/test data and divide into to disjoint sets
# - reverse data sets for the other translation direction (bilingual models only)
# - run word alignment if necessary (models with guided alignment = transformer-align)
#
#
# TODO: write data info to some model-specific file insetad of README.md
# (applies for train/val/test!)
## training data size (generates count if not in README.md)
## TODO: do we still need this?
TRAINDATA_SIZE = ${shell \
if [ -e ${WORKDIR}/train/README.md ]; then \
if [ `grep 'total size (${DATASET}):' ${WORKDIR}/train/README.md | wc -l` -gt 0 ]; then \
grep 'total size (${DATASET}):' ${WORKDIR}/train/README.md | cut -f2 -d':' ; \
elif [ -e ${TRAIN_SRC}.clean.${PRE_SRC}.gz ]; then \
echo -n '* total size (${DATASET}): ' >> ${WORKDIR}/train/README.md; \
${GZIP} -cd < ${TRAIN_SRC}.clean.${PRE_SRC}.gz | wc -l >> ${WORKDIR}/train/README.md; \
grep 'total size (${DATASET}):' ${WORKDIR}/train/README.md | cut -f2 -d':' ; \
fi \
elif [ -e ${TRAIN_SRC}.clean.${PRE_SRC}.gz ]; then \
echo '\# ${DATASET}' >> ${WORKDIR}/train/README.md; \
echo '' >> ${WORKDIR}/train/README.md; \
echo -n '* total size (${DATASET}): ' >> ${WORKDIR}/train/README.md; \
${GZIP} -cd < ${TRAIN_SRC}.clean.${PRE_SRC}.gz | wc -l >> ${WORKDIR}/train/README.md; \
grep 'total size (${DATASET}):' ${WORKDIR}/train/README.md | cut -f2 -d':' ; \
fi }
## set sample data size for mulitlingual models
## (only if DATA_SAMPLING_WEIGHT is set)
## MAX_DATA_SIZE controls the maximum data size for a language pair
## default: keep all data for the largest language pair (no down-sampling)
## DATA_SAMPLING_WEIGHT balances between uniform and distributional sampling
## = 1: sample proportional to the size of bitext per language pair
## = 0.2: corresponds to temperature 5 to up-sample low resource languages
ifneq (${words ${SRCLANGS} ${TRGLANGS}},2)
ifdef DATA_SAMPLING_WEIGHT
ifneq (${wildcard ${WORKDIR}/train/size_per_language_pair.txt},)
ifdef MAX_DATA_SIZE
FIT_DATA_SIZE = ${shell ${REPOHOME}scripts/data-sample-sizes.pl -w ${DATA_SAMPLING_WEIGHT} -m ${MAX_DATA_SIZE} \
${WORKDIR}/train/size_per_language_pair.txt | grep '^${LANGPAIR} ' | cut -f2}
else
FIT_DATA_SIZE = ${shell ${REPOHOME}scripts/data-sample-sizes.pl -w ${DATA_SAMPLING_WEIGHT} \
${WORKDIR}/train/size_per_language_pair.txt | grep '^${LANGPAIR} ' | cut -f2}
endif
endif
endif
endif
print-data-sampling-size:
${REPOHOME}scripts/data-sample-sizes.pl -w 0.3 \
${WORKDIR}/train/size_per_language_pair.txt | grep '^${LANGPAIR} ' | cut -f2
@echo "sample size for ${LANGPAIR}: ${FIT_DATA_SIZE}"
print-data-sampling-sizes:
for s in ${SRCLANGS}; do \
for t in ${TRGLANGS}; do \
if [ ! `echo "$$s-$$t $$t-$$s" | egrep '${SKIP_LANGPAIRS}' | wc -l` -gt 0 ]; then \
if [ "${SKIP_SAME_LANG}" != "1" ] || [ "$$s" != "$$t" ]; then \
${MAKE} SRC:=$$s TRG:=$$t print-data-sampling-size; \
fi \
fi \
done \
done
## look for cleanup scripts and put them into a pipe
## they should be executable and should basically read STDIN and print to STDOUT
## no further arguments are supported
ifneq (${wildcard ${REPOHOME}scripts/cleanup/${SRC}},)
SRC_CLEANUP_SCRIPTS = | ${subst ${SPACE}, | ,${shell find ${REPOHOME}scripts/cleanup/${SRC} -executable -type f}}
endif
ifneq (${wildcard ${REPOHOME}scripts/cleanup/${TRG}},)
TRG_CLEANUP_SCRIPTS = | ${subst ${SPACE}, | ,${shell find ${REPOHOME}scripts/cleanup/${TRG} -executable -type f}}
endif
##-------------------------------------------------------------
## backtranslated data and pivot-based synthetic training data
##-------------------------------------------------------------
## back translation data
## - use only the latest backtranslations
## if such a subdir exists
ifneq (${wildcard ${BACKTRANS_HOME}/${TRG}-${SRC}/latest},)
BACKTRANS_DIR = ${BACKTRANS_HOME}/${TRG}-${SRC}/latest
else
BACKTRANS_DIR = ${BACKTRANS_HOME}/${TRG}-${SRC}
endif
## TODO: make it possible to select only parts of the BT data
## ---> use TRAINDATA_SIZE to take max the same amount of all shuffled BT data
# back-translation data (target-to-source)
ifeq (${USE_BACKTRANS},1)
BACKTRANS_SRC = ${sort ${wildcard ${BACKTRANS_DIR}/*.${SRCEXT}.gz}}
BACKTRANS_TRG = ${patsubst %.${SRCEXT}.gz,%.${TRGEXT}.gz,${BACKTRANS_SRC}}
endif
# forward-translation data (source-to-target)
ifeq (${USE_FORWARDTRANS},1)
FORWARDTRANS_SRC = ${sort ${wildcard ${FORWARDTRANS_HOME}/${SRC}-${TRG}/latest/*.${SRCEXT}.gz}}
FORWARDTRANS_TRG = ${patsubst %.${SRCEXT}.gz,%.${TRGEXT}.gz,${FORWARDTRANS_SRC}}
endif
# forward-translation data (source-to-target)
# filtered by reconstruction scores (ce filter)
ifneq (${USE_FORWARDTRANS_SELECTED},)
FORWARDTRANS_SRC += ${sort ${wildcard ${FORWARDTRANS_HOME}/${SRC}-${TRG}/latest/*.${SRCEXT}.best${USE_FORWARDTRANS_SELECTED}.gz}}
FORWARDTRANS_TRG += ${sort ${wildcard ${FORWARDTRANS_HOME}/${SRC}-${TRG}/latest/*.${TRGEXT}.best${USE_FORWARDTRANS_SELECTED}.gz}}
endif
## selected by "raw" (unnormalised) scores
ifneq (${USE_FORWARDTRANS_SELECTED_RAW},)
FORWARDTRANS_SRC += ${sort ${wildcard ${FORWARDTRANS_HOME}/${SRC}-${TRG}/latest/*.${SRCEXT}.rawbest${USE_FORWARDTRANS_SELECTED_RAW}.gz}}
FORWARDTRANS_TRG += ${sort ${wildcard ${FORWARDTRANS_HOME}/${SRC}-${TRG}/latest/*.${TRGEXT}.rawbest${USE_FORWARDTRANS_SELECTED_RAW}.gz}}
endif
# forward-translation data of monolingual data (source-to-target)
ifeq (${USE_FORWARDTRANSMONO},1)
FORWARDTRANSMONO_SRC = ${sort ${wildcard ${BACKTRANS_HOME}/${SRC}-${TRG}/latest/*.${SRCEXT}.gz}}
FORWARDTRANSMONO_TRG = ${patsubst %.${SRCEXT}.gz,%.${TRGEXT}.gz,${FORWARDTRANSMONO_SRC}}
endif
# forward translation using pivoting (target language is automatically created)
ifeq (${USE_FORWARD_PIVOTING},1)
PIVOTING_SRC = ${sort ${wildcard ${PIVOTTRANS_HOME}/${TRG}-${SRC}/latest/*.${SRCEXT}.gz}}
PIVOTING_TRG = ${patsubst %.${SRCEXT}.gz,%.${TRGEXT}.gz,${PIVOTING_SRC}}
endif
# backward translation using pivoting (source language is automatically created)
ifeq (${USE_BACKWARD_PIVOTING},1)
PIVOTING_SRC += ${sort ${wildcard ${PIVOTTRANS_HOME}/${SRC}-${TRG}/latest/*.${SRCEXT}.gz}}
PIVOTING_TRG = ${patsubst %.${SRCEXT}.gz,%.${TRGEXT}.gz,${PIVOTING_SRC}}
endif
# pivot-based data augmentation data (in both directions)
ifeq (${USE_PIVOTING},1)
PIVOTING_SRC = ${sort ${wildcard ${PIVOTTRANS_HOME}/${SRC}-${TRG}/latest/*.${SRCEXT}.gz} \
${wildcard ${PIVOTTRANS_HOME}/${TRG}-${SRC}/latest/*.${SRCEXT}.gz}}
PIVOTING_TRG = ${patsubst %.${SRCEXT}.gz,%.${TRGEXT}.gz,${PIVOTING_SRC}}
endif
# additional data sets that might be available ...
ifeq (${USE_EXTRA_BITEXTS},1)
EXTRA_BITEXTS_SRC = ${sort ${wildcard ${DATADIR}/extra/${SRC}-${TRG}/*.${SRCEXT}.gz}}
EXTRA_BITEXTS_TRG = ${patsubst %.${SRCEXT}.gz,%.${TRGEXT}.gz,${EXTRA_BITEXTS_SRC}}
endif
print-datasets:
-@for s in ${SRCLANGS}; do \
for t in ${TRGLANGS}; do \
${MAKE} SRC=$$s TRG=$$t print-datasets-current-langpair; \
done \
done
print-datasets-current-langpair:
@echo ${TATOEBA_TRAINSET}
@echo ${TRAINSET}
@echo "all data:"
@echo ${CLEAN_TRAIN_SRC}
@echo ${CLEAN_TRAIN_TRG}
@echo "back-translation data:"
@echo ${BACKTRANS_SRC}
@echo ${BACKTRANS_TRG}
@echo "forward translation data:"
@echo ${FORWARDTRANS_SRC}
@echo ${FORWARDTRANS_TRG}
@echo "monolingual forward translation data:"
@echo ${FORWARDTRANSMONO_SRC}
@echo ${FORWARDTRANSMONO_TRG}
@echo "pivot-based translation data:"
@echo ${PIVOTING_SRC}
@echo ${PIVOTING_TRG}
@echo "extra bitexts:"
@echo ${EXTRA_BITEXTS_SRC}
@echo ${EXTRA_BITEXTS_TRG}
##-------------------------------------------------------------
## data sets (train/dev/test)
##-------------------------------------------------------------
## data sets to be included in the train/dev/test sets
## with some basic pre-processing (see lib/preprocess.mk)
CLEAN_TRAIN_SRC = ${patsubst %,${DATADIR}/${PRE}/%.${LANGPAIR}.${CLEAN_TRAINDATA_TYPE}.${SRCEXT}.gz,${TRAINSET}} \
${BACKTRANS_SRC} ${FORWARDTRANS_SRC} ${FORWARDTRANSMONO_SRC} ${PIVOTING_SRC} ${EXTRA_BITEXTS_SRC}
CLEAN_TRAIN_TRG = ${patsubst %,${DATADIR}/${PRE}/%.${LANGPAIR}.${CLEAN_TRAINDATA_TYPE}.${TRGEXT}.gz,${TRAINSET}} \
${BACKTRANS_TRG} ${FORWARDTRANS_TRG} ${FORWARDTRANSMONO_TRG} ${PIVOTING_TRG} ${EXTRA_BITEXTS_TRG}
CLEAN_DEV_SRC = ${patsubst %,${DATADIR}/${PRE}/%.${LANGPAIR}.${CLEAN_DEVDATA_TYPE}.${SRCEXT}.gz,${DEVSET}}
CLEAN_DEV_TRG = ${patsubst %.${SRCEXT}.gz,%.${TRGEXT}.gz,${CLEAN_DEV_SRC}}
CLEAN_TEST_SRC = ${patsubst %,${DATADIR}/${PRE}/%.${LANGPAIR}.${CLEAN_TESTDATA_TYPE}.${SRCEXT}.gz,${TESTSET}}
CLEAN_TEST_TRG = ${patsubst %.${SRCEXT}.gz,%.${TRGEXT}.gz,${CLEAN_TEST_SRC}}
CLEAN_TEST_SRC_STATS = ${CLEAN_TEST_SRC:.gz=.stats}
CLEAN_TEST_TRG_STATS = ${CLEAN_TEST_TRG:.gz=.stats}
DATA_SRC := ${sort ${CLEAN_TRAIN_SRC} ${CLEAN_DEV_SRC} ${CLEAN_TEST_SRC}}
DATA_TRG := ${sort ${CLEAN_TRAIN_TRG} ${CLEAN_DEV_TRG} ${CLEAN_TEST_TRG}}
##-------------------------------------------------------------
## make data in reverse direction without re-doing word alignment etc ...
## ---> this is dangerous when things run in parallel
## ---> only works for bilingual models
##-------------------------------------------------------------
REV_LANGSTR = ${subst ${SPACE},+,$(TRGLANGS)}-${subst ${SPACE},+,$(SRCLANGS)}
REV_WORKDIR = ${WORKHOME}/${REV_LANGSTR}
.PHONY: reverse-data
reverse-data:
ifeq (${PRE_SRC},${PRE_TRG})
ifeq (${words ${SRCLANGS}},1)
ifeq (${words ${TRGLANGS}},1)
mkdir -p ${REV_WORKDIR}/train
-if [ -e ${TRAIN_SRC}.clean.${PRE_SRC}.gz ]; then \
ln -s ${TRAIN_SRC}.clean.${PRE_SRC}.gz ${REV_WORKDIR}/train/${notdir ${TRAIN_TRG}.clean.${PRE_TRG}.gz}; \
ln -s ${TRAIN_TRG}.clean.${PRE_TRG}.gz ${REV_WORKDIR}/train/${notdir ${TRAIN_SRC}.clean.${PRE_SRC}.gz}; \
cp ${WORKDIR}/train/README.md ${REV_WORKDIR}/train/README.md; \
fi
-if [ -e ${SUBWORD_SRC_MODEL} ]; then \
ln -s ${SUBWORD_SRC_MODEL} ${REV_WORKDIR}/train/${notdir ${SUBWORD_TRG_MODEL}}; \
fi
-if [ -e ${SUBWORD_TRG_MODEL} ]; then \
ln -s ${SUBWORD_TRG_MODEL} ${REV_WORKDIR}/train/${notdir ${SUBWORD_SRC_MODEL}}; \
fi
-if [ -e ${SUBWORD_SRC_MODEL}.vocab ]; then \
ln -s ${SUBWORD_SRC_MODEL}.vocab ${REV_WORKDIR}/train/${notdir ${SUBWORD_TRG_MODEL}}.vocab; \
fi
-if [ -e ${SUBWORD_TRG_MODEL}.vocab ]; then \
ln -s ${SUBWORD_TRG_MODEL}.vocab ${REV_WORKDIR}/train/${notdir ${SUBWORD_SRC_MODEL}}.vocab; \
fi
-if [ -e ${TRAIN_ALG} ]; then \
if [ ! -e ${REV_WORKDIR}/train/${notdir ${TRAIN_ALG}} ]; then \
${GZIP} -cd < ${TRAIN_ALG} | ${MOSESSCRIPTS}/generic/reverse-alignment.perl |\
${GZIP} -c > ${REV_WORKDIR}/train/${notdir ${TRAIN_ALG}}; \
fi \
fi
-if [ -e ${DEV_SRC}.${PRE_SRC} ]; then \
mkdir -p ${REV_WORKDIR}/val; \
ln -s ${DEV_SRC}.${PRE_SRC} ${REV_WORKDIR}/val/${notdir ${DEV_TRG}.${PRE_TRG}}; \
ln -s ${DEV_TRG}.${PRE_TRG} ${REV_WORKDIR}/val/${notdir ${DEV_SRC}.${PRE_SRC}}; \
ln -s ${DEV_SRC} ${REV_WORKDIR}/val/${notdir ${DEV_TRG}}; \
ln -s ${DEV_TRG} ${REV_WORKDIR}/val/${notdir ${DEV_SRC}}; \
ln -s ${DEV_SRC}.shuffled.gz ${REV_WORKDIR}/val/${notdir ${DEV_SRC}.shuffled.gz}; \
ln -s ${DEV_SRC}.notused.gz ${REV_WORKDIR}/val/${notdir ${DEV_TRG}.notused.gz}; \
ln -s ${DEV_TRG}.notused.gz ${REV_WORKDIR}/val/${notdir ${DEV_SRC}.notused.gz}; \
cp ${WORKDIR}/val/README.md ${REV_WORKDIR}/val/README.md; \
fi
-if [ -e ${TEST_SRC} ]; then \
mkdir -p ${REV_WORKDIR}/test; \
ln -s ${TEST_SRC} ${REV_WORKDIR}/test/${notdir ${TEST_TRG}}; \
ln -s ${TEST_TRG} ${REV_WORKDIR}/test/${notdir ${TEST_SRC}}; \
cp ${WORKDIR}/test/README.md ${REV_WORKDIR}/test/README.md; \
fi
-if [ -e ${MODEL_SRCVOCAB} ]; then \
ln -s ${MODEL_SRCVOCAB} ${REV_WORKDIR}/${notdir ${MODEL_TRGVOCAB}}; \
fi
-if [ -e ${MODEL_TRGVOCAB} ]; then \
ln -s ${MODEL_TRGVOCAB} ${REV_WORKDIR}/${notdir ${MODEL_SRCVOCAB}}; \
fi
-if [ -e ${MODEL_VOCAB} ]; then \
ln -s ${MODEL_VOCAB} ${REV_WORKDIR}/${notdir ${MODEL_VOCAB}}; \
fi
##
## this is a bit dangerous with some trick to
## swap parameters between SRC and TRG
##
-if [ -e ${WORKDIR}/${MODELCONFIG} ]; then \
if [ ! -e ${REV_WORKDIR}/${MODELCONFIG} ]; then \
cat ${WORKDIR}/${MODELCONFIG} |\
sed -e 's/SRC/TTT/g;s/TRG/SRC/g;s/TTT/TRG/' |\
grep -v LANGPAIRSTR > ${REV_WORKDIR}/$(notdir ${MODELCONFIG}); \
fi \
fi
endif
endif
endif
## generate a file with the size per language pair
## NOTE: those are taken from CLEAN_TRAIN_SIZE and not the final
## training data after the additional call to the clean-corpus script!
## --> counts are used to estimate temperature-based sample sizes
## for multilingual models
train-size-per-language: ${WORKDIR}/train/size_per_language_pair.txt
add-size-per-language-pair-info:
ifneq (${wildcard ${CLEAN_TRAIN_SRC}},)
( s=`${GZCAT} ${wildcard ${CLEAN_TRAIN_SRC}} | wc -l`; \
echo "${LANGPAIR} $$s" >> ${WORKDIR}/train/size_per_language_pair.txt )
endif
${WORKDIR}/train/size_per_language_pair.txt: ${LOCAL_TRAINDATA_DEPENDENCIES}
mkdir -p $(dir $@)
rm -f $@
for s in ${SRCLANGS}; do \
for t in ${TRGLANGS}; do \
if [ ! `echo "$$s-$$t $$t-$$s" | egrep '${SKIP_LANGPAIRS}' | wc -l` -gt 0 ]; then \
if [ "${SKIP_SAME_LANG}" != "1" ] || [ "$$s" != "$$t" ]; then \
${MAKE} DATASET=${DATASET} SRC:=$$s TRG:=$$t add-size-per-language-pair-info; \
fi \
fi \
done \
done
cut -f2 $@ | paste -sd+ | bc | sed 's/^/TOTAL /' >> $@
.PHONY: clean-data rawdata
clean-data rawdata:
@for s in ${SRCLANGS}; do \
for t in ${TRGLANGS}; do \
echo "..... create raw data for $$s-$$t"; \
${MAKE} SRC=$$s TRG=$$t clean-data-source; \
done \
done
.PHONY: clean-data-source
clean-data-source:
@${MAKE} ${CLEAN_TEST_SRC} ${CLEAN_TEST_TRG}
@${MAKE} ${CLEAN_TEST_SRC_STATS} ${CLEAN_TEST_TRG_STATS}
@${MAKE} ${DATA_SRC} ${DATA_TRG}
## shuffle training data (if one wants to do that after they have been created already)
.PHONY: shuffle-training-data
shuffle-training-data:
ifneq (${wildcard ${TRAIN_ALG}},)
paste <(gzip -cd ${TRAIN_SRC}.clean.${PRE_SRC}.gz) \
<(gzip -cd ${TRAIN_TRG}.clean.${PRE_TRG}.gz) \
<(gzip -cd ${TRAIN_ALG}) |\
${SHUFFLE} |\
tee >(cut -f1 | gzip -c >${TRAIN_SRC}.clean.${PRE_SRC}.new.gz) \
>(cut -f2 | gzip -c >${TRAIN_TRG}.clean.${PRE_TRG}.new.gz) | \
cut -f3 | gzip -c > ${TRAIN_ALG}.new.gz
mv -f ${TRAIN_SRC}.clean.${PRE_SRC}.new.gz ${TRAIN_SRC}.clean.${PRE_SRC}.gz
mv -f ${TRAIN_TRG}.clean.${PRE_TRG}.new.gz ${TRAIN_TRG}.clean.${PRE_TRG}.gz
mv ${TRAIN_ALG}.new.gz ${TRAIN_ALG}
else
paste <(gzip -cd ${TRAIN_SRC}.clean.${PRE_SRC}.gz) \
<(gzip -cd ${TRAIN_TRG}.clean.${PRE_TRG}.gz) |\
${SHUFFLE} |\
tee >(cut -f1 | gzip -c >${TRAIN_SRC}.clean.${PRE_SRC}.new.gz) \
>(cut -f2 | gzip -c >${TRAIN_TRG}.clean.${PRE_TRG}.new.gz)
mv -f ${TRAIN_SRC}.clean.${PRE_SRC}.new.gz ${TRAIN_SRC}.clean.${PRE_SRC}.gz
mv -f ${TRAIN_TRG}.clean.${PRE_TRG}.new.gz ${TRAIN_TRG}.clean.${PRE_TRG}.gz
endif
## monolingual data sets (for sentence piece models)
.INTERMEDIATE: ${LOCAL_MONO_DATA}.${PRE} ${LOCAL_MONO_DATA}.raw
.PHONY: mono-data
mono-data: ${LOCAL_MONO_DATA}.${PRE}
## word alignment used for guided alignment
## (always remove intermediate files)
.INTERMEDIATE: ${LOCAL_TRAIN_SRC}.algtmp ${LOCAL_TRAIN_TRG}.algtmp
${LOCAL_TRAIN_SRC}.algtmp: ${TRAIN_SRC}.clean.${PRE_SRC}${TRAINSIZE}.gz
mkdir -p ${dir $@}
${GZIP} -cd < $< > $@
${LOCAL_TRAIN_TRG}.algtmp: ${TRAIN_TRG}.clean.${PRE_TRG}${TRAINSIZE}.gz
mkdir -p ${dir $@}
${GZIP} -cd < $< > $@
## max number of lines in a corpus for running word alignment
## (split into chunks of max that size before aligning)
MAX_WORDALIGN_SIZE = 5000000
# MAX_WORDALIGN_SIZE = 10000000
# MAX_WORDALIGN_SIZE = 25000000
## nr of simultaneous word alignment jobs
## (assuming that each of them occupies up to 6 cores
NR_ALIGN_JOBS ?= $$(( ${CPU_CORES} / 6 + 1 ))
## job forcing doesn't work within recipes
# ${MAKE} -j ${NR_ALIGN_JOBS} $$a
${TRAIN_ALG}: ${TRAIN_SRC}.clean.${PRE_SRC}${TRAINSIZE}.gz \
${TRAIN_TRG}.clean.${PRE_TRG}${TRAINSIZE}.gz
${MAKE} ${LOCAL_TRAIN_SRC}.algtmp ${LOCAL_TRAIN_TRG}.algtmp
if [ `head $(LOCAL_TRAIN_SRC).algtmp | wc -l` -gt 0 ]; then \
mkdir -p $(LOCAL_TRAIN_SRC).algtmp.d; \
mkdir -p $(LOCAL_TRAIN_TRG).algtmp.d; \
split -l ${MAX_WORDALIGN_SIZE} $(LOCAL_TRAIN_SRC).algtmp $(LOCAL_TRAIN_SRC).algtmp.d/; \
split -l ${MAX_WORDALIGN_SIZE} $(LOCAL_TRAIN_TRG).algtmp $(LOCAL_TRAIN_TRG).algtmp.d/; \
a=`ls $(LOCAL_TRAIN_SRC).algtmp.d/* | sed 's#$$#.alg#' | xargs`; \
if [ "$$a" != "" ]; then \
${MAKE} $$a; \
cat $(LOCAL_TRAIN_SRC).algtmp.d/*.alg | ${GZIP} -c > $@; \
rm -f ${LOCAL_TRAIN_SRC}.algtmp.d/*; \
rm -f ${LOCAL_TRAIN_TRG}.algtmp.d/*; \
fi; \
rmdir ${LOCAL_TRAIN_SRC}.algtmp.d; \
rmdir ${LOCAL_TRAIN_TRG}.algtmp.d; \
fi
rm -f ${LOCAL_TRAIN_SRC}.algtmp ${LOCAL_TRAIN_TRG}.algtmp
## old: do this sequenctially
## new: do this in parallel via make (see above)
## disadvantage: may require more memory!
# for s in `ls $(LOCAL_TRAIN_SRC).algtmp.d`; do \
# echo "align part $$s"; \
# ${WORDALIGN} --overwrite \
# -s $(LOCAL_TRAIN_SRC).algtmp.d/$$s \
# -t $(LOCAL_TRAIN_TRG).algtmp.d/$$s \
# -f $(LOCAL_TRAIN_SRC).algtmp.d/$$s.fwd \
# -r $(LOCAL_TRAIN_TRG).algtmp.d/$$s.rev; \
# done;
$(LOCAL_TRAIN_SRC).algtmp.d/%.alg: $(LOCAL_TRAIN_SRC).algtmp.d/% $(LOCAL_TRAIN_TRG).algtmp.d/%
echo "align part ${notdir $<}"
${WORDALIGN} --overwrite \
-s $(word 1,$^) \
-t $(word 2,$^) \
-f $(word 1,$^).fwd \
-r $(word 2,$^).rev
echo "merge and symmetrize part ${notdir $<}"
${ATOOLS} -c grow-diag-final -i $(word 1,$^).fwd -j $(word 2,$^).rev > $@
rm -f $(word 1,$^).fwd $(word 2,$^).rev
## fetch OPUS data, try in this order
##
## (1) check first whether they exist on the local file system
## (2) check that Moses files can be downloaded
## (3) read with opus_read from local file system
## (4) fetch and read with opus_read
##
## TODO:
## - should we do langid filtering and link prob filtering here?
## (could set OPUSREAD_ARGS for that)
##
%.${SRCEXT}.raw:
mkdir -p ${dir $@}
-( c=${patsubst %.${LANGPAIR}.${SRCEXT}.raw,%,${notdir $@}}; \
if [ -e ${OPUSHOME}/$$c/latest/moses/${LANGPAIR}.txt.zip ]; then \
unzip -d ${dir $@} -n ${OPUSHOME}/$$c/latest/moses/${LANGPAIR}.txt.zip; \
mv ${dir $@}$$c*.${LANGPAIR}.${SRCEXT} $@; \
mv ${dir $@}$$c*.${LANGPAIR}.${TRGEXT} ${@:.${SRCEXT}.raw=.${TRGEXT}.raw}; \
rm -f ${@:.${SRCEXT}.raw=.xml} ${@:.${SRCEXT}.raw=.ids} ${dir $@}/README ${dir $@}/LICENSE; \
elif [ "${call url-exists,${call resource-url,${SRCEXT},${TRGEXT},${patsubst %.${LANGPAIR}.${SRCEXT}.raw,%,${notdir $@}}}}" == "1" ]; then \
l="${call resource-url,${SRCEXT},${TRGEXT},${patsubst %.${LANGPAIR}.${SRCEXT}.raw,%,${notdir $@}}}"; \
echo "============================================"; \
echo "fetch moses data from $$l"; \
echo "============================================"; \
${WGET} -qq -O $@-$$c-${LANGPAIR}.zip $$l; \
unzip -d ${dir $@} -n $@-$$c-${LANGPAIR}.zip; \
mv ${dir $@}$$c*.${LANGPAIR}.${SRCEXT} $@; \
mv ${dir $@}$$c*.${LANGPAIR}.${TRGEXT} ${@:.${SRCEXT}.raw=.${TRGEXT}.raw}; \
rm -f ${@:.${SRCEXT}.raw=.xml} ${@:.${SRCEXT}.raw=.ids} ${dir $@}/README ${dir $@}/LICENSE; \
rm -f $@-$$c-${LANGPAIR}.zip; \
elif [ -e ${OPUSHOME}/$$c/latest/xml/${LANGPAIR}.xml.gz ]; then \
echo "============================================"; \
echo "extract $$c (${LANGPAIR}) from XML in local OPUS copy"; \
echo "============================================"; \
opus_read ${OPUSREAD_ARGS} -ln -rd ${OPUSHOME} -d $$c -s ${SRC} -t ${TRG} \
-wm moses -p raw -w $@ ${@:.${SRCEXT}.raw=.${TRGEXT}.raw}; \
else \
echo "============================================"; \
echo "fetch $$c (${LANGPAIR}) from OPUS"; \
echo "============================================"; \
opus_read ${OPUSREAD_ARGS} -ln -q -dl ${TMPWORKDIR} -d $$c -s ${SRC} -t ${TRG} \
-wm moses -p raw -w $@ ${@:.${SRCEXT}.raw=.${TRGEXT}.raw}; \
fi )
# echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"; \
# echo "!! skip $@"; \
# echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"; \
%.${TRGEXT}.raw: %.${SRCEXT}.raw
@echo "done!"
## TODO: does this causes make to frequently redo the same data?
## --> could be a problem with large models!
.INTERMEDIATE: ${LOCAL_TRAIN_SRC} ${LOCAL_TRAIN_TRG}
## define dependency on DEVDATA if they need to be added to the train data
ifeq (${USE_REST_DEVDATA},1)
LOCAL_TRAINDATA_DEPENDENCIES = ${DEV_SRC} ${DEV_TRG}
endif
## for multilingual systems:
## shuffle the complete training data set
## if the option is set to 1
ifeq (${SHUFFLE_MULTILINGUAL_DATA},1)
ifneq ($(words ${SRCLANGS} ${TRGLANGS}),2)
SHUFFLE_TRAINING_DATA = 1
endif
endif
local-train-data: ${LOCAL_TRAIN_SRC} ${LOCAL_TRAIN_TRG}
## add training data for each language combination
## and put it together in local space
${LOCAL_TRAIN_SRC}: ${LOCAL_TRAINDATA_DEPENDENCIES}
${MAKE} ${WORKDIR}/train/size_per_language_pair.txt
@mkdir -p ${dir $@}
@echo "" > ${dir $@}README.md
@echo "# ${notdir ${TRAIN_BASE}}" >> ${dir $@}README.md
@echo "" >> ${dir $@}README.md
@rm -f ${LOCAL_TRAIN_SRC} ${LOCAL_TRAIN_TRG}
-@for s in ${SRCLANGS}; do \
for t in ${TRGLANGS}; do \
if [ ! `echo "$$s-$$t $$t-$$s" | egrep '${SKIP_LANGPAIRS}' | wc -l` -gt 0 ]; then \
if [ "${SKIP_SAME_LANG}" == "1" ] && [ "$$s" == "$$t" ]; then \
echo "!!!!!!!!!!! skip language pair $$s-$$t !!!!!!!!!!!!!!!!"; \
else \
echo "..... add data for $$s-$$t"; \
${MAKE} DATASET=${DATASET} SRC:=$$s TRG:=$$t add-to-local-train-data; \
fi \
else \
echo "!!!!!!!!!!! skip language pair $$s-$$t !!!!!!!!!!!!!!!!"; \
fi \
done \
done
ifeq (${USE_REST_DEVDATA},1)
@if [ -e ${DEV_SRC}.notused.gz ]; then \
echo "..... add unused devdata to training data"; \
echo "* unused dev/test data is added to training data" >> ${dir $@}README.md; \
${GZIP} -cd < ${DEV_SRC}.notused.gz >> ${LOCAL_TRAIN_SRC}; \
${GZIP} -cd < ${DEV_TRG}.notused.gz >> ${LOCAL_TRAIN_TRG}; \
fi
endif
######################################
# run another round of cleaning if
# CLEAN_CORPUS_TRAINING_DATA is set
# --> could be useful if there is
# noisy data in back-translations etc
######################################
ifeq (${CLEAN_CORPUS_TRAINING_DATA},1)
@echo ".... another cleanup of local training data"
@ln -s ${LOCAL_TRAIN_SRC} ${LOCAL_TRAIN_SRC}.${SRCEXT}
@ln -s ${LOCAL_TRAIN_TRG} ${LOCAL_TRAIN_SRC}.${TRGEXT}
@$(MOSESSCRIPTS)/training/clean-corpus-n.perl \
-ratio ${NR_TOKEN_RATIO} \
-max-word-length ${MAX_TOKEN_LENGTH} \
${LOCAL_TRAIN_SRC} $(SRCEXT) $(TRGEXT) \
${LOCAL_TRAIN_SRC}.clean \
${MIN_NR_TOKENS} ${MAX_NR_TOKENS}
@mv -f ${LOCAL_TRAIN_SRC}.clean.${SRCEXT} ${LOCAL_TRAIN_SRC}
@mv -f ${LOCAL_TRAIN_SRC}.clean.${TRGEXT} ${LOCAL_TRAIN_TRG}
@rm -f ${LOCAL_TRAIN_SRC}.${SRCEXT} ${LOCAL_TRAIN_SRC}.${TRGEXT}
endif
ifeq (${SHUFFLE_TRAINING_DATA},1)
@echo ".... shuffle complete training data"
@paste ${LOCAL_TRAIN_SRC} ${LOCAL_TRAIN_TRG} | ${SHUFFLE} | ${GZIP} -c > ${LOCAL_TRAIN_SRC}.shuffled.gz
@${GZIP} -cd ${LOCAL_TRAIN_SRC}.shuffled.gz | cut -f1 > ${LOCAL_TRAIN_SRC}
@${GZIP} -cd ${LOCAL_TRAIN_SRC}.shuffled.gz | cut -f2 > ${LOCAL_TRAIN_TRG}
@rm -f ${LOCAL_TRAIN_SRC}.shuffled.gz
# @paste ${LOCAL_TRAIN_SRC} ${LOCAL_TRAIN_TRG} | ${SHUFFLE} > ${LOCAL_TRAIN_SRC}.shuffled
# @cut -f1 ${LOCAL_TRAIN_SRC}.shuffled > ${LOCAL_TRAIN_SRC}
# @cut -f2 ${LOCAL_TRAIN_SRC}.shuffled > ${LOCAL_TRAIN_TRG}
# @rm -f ${LOCAL_TRAIN_SRC}.shuffled
endif
## everything is done in the target above
${LOCAL_TRAIN_TRG}: ${LOCAL_TRAIN_SRC}
@echo "done!"
## cut the data sets immediately if we don't have
## to shuffle first! This saves a lot of time!
ifneq (${SHUFFLE_DATA},1)
ifdef FIT_DATA_SIZE
CUT_DATA_SETS = | head -${FIT_DATA_SIZE}
endif
endif
## add language labels to the source language
## if we have multiple target languages
ifeq (${USE_TARGET_LABELS},1)
LABEL_SOURCE_DATA = | sed "s/^/>>${TRG}<< /"
endif
## add to the training data
.PHONY: add-to-local-train-data
add-to-local-train-data: ${CLEAN_TRAIN_SRC} ${CLEAN_TRAIN_TRG}
ifneq (${wildcard ${CLEAN_TRAIN_SRC}},)
ifneq (${wildcard ${CLEAN_TRAIN_TRG}},)
ifdef CHECK_TRAINDATA_SIZE
@if [ `${GZCAT} ${wildcard ${CLEAN_TRAIN_SRC}} | wc -l` != `${GZCAT} ${wildcard ${CLEAN_TRAIN_TRG}} | wc -l` ]; then \
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"; \
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"; \
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"; \
echo "source and target are not of same length!"; \
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"; \
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"; \
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"; \
echo ${CLEAN_TRAIN_SRC}; \
echo ${CLEAN_TRAIN_TRG}; \
fi
endif
@echo "..... add info about training data"
@mkdir -p ${dir ${LOCAL_TRAIN_SRC}} ${dir ${LOCAL_TRAIN_TRG}}
@echo -n "* ${SRC}-${TRG}: " >> ${dir ${LOCAL_TRAIN_SRC}}README.md
for d in ${wildcard ${CLEAN_TRAIN_SRC}}; do \
l=`${GZIP} -cd < $$d ${CUT_DATA_SETS} 2>/dev/null | wc -l`; \
if [ $$l -gt 0 ]; then \
echo "$$d" | xargs basename | \
sed -e 's#.${SRC}.gz$$##' \
-e 's#.clean$$##'\
-e 's#.${LANGPAIR}$$##' | tr "\n" ' ' >> ${dir ${LOCAL_TRAIN_SRC}}README.md; \
echo -n "($$l) " >> ${dir ${LOCAL_TRAIN_SRC}}README.md; \
fi \
done
@echo "" >> ${dir ${LOCAL_TRAIN_SRC}}README.md
######################################
# create local data files (add label if necessary)
######################################
@echo "..... create training data in local scratch space"
@${GZCAT} ${wildcard ${CLEAN_TRAIN_SRC}} ${CUT_DATA_SETS} 2>/dev/null \
${LABEL_SOURCE_DATA} > ${LOCAL_TRAIN_SRC}.${LANGPAIR}.src
@${GZCAT} ${wildcard ${CLEAN_TRAIN_TRG}} ${CUT_DATA_SETS} 2>/dev/null \
> ${LOCAL_TRAIN_TRG}.${LANGPAIR}.trg
@touch ${LOCAL_TRAIN_SRC}.${LANGPAIR}.src ${LOCAL_TRAIN_TRG}.${LANGPAIR}.trg
######################################
# SHUFFLE_DATA is set?
# --> shuffle data for each langpair
# --> do this when FIT_DATA_SIZE is set!
######################################
ifeq (${SHUFFLE_DATA},1)
if [ -s ${LOCAL_TRAIN_SRC}.${LANGPAIR}.src ]; then \
echo "..... shuffle training data"; \
paste ${LOCAL_TRAIN_SRC}.${LANGPAIR}.src ${LOCAL_TRAIN_TRG}.${LANGPAIR}.trg |\
${SHUFFLE} > ${LOCAL_TRAIN_SRC}.shuffled; \
cut -f1 ${LOCAL_TRAIN_SRC}.shuffled > ${LOCAL_TRAIN_SRC}.${LANGPAIR}.src; \
cut -f2 ${LOCAL_TRAIN_SRC}.shuffled > ${LOCAL_TRAIN_TRG}.${LANGPAIR}.trg; \
rm -f ${LOCAL_TRAIN_SRC}.shuffled; \
else \
echo "..... empty training data: ${LOCAL_TRAIN_SRC}.${LANGPAIR}.src"; \
fi
endif
######################################
# FIT_DATA_SIZE is set?
# --> fit data to specific size
# --> under/over sampling!
######################################
@echo -n "* ${SRC}-${TRG}: total size = " >> ${dir ${LOCAL_TRAIN_SRC}}README.md
ifdef FIT_DATA_SIZE
@echo "sample data to fit size = ${FIT_DATA_SIZE}"
@${REPOHOME}scripts/fit-data-size.pl -m ${MAX_OVER_SAMPLING} ${FIT_DATA_SIZE} \
${LOCAL_TRAIN_SRC}.${LANGPAIR}.src | wc -l >> ${dir ${LOCAL_TRAIN_SRC}}README.md
@${REPOHOME}scripts/fit-data-size.pl -m ${MAX_OVER_SAMPLING} ${FIT_DATA_SIZE} \
${LOCAL_TRAIN_SRC}.${LANGPAIR}.src >> ${LOCAL_TRAIN_SRC}
@${REPOHOME}scripts/fit-data-size.pl -m ${MAX_OVER_SAMPLING} ${FIT_DATA_SIZE} \
${LOCAL_TRAIN_TRG}.${LANGPAIR}.trg >> ${LOCAL_TRAIN_TRG}
else
@cat ${LOCAL_TRAIN_SRC}.${LANGPAIR}.src | wc -l >> ${dir ${LOCAL_TRAIN_SRC}}README.md
@cat ${LOCAL_TRAIN_SRC}.${LANGPAIR}.src >> ${LOCAL_TRAIN_SRC}
@cat ${LOCAL_TRAIN_TRG}.${LANGPAIR}.trg >> ${LOCAL_TRAIN_TRG}
endif
@rm -f ${LOCAL_TRAIN_SRC}.${LANGPAIR}.src ${LOCAL_TRAIN_TRG}.${LANGPAIR}.trg
endif
endif
####################
# development data
####################
.PHONY: show-devdata
show-devdata:
@echo "${CLEAN_DEV_SRC}"
@echo "${CLEAN_DEV_TRG}"
@echo ${SUBWORD_SRC_MODEL}
@echo ${SUBWORD_TRG_MODEL}
@echo "${DEV_SRC}.${PRE_SRC}"
@echo "${DEV_TRG}.${PRE_TRG}"
.PHONY: raw-devdata
raw-devdata: ${DEV_SRC} ${DEV_TRG}
## TODO: should we have some kind of balanced shuffling
## to avoid bias towards bigger language pairs?
## maybe introduce over/undersampling of dev data like we have for train data?
${DEV_SRC}.shuffled.gz:
mkdir -p ${sort ${dir $@} ${dir ${DEV_SRC}} ${dir ${DEV_TRG}}}
rm -f ${DEV_SRC} ${DEV_TRG}
echo "# Validation data" > ${dir ${DEV_SRC}}README.md
echo "" >> ${dir ${DEV_SRC}}README.md
-for s in ${SRCLANGS}; do \
for t in ${TRGLANGS}; do \
if [ ! `echo "$$s-$$t $$t-$$s" | egrep '${SKIP_LANGPAIRS}' | wc -l` -gt 0 ]; then \
if [ "${SKIP_SAME_LANG}" == "1" ] && [ "$$s" == "$$t" ]; then \
echo "!!!!!!!!!!! skip language pair $$s-$$t !!!!!!!!!!!!!!!!"; \
else \
${MAKE} SRC=$$s TRG=$$t add-to-dev-data; \
fi \
else \
echo "!!!!!!!!!!! skip language pair $$s-$$t !!!!!!!!!!!!!!!!"; \
fi \
done \
done
ifeq (${SHUFFLE_DEVDATA},0)
paste ${DEV_SRC} ${DEV_TRG} | ${GZIP} -c > $@
else
paste ${DEV_SRC} ${DEV_TRG} | ${UNIQ} | ${SHUFFLE} | ${GZIP} -c > $@
endif
echo -n "* total-size-shuffled: " >> ${dir ${DEV_SRC}}README.md
${GZIP} -cd < $@ | wc -l >> ${dir ${DEV_SRC}}README.md
## OLD: don't uniq the dev-data ...
##
# paste ${DEV_SRC} ${DEV_TRG} | ${SHUFFLE} | ${GZIP} -c > $@
# echo -n "* total size of shuffled dev data: " >> ${dir ${DEV_SRC}}README.md
## if we have less than twice the amount of DEVMINSIZE in the data set
## --> extract some data from the training data to be used as devdata
${DEV_SRC}: %: %.shuffled.gz
## if we extract test and dev data from the same data set
## ---> make sure that we do not have any overlap between the two data sets
## ---> reserve at least DEVMINSIZE data for dev data and keep the rest for testing
ifeq (${DEVSET},${TESTSET})
@if (( `${GZIP} -cd < $< | wc -l` < $$((${DEVSIZE} + ${TESTSIZE})) )); then \
if (( `${GZIP} -cd < $< | wc -l` < $$((${DEVSMALLSIZE} + ${DEVMINSIZE})) )); then \
echo "extract ${DEVMINSIZE} examples from ${DEVSET} for dev and test"; \
${GZIP} -cd < $< | cut -f1 | head -${DEVMINSIZE} > ${DEV_SRC}; \
${GZIP} -cd < $< | cut -f2 | head -${DEVMINSIZE} > ${DEV_TRG}; \
mkdir -p ${dir ${TEST_SRC}}; \
${GZIP} -cd < $< | cut -f1 | tail -n +$$((${DEVMINSIZE} + 1)) > ${TEST_SRC}; \
${GZIP} -cd < $< | cut -f2 | tail -n +$$((${DEVMINSIZE} + 1)) > ${TEST_TRG}; \
else \
echo "extract ${DEVSMALLSIZE} examples from ${DEVSET} for dev and test"; \
${GZIP} -cd < $< | cut -f1 | head -${DEVSMALLSIZE} > ${DEV_SRC}; \
${GZIP} -cd < $< | cut -f2 | head -${DEVSMALLSIZE} > ${DEV_TRG}; \
mkdir -p ${dir ${TEST_SRC}}; \
${GZIP} -cd < $< | cut -f1 | tail -n +$$((${DEVSMALLSIZE} + 1)) > ${TEST_SRC}; \
${GZIP} -cd < $< | cut -f2 | tail -n +$$((${DEVSMALLSIZE} + 1)) > ${TEST_TRG}; \
fi; \
else \
echo "extract ${DEVSIZE} examples from ${DEVSET} for dev"; \
echo "extract ${TESTSIZE} examples from ${DEVSET} for test"; \
${GZIP} -cd < $< | cut -f1 | head -${DEVSIZE} > ${DEV_SRC}; \
${GZIP} -cd < $< | cut -f2 | head -${DEVSIZE} > ${DEV_TRG}; \
mkdir -p ${dir ${TEST_SRC}}; \
${GZIP} -cd < $< | cut -f1 | head -$$((${DEVSIZE} + ${TESTSIZE})) | tail -${TESTSIZE} > ${TEST_SRC}; \
${GZIP} -cd < $< | cut -f2 | head -$$((${DEVSIZE} + ${TESTSIZE})) | tail -${TESTSIZE} > ${TEST_TRG}; \
${GZIP} -cd < $< | cut -f1 | tail -n +$$((${DEVSIZE} + ${TESTSIZE} + 1)) | ${GZIP} -c > ${DEV_SRC}.notused.gz; \
${GZIP} -cd < $< | cut -f2 | tail -n +$$((${DEVSIZE} + ${TESTSIZE} + 1)) | ${GZIP} -c > ${DEV_TRG}.notused.gz; \
fi
else
@echo "extract ${DEVSIZE} examples from ${DEVSET} for dev"
@${GZIP} -cd < $< | cut -f1 | head -${DEVSIZE} > ${DEV_SRC}
@${GZIP} -cd < $< | cut -f2 | head -${DEVSIZE} > ${DEV_TRG}
@${GZIP} -cd < $< | cut -f1 | tail -n +$$((${DEVSIZE} + 1)) | ${GZIP} -c > ${DEV_SRC}.notused.gz
@${GZIP} -cd < $< | cut -f2 | tail -n +$$((${DEVSIZE} + 1)) | ${GZIP} -c > ${DEV_TRG}.notused.gz
endif
@echo "" >> ${dir ${DEV_SRC}}/README.md
@echo -n "* devset-selected: top " >> ${dir ${DEV_SRC}}/README.md
@wc -l < ${DEV_SRC} | tr "\n" ' ' >> ${dir ${DEV_SRC}}/README.md
@echo " lines of ${notdir $@}.shuffled" >> ${dir ${DEV_SRC}}/README.md
ifeq (${DEVSET},${TESTSET})
@echo -n "* testset-selected: next " >> ${dir ${DEV_SRC}}/README.md
@wc -l < ${TEST_SRC} | tr "\n" ' ' >> ${dir ${DEV_SRC}}/README.md
@echo " lines of ${notdir $@}.shuffled " >> ${dir ${DEV_SRC}}/README.md
@echo "* devset-unused: added to traindata" >> ${dir ${DEV_SRC}}/README.md
@echo "# Test data" > ${dir ${TEST_SRC}}/README.md
@echo "" >> ${dir ${TEST_SRC}}/README.md
@echo -n "testset-selected: next " >> ${dir ${TEST_SRC}}/README.md
@wc -l < ${TEST_SRC} | tr "\n" ' ' >> ${dir ${TEST_SRC}}/README.md
@echo " lines of ../val/${notdir $@}.shuffled" >> ${dir ${TEST_SRC}}/README.md
endif
${DEV_TRG}: ${DEV_SRC}
@echo "done!"
.PHONY: add-to-dev-data
add-to-dev-data: ${CLEAN_DEV_SRC} ${CLEAN_DEV_TRG}
@echo "add to devset: ${CLEAN_DEV_SRC}"
@mkdir -p ${dir ${DEV_SRC}}
@echo -n "* ${LANGPAIR}: ${DEVSET}, " >> ${dir ${DEV_SRC}}README.md
@${GZCAT} ${CLEAN_DEV_SRC} 2>/dev/null | wc -l >> ${dir ${DEV_SRC}}README.md
#-----------------------------------------------------------------
# sample devdata to balance size between different language pairs
# (only if FIT_DEVDATA_SIZE is set)
#-----------------------------------------------------------------
ifdef FIT_DEVDATA_SIZE
@echo "sample dev data to fit size = ${FIT_DEVDATA_SIZE}"
@${REPOHOME}scripts/fit-data-size.pl -m ${MAX_OVER_SAMPLING} ${FIT_DEVDATA_SIZE} \
${CLEAN_DEV_SRC} 2>/dev/null ${LABEL_SOURCE_DATA} >> ${DEV_SRC}
@${REPOHOME}scripts/fit-data-size.pl -m ${MAX_OVER_SAMPLING} ${FIT_DEVDATA_SIZE} \
${CLEAN_DEV_TRG} 2>/dev/null >> ${DEV_TRG}
else
@${GZCAT} ${CLEAN_DEV_SRC} 2>/dev/null ${LABEL_SOURCE_DATA} >> ${DEV_SRC}
@${GZCAT} ${CLEAN_DEV_TRG} 2>/dev/null >> ${DEV_TRG}
endif
####################
# test data
####################
##
## if devset and testset are from the same source:
## --> use part of the shuffled devset
## otherwise: create the testset
## exception: TESTSET exists in TESTSET_DIR
## --> just use that one
${TEST_SRC}: ${DEV_SRC}
ifneq (${TESTSET},${DEVSET})
mkdir -p ${dir $@}
rm -f ${TEST_SRC} ${TEST_TRG}
echo "# Test data" > ${dir ${TEST_SRC}}/README.md
echo "" >> ${dir ${TEST_SRC}}/README.md
if [ -e ${TESTSET_DIR}/${TESTSET}.${SRCEXT}.${PRE}.gz ]; then \
${MAKE} CLEAN_TEST_SRC=${TESTSET_DIR}/${TESTSET}.${SRCEXT}.${PRE}.gz \
CLEAN_TEST_TRG=${TESTSET_DIR}/${TESTSET}.${TRGEXT}.${PRE}.gz \
add-to-test-data; \
elif [ ! -e $@ ]; then \
for s in ${SRCLANGS}; do \
for t in ${TRGLANGS}; do \
if [ ! `echo "$$s-$$t $$t-$$s" | egrep '${SKIP_LANGPAIRS}' | wc -l` -gt 0 ]; then \
if [ "${SKIP_SAME_LANG}" == "1" ] && [ "$$s" == "$$t" ]; then \
echo "!!!!!!!!!!! skip language pair $$s-$$t !!!!!!!!!!!!!!!!"; \
else \
${MAKE} SRC=$$s TRG=$$t add-to-test-data; \
fi \
else \
echo "!!!!!!!!!!! skip language pair $$s-$$t !!!!!!!!!!!!!!!!"; \
fi \
done \
done; \
if [ ${TESTSIZE} -lt `cat $@ | wc -l` ]; then \
paste ${TEST_SRC} ${TEST_TRG} | ${SHUFFLE} | ${GZIP} -c > $@.shuffled.gz; \
${GZIP} -cd < $@.shuffled.gz | cut -f1 | tail -${TESTSIZE} > ${TEST_SRC}; \
${GZIP} -cd < $@.shuffled.gz | cut -f2 | tail -${TESTSIZE} > ${TEST_TRG}; \
echo "" >> ${dir $@}/README.md; \
echo "testset-selected: top ${TESTSIZE} lines of $@.shuffled!" >> ${dir $@}/README.md; \
fi \
else \
echo "test set $@ exists already! Don't overwrite!"; \
echo "TODO: should we touch it?"; \
fi
else
mkdir -p ${dir $@}
if [ -e ${TESTSET_DIR}/${TESTSET}.${SRCEXT}.${PRE}.gz ]; then \
${MAKE} CLEAN_TEST_SRC=${TESTSET_DIR}/${TESTSET}.${SRCEXT}.${PRE}.gz \
CLEAN_TEST_TRG=${TESTSET_DIR}/${TESTSET}.${TRGEXT}.${PRE}.gz \
add-to-test-data; \
elif (( `${GZIP} -cd < $<.shuffled.gz | wc -l` < $$((${DEVSIZE} + ${TESTSIZE})) )); then \
${GZIP} -cd < $<.shuffled.gz | cut -f1 | tail -n +$$((${DEVMINSIZE} + 1)) > ${TEST_SRC}; \
${GZIP} -cd < $<.shuffled.gz | cut -f2 | tail -n +$$((${DEVMINSIZE} + 1)) > ${TEST_TRG}; \
else \
${GZIP} -cd < $<.shuffled.gz | cut -f1 | tail -${TESTSIZE} > ${TEST_SRC}; \
${GZIP} -cd < $<.shuffled.gz | cut -f2 | tail -${TESTSIZE} > ${TEST_TRG}; \
fi
endif
${TEST_TRG}: ${TEST_SRC}
@echo "done!"
.PHONY: add-to-test-data
add-to-test-data: ${CLEAN_TEST_SRC}
@echo "add to testset: ${CLEAN_TEST_SRC}"
@echo "* ${LANGPAIR}: ${TESTSET}" >> ${dir ${TEST_SRC}}README.md
@${GZCAT} ${CLEAN_TEST_SRC} 2>/dev/null ${LABEL_SOURCE_DATA} >> ${TEST_SRC}
@${GZCAT} ${CLEAN_TEST_TRG} 2>/dev/null >> ${TEST_TRG}
## reduce training data size if necessary
ifdef TRAINSIZE
${TRAIN_SRC}.clean.${PRE_SRC}${TRAINSIZE}.gz: ${TRAIN_SRC}.clean.${PRE_SRC}.gz
${GZIP} -cd < $< | head -${TRAINSIZE} | ${GZIP} -c > $@
${TRAIN_TRG}.clean.${PRE_TRG}${TRAINSIZE}.gz: ${TRAIN_TRG}.clean.${PRE_TRG}.gz
${GZIP} -cd < $< | head -${TRAINSIZE} | ${GZIP} -c > $@
endif
## monolingual data: for language-specific sentence piece models
## that are independent of bitexts
## TODO: do we use this?
${LOCAL_MONO_DATA}.raw:
mkdir -p ${dir $@}
rm -f $@
-for l in ${LANGS}; do \
${MAKE} DATASET=${DATASET} LANGID:=$$l \
add-to-local-mono-data; \
done
## TODO: if it does not exist in local file system then use opus-tools to fetch!
.PHONY: add-to-local-mono-data
add-to-local-mono-data:
for c in ${MONOSET}; do \
if [ -e ${OPUSHOME}/$$c/latest/mono/${LANGID}.txt.gz ]; then \
${GZIP} -cd < ${OPUSHOME}/$$c/latest/mono/${LANGID}.txt.gz |\
${REPOHOME}scripts/filter/mono-match-lang.py -l ${LANGID} >> ${LOCAL_MONO_DATA}.raw; \
fi \
done
##----------------------------------------------
## get data from local space and compress ...
##----------------------------------------------
${WORKDIR}/%.${PRE_SRC}.gz: ${TMPWORKDIR}/${LANGPAIRSTR}/%.${PRE_SRC}
mkdir -p ${dir $@}
${GZIP} -c < $< > $@
-cat ${dir $<}README.md >> ${dir $@}README.md
ifneq (${PRE_SRC},${PRE_TRG})
${WORKDIR}/%.${PRE_TRG}.gz: ${TMPWORKDIR}/${LANGPAIRSTR}/%.${PRE_TRG}
mkdir -p ${dir $@}
${GZIP} -c < $< > $@
endif
include ${REPOHOME}lib/preprocess.mk
include ${REPOHOME}lib/bpe.mk
include ${REPOHOME}lib/sentencepiece.mk