tico19 benchmark added

doc/TatoebaChallenge.md

@@ -172,3 +172,16 @@ Various lists and tables can be generated from the evaluated model files. Remove
 rm -f tatoeba-results* results/*.md
 make tatoeba-results-md
 ```
+
+
+## Convert models to huggingface
+
+
+Look at this [README](https://github.com/huggingface/transformers/tree/master/scripts/tatoeba) in the transformers library.
+
+```
+python src/transformers/convert_marian_tatoeba_to_pytorch.py --models heb-eng eng-heb --save_dir converted
+cd converted
+transformers-cli login
+for FILE in *; do transformers-cli upload --organization Helsinki-NLP $FILE; done
+```

lib/dist.mk

@@ -283,9 +283,9 @@ endif
 
 .PHONY: upload
 upload:
-	find ${MODELSHOME}/ -type l | tar -cf models-links.tar -T -
-	find ${MODELSHOME}/ -type l -delete
-	cd ${MODELSHOME} && swift upload ${MODEL_CONTAINER} --changed --skip-identical *
+	find ${RELEASEDIR}/ -type l | tar -cf models-links.tar -T -
+	find ${RELEASEDIR}/ -type l -delete
+	cd ${RELEASEDIR} && swift upload ${MODEL_CONTAINER} --changed --skip-identical *
 	tar -xf models-links.tar
 	rm -f models-links.tar
 	swift post ${MODEL_CONTAINER} --read-acl ".r:*"
@@ -296,8 +296,8 @@ upload:
 
 .PHONY: upload-models
 upload-models:
-	find ${WORKHOME}/models -type l | tar -cf dev-models-links.tar -T -
-	find ${WORKHOME}/models -type l -delete
+	find ${MODELSHOME} -type l | tar -cf dev-models-links.tar -T -
+	find ${MODELSHOME} -type l -delete
 	cd ${WORKHOME} && swift upload ${DEV_MODEL_CONTAINER} --changed --skip-identical models
 	tar -xf dev-models-links.tar
 	rm -f dev-models-links.tar
@@ -352,12 +352,12 @@ ${WORKHOME}/eval/scores.txt: ${EVALSCORES} ${EVALTRANSL}
 	rm -f $@.1 $@.2 $@.3 $@.4
 
 
-${EVALSCORES}: # ${WORKHOME}/eval/%.eval.txt: ${WORKHOME}/models/%.eval
+${EVALSCORES}: # ${WORKHOME}/eval/%.eval.txt: ${MODELSHOME}/%.eval
 	mkdir -p ${dir $@}
 	cp ${patsubst ${WORKHOME}/eval/%.eval.txt,${WORKHOME}/%.eval,$@} $@
 #	cp $< $@
 
-${EVALTRANSL}: # ${WORKHOME}/eval/%.test.txt: ${WORKHOME}/models/%.compare
+${EVALTRANSL}: # ${WORKHOME}/eval/%.test.txt: ${MODELSHOME}/%.compare
 	mkdir -p ${dir $@}
 	cp ${patsubst ${WORKHOME}/eval/%.test.txt,${WORKHOME}/%.compare,$@} $@
 #	cp $< $@
@@ -429,7 +429,7 @@ old-models-dist:
 	  ${MAKE} SRCLANGS="`echo $$l | cut -f1 -d'-' | sed 's/\\+/ /g'`" \
 		  TRGLANGS="`echo $$l | cut -f2 -d'-' | sed 's/\\+/ /g'`" \
 	          WORKHOME=${WORKHOME}/old-models \
-	          MODELSHOME=${WORKHOME}/models dist; \
+	          MODELSHOME=${MODELSHOME} dist; \
 	done
 	@echo "trained double ${words ${TRAINED_DOUBLE_MODELS}}"
 	for l in ${TRAINED_DOUBLE_MODELS}; do \
@@ -439,7 +439,7 @@ old-models-dist:
 	    ${MAKE} SRCLANGS="`echo $$l | cut -f1 -d'-' | sed 's/\\+/ /g'`" \
 		    TRGLANGS="`echo $$l | cut -f2 -d'-' | sed 's/\\+/ /g'`" \
 	            WORKHOME=${WORKHOME}/old-models \
-	            MODELSHOME=${WORKHOME}/models dist; \
+	            MODELSHOME=${MODELSHOME} dist; \
 	  else \
 	    echo "$$l: new better than old"; \
 	  fi \

lib/projects/wikimedia.mk

@@ -10,7 +10,8 @@ as-en:
 	${MAKE} train-dynamic-en-as
 
 # ENAS_BPE = 4000
-ENAS_BPE = 1000
+ENAS_BPE  = 1000
+ENBCL_BPE = 1000
 
 %-as-en:
 	${MAKE} HELDOUTSIZE=0 DEVSIZE=1000 TESTSIZE=1000 DEVMINSIZE=100 BPESIZE=${ENAS_BPE} \
@@ -23,3 +24,9 @@ ENAS_BPE = 1000
 	${@:-en-as=}
 
 
+%-en-bcl:
+	${MAKE} HELDOUTSIZE=0 DEVSIZE=1000 TESTSIZE=1000 DEVMINSIZE=100 BPESIZE=${ENBCL_BPE} \
+		SRCLANGS="en" TRGLANGS="bcl" EXCLUDE_CORPORA="WMT-News MPC1 wikimedia" \
+	${@:-en-bcl=}
+
+

testsets/Makefile

@@ -7,6 +7,32 @@ TXT = $(patsubst %.sgm,%,${wildcard *.sgm})
 
 all: ${TXT}
 
+
+
+## TICO-19 translation benchmark
+## from https://tico-19.github.io/index.html
+
+TICO19_TEST = ${patsubst tico19-testset/test/test.%.tsv,%/tico19-test.en.gz,${wildcard tico19-testset/test/*.tsv}}
+
+tico19-testset:
+	wget https://tico-19.github.io/data/tico19-testset.zip
+	unzip tico19-testset.zip
+	rm -f tico19-testset.zip
+	rm -fr __MACOSX
+	${MAKE} tico19-testdata
+
+.PHONY: tico19-testdata
+tico19-testdata: ${TICO19_TEST}
+
+${TICO19_TEST}: %/tico19-test.en.gz: tico19-testset/test/test.%.tsv
+	mkdir -p ${dir $@}
+	cut -f3 $< | tail -n +2 | sed 's/^ *//;s/ *$$//' | gzip -c > $@
+	cut -f4 $< | tail -n +2 | sed 's/^ *//;s/ *$$//' | gzip -c > ${@:en.gz=${patsubst en-%/,%,$(dir $@)}}.gz
+
+
+
+
+
 list-files:
 	@echo "${2LETTER_FILES}" | tr ' ' "\n"
 

testsets/en-am/tico19-test.amh.gz

@@ -0,0 +1 @@
+tico19-test.am.gz

testsets/en-am/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-ar/tico19-test.ara.gz

@@ -0,0 +1 @@
+tico19-test.ar.gz

testsets/en-ar/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-bn/tico19-test.ben.gz

@@ -0,0 +1 @@
+tico19-test.bn.gz

testsets/en-bn/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-fa/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-fa/tico19-test.fas.gz

@@ -0,0 +1 @@
+tico19-test.fa.gz

testsets/en-fr/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-fr/tico19-test.fra.gz

@@ -0,0 +1 @@
+tico19-test.fr.gz

testsets/en-ha/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-ha/tico19-test.hau.gz

@@ -0,0 +1 @@
+tico19-test.ha.gz

testsets/en-hi/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-hi/tico19-test.hin.gz

@@ -0,0 +1 @@
+tico19-test.hi.gz

testsets/en-id/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-id/tico19-test.msa.gz

@@ -0,0 +1 @@
+tico19-test.id.gz

testsets/en-km/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-km/tico19-test.khm.gz

@@ -0,0 +1 @@
+tico19-test.km.gz

testsets/en-kr/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-kr/tico19-test.kau.gz

@@ -0,0 +1 @@
+tico19-test.kr.gz

testsets/en-ku/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-ku/tico19-test.kur.gz

@@ -0,0 +1 @@
+tico19-test.ku.gz

testsets/en-lg/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-lg/tico19-test.lug.gz

@@ -0,0 +1 @@
+tico19-test.lg.gz

testsets/en-ln/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-ln/tico19-test.lin.gz

@@ -0,0 +1 @@
+tico19-test.ln.gz

testsets/en-mr/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-mr/tico19-test.mar.gz

@@ -0,0 +1 @@
+tico19-test.mr.gz

testsets/en-ms/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-ms/tico19-test.msa.gz

@@ -0,0 +1 @@
+tico19-test.ms.gz

testsets/en-my/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-my/tico19-test.mya.gz

@@ -0,0 +1 @@
+tico19-test.my.gz

testsets/en-ne/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-ne/tico19-test.nep.gz

@@ -0,0 +1 @@
+tico19-test.ne.gz

testsets/en-om/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-om/tico19-test.orm.gz

@@ -0,0 +1 @@
+tico19-test.om.gz

testsets/en-ps/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-ps/tico19-test.pus.gz

@@ -0,0 +1 @@
+tico19-test.ps.gz

testsets/en-ru/tico19-test.eng.gz

@@ -0,0 +1 @@
+tico19-test.en.gz

testsets/en-ru/tico19-test.rus.gz

@@ -0,0 +1 @@
+tico19-test.ru.gz