diff --git a/Makefile.dist b/Makefile.dist index 0892df3e..32202008 100644 --- a/Makefile.dist +++ b/Makefile.dist @@ -202,14 +202,23 @@ endif # - make upload-images .. images of VMs that run OPUS-MT upload: + find models/ -type l | tar -cf models-links.tar -T - + find models/ -type l -delete cd models && swift upload OPUS-MT-models --changed --skip-identical * + tar -xf models-links.tar + rm -f models-links.tar swift post OPUS-MT-models --read-acl ".r:*" swift list OPUS-MT-models > index.txt swift upload OPUS-MT-models index.txt rm -f index.txt + upload-models: + find ${WORKHOME}/models -type l | tar -cf dev-models-links.tar -T - + find ${WORKHOME}/models -type l -delete cd ${WORKHOME} && swift upload OPUS-MT-dev --changed --skip-identical models + tar -xf dev-models-links.tar + rm -f dev-models-links.tar swift post OPUS-MT-dev --read-acl ".r:*" swift list OPUS-MT-dev > index.txt swift upload OPUS-MT-dev index.txt diff --git a/Makefile.generic b/Makefile.generic index d03f1dc4..932ce8f3 100644 --- a/Makefile.generic +++ b/Makefile.generic @@ -84,6 +84,7 @@ endif ${MAKE} DATASET=opus+bt \ CLEAN_TRAIN_SRC="${CLEAN_TRAIN_SRC} ${BACKTRANS_SRC}" \ CLEAN_TRAIN_TRG="${CLEAN_TRAIN_TRG} ${BACKTRANS_TRG}" \ + MARIAN_EARLY_STOPPING=15 \ ${@:-add-backtranslations=} diff --git a/backtranslate/Makefile b/backtranslate/Makefile index dab49b56..5723accb 100644 --- a/backtranslate/Makefile +++ b/backtranslate/Makefile @@ -49,6 +49,12 @@ WIKI_SRC = ${LANGPAIR}/${WIKISOURCE}.${PART}_${MODELNAME}.${LANGPAIR}.${SRC}.gz WIKI_PRE = ${LANGPAIR}/${WIKISOURCE}.${PART}_${MODELNAME}.${LANGPAIR}.${SRC}.spm.gz WIKI_TRG = ${LANGPAIR}/${WIKISOURCE}.${PART}_${MODELNAME}.${LANGPAIR}.${TRG}.gz +## all parts of this wiki +PARTS = ${sort ${patsubst ${WIKI_DIR}/${WIKISOURCE}.${LANGID}.%.gz,%,${wildcard ${WIKI_DIR}/${WIKISOURCE}.${LANGID}.*.gz}}} + +ttt: + echo ${PARTS} + ## don't delete translated text if the process crashes .PRECIOUS: ${WIKI_TRG} @@ -81,7 +87,7 @@ all-wikis: echo "find ${WIKI_DIR} -name '$$w.${LANGID}.${PART}.gz'"; \ if [ `find ${WIKI_DIR} -name "$$w.${LANGID}.${PART}.gz" | wc -l` -gt 0 ]; then \ echo "${MAKE} WIKISOURCE=$$w HPC_CORES=1 WALLTIME=72 translate.submit"; \ - ${MAKE} WIKISOURCE=$$w HPC_CORES=1 WALLTIME=72 translate.submit; \ + ${MAKE} WIKISOURCE=$$w HPC_MEM=4g HPC_CORES=1 WALLTIME=72 translate.submit; \ fi \ done @@ -103,6 +109,20 @@ prepare-model: ${LANGPAIR}/decoder.yml prepare-data: ${WIKI_PRE} translate: ${WIKI_SRC} ${WIKI_TRG} +## translate all parts +translate-all-parts: + for p in ${PARTS}; do \ + ${MAKE} PART=$$p translate; \ + done + +## create jobs for translating all parts +submit-translate-all-parts: + for p in ${PARTS}; do \ + ${MAKE} PART=$$p translate.submit; \ + done + + + print-names: echo ${LANGNAME} echo ${UDPIPE_MODEL} diff --git a/models/da-de/README.md b/models/da-de/README.md index ae92e8f7..016276d3 100644 --- a/models/da-de/README.md +++ b/models/da-de/README.md @@ -13,3 +13,18 @@ |-----------------------|-------|-------| | Tatoeba.da.de | 57.4 | 0.741 | +# opus-2020-01-26.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-26.zip](https://object.pouta.csc.fi/OPUS-MT-models/da-de/opus-2020-01-26.zip) +* test set translations: [opus-2020-01-26.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/da-de/opus-2020-01-26.test.txt) +* test set scores: [opus-2020-01-26.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/da-de/opus-2020-01-26.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.da.de | 57.4 | 0.740 | + diff --git a/models/de-da/README.md b/models/de-da/README.md new file mode 100644 index 00000000..4c81c20c --- /dev/null +++ b/models/de-da/README.md @@ -0,0 +1,15 @@ +# opus-2020-01-29.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-29.zip](https://object.pouta.csc.fi/OPUS-MT-models/de-da/opus-2020-01-29.zip) +* test set translations: [opus-2020-01-29.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/de-da/opus-2020-01-29.test.txt) +* test set scores: [opus-2020-01-29.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/de-da/opus-2020-01-29.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.de.da | 57.2 | 0.730 | + diff --git a/models/de-he/README.md b/models/de-he/README.md new file mode 100644 index 00000000..93528ec7 --- /dev/null +++ b/models/de-he/README.md @@ -0,0 +1,30 @@ +# opus-2020-01-26.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-26.zip](https://object.pouta.csc.fi/OPUS-MT-models/de-he/opus-2020-01-26.zip) +* test set translations: [opus-2020-01-26.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/de-he/opus-2020-01-26.test.txt) +* test set scores: [opus-2020-01-26.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/de-he/opus-2020-01-26.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.de.he | 36.6 | 0.580 | + +# opus-2020-01-29.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-29.zip](https://object.pouta.csc.fi/OPUS-MT-models/de-he/opus-2020-01-29.zip) +* test set translations: [opus-2020-01-29.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/de-he/opus-2020-01-29.test.txt) +* test set scores: [opus-2020-01-29.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/de-he/opus-2020-01-29.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.de.he | 36.6 | 0.581 | + diff --git a/models/de-hr/README.md b/models/de-hr/README.md new file mode 100644 index 00000000..a3e0d562 --- /dev/null +++ b/models/de-hr/README.md @@ -0,0 +1,15 @@ +# opus-2020-01-26.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-26.zip](https://object.pouta.csc.fi/OPUS-MT-models/de-hr/opus-2020-01-26.zip) +* test set translations: [opus-2020-01-26.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/de-hr/opus-2020-01-26.test.txt) +* test set scores: [opus-2020-01-26.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/de-hr/opus-2020-01-26.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.de.hr | 42.6 | 0.643 | + diff --git a/models/es-el/README.md b/models/es-el/README.md new file mode 100644 index 00000000..79d980df --- /dev/null +++ b/models/es-el/README.md @@ -0,0 +1,15 @@ +# opus-2020-01-29.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-29.zip](https://object.pouta.csc.fi/OPUS-MT-models/es-el/opus-2020-01-29.zip) +* test set translations: [opus-2020-01-29.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/es-el/opus-2020-01-29.test.txt) +* test set scores: [opus-2020-01-29.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/es-el/opus-2020-01-29.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.es.el | 48.6 | 0.661 | + diff --git a/models/es-it/README.md b/models/es-it/README.md new file mode 100644 index 00000000..816e5270 --- /dev/null +++ b/models/es-it/README.md @@ -0,0 +1,15 @@ +# opus-2020-01-29.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-29.zip](https://object.pouta.csc.fi/OPUS-MT-models/es-it/opus-2020-01-29.zip) +* test set translations: [opus-2020-01-29.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/es-it/opus-2020-01-29.test.txt) +* test set scores: [opus-2020-01-29.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/es-it/opus-2020-01-29.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.es.it | 55.9 | 0.751 | + diff --git a/models/fi-ru/README.md b/models/fi-ru/README.md new file mode 100644 index 00000000..a4549bb1 --- /dev/null +++ b/models/fi-ru/README.md @@ -0,0 +1,15 @@ +# opus-2020-01-26.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-26.zip](https://object.pouta.csc.fi/OPUS-MT-models/fi-ru/opus-2020-01-26.zip) +* test set translations: [opus-2020-01-26.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/fi-ru/opus-2020-01-26.test.txt) +* test set scores: [opus-2020-01-26.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/fi-ru/opus-2020-01-26.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.fi.ru | 45.9 | 0.662 | + diff --git a/models/fi-tr/README.md b/models/fi-tr/README.md new file mode 100644 index 00000000..5bbe3026 --- /dev/null +++ b/models/fi-tr/README.md @@ -0,0 +1,15 @@ +# opus-2020-01-26.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-26.zip](https://object.pouta.csc.fi/OPUS-MT-models/fi-tr/opus-2020-01-26.zip) +* test set translations: [opus-2020-01-26.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/fi-tr/opus-2020-01-26.test.txt) +* test set scores: [opus-2020-01-26.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/fi-tr/opus-2020-01-26.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.fi.tr | 30.5 | 0.601 | + diff --git a/models/fr-hu/README.md b/models/fr-hu/README.md new file mode 100644 index 00000000..50c848ca --- /dev/null +++ b/models/fr-hu/README.md @@ -0,0 +1,15 @@ +# opus-2020-01-26.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-26.zip](https://object.pouta.csc.fi/OPUS-MT-models/fr-hu/opus-2020-01-26.zip) +* test set translations: [opus-2020-01-26.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/fr-hu/opus-2020-01-26.test.txt) +* test set scores: [opus-2020-01-26.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/fr-hu/opus-2020-01-26.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.fr.hu | 41.3 | 0.629 | + diff --git a/models/he-de/README.md b/models/he-de/README.md new file mode 100644 index 00000000..c44c104b --- /dev/null +++ b/models/he-de/README.md @@ -0,0 +1,15 @@ +# opus-2020-01-26.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-26.zip](https://object.pouta.csc.fi/OPUS-MT-models/he-de/opus-2020-01-26.zip) +* test set translations: [opus-2020-01-26.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/he-de/opus-2020-01-26.test.txt) +* test set scores: [opus-2020-01-26.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/he-de/opus-2020-01-26.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.he.de | 45.5 | 0.647 | + diff --git a/models/hu-sv/README.md b/models/hu-sv/README.md new file mode 100644 index 00000000..19958d9e --- /dev/null +++ b/models/hu-sv/README.md @@ -0,0 +1,15 @@ +# opus-2020-01-26.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-26.zip](https://object.pouta.csc.fi/OPUS-MT-models/hu-sv/opus-2020-01-26.zip) +* test set translations: [opus-2020-01-26.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/hu-sv/opus-2020-01-26.test.txt) +* test set scores: [opus-2020-01-26.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/hu-sv/opus-2020-01-26.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.hu.sv | 52.6 | 0.686 | + diff --git a/models/it-es/README.md b/models/it-es/README.md new file mode 100644 index 00000000..629278b4 --- /dev/null +++ b/models/it-es/README.md @@ -0,0 +1,15 @@ +# opus-2020-01-26.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-26.zip](https://object.pouta.csc.fi/OPUS-MT-models/it-es/opus-2020-01-26.zip) +* test set translations: [opus-2020-01-26.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/it-es/opus-2020-01-26.test.txt) +* test set scores: [opus-2020-01-26.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/it-es/opus-2020-01-26.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.it.es | 61.2 | 0.761 | + diff --git a/models/ru-fr/README.md b/models/ru-fr/README.md index da6572f2..5ad6ca60 100644 --- a/models/ru-fr/README.md +++ b/models/ru-fr/README.md @@ -13,3 +13,20 @@ |-----------------------|-------|-------| | Tatoeba.ru.fr | 51.1 | 0.666 | +# opus-2020-01-26.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-26.zip](https://object.pouta.csc.fi/OPUS-MT-models/ru-fr/opus-2020-01-26.zip) +* test set translations: [opus-2020-01-26.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/ru-fr/opus-2020-01-26.test.txt) +* test set scores: [opus-2020-01-26.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/ru-fr/opus-2020-01-26.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| newstest2012.ru.fr | 18.3 | 0.497 | +| newstest2013.ru.fr | 21.6 | 0.516 | +| Tatoeba.ru.fr | 51.5 | 0.670 | + diff --git a/models/sv-hu/README.md b/models/sv-hu/README.md new file mode 100644 index 00000000..4083959c --- /dev/null +++ b/models/sv-hu/README.md @@ -0,0 +1,15 @@ +# opus-2020-01-26.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-26.zip](https://object.pouta.csc.fi/OPUS-MT-models/sv-hu/opus-2020-01-26.zip) +* test set translations: [opus-2020-01-26.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/sv-hu/opus-2020-01-26.test.txt) +* test set scores: [opus-2020-01-26.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/sv-hu/opus-2020-01-26.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.sv.hu | 44.6 | 0.660 | + diff --git a/models/tr-es/README.md b/models/tr-es/README.md index 909a8ba8..fe2e2fca 100644 --- a/models/tr-es/README.md +++ b/models/tr-es/README.md @@ -13,3 +13,18 @@ |-----------------------|-------|-------| | Tatoeba.tr.es | 56.3 | 0.722 | +# opus-2020-01-26.zip + +* dataset: opus +* model: transformer-align +* pre-processing: normalization + SentencePiece +* download: [opus-2020-01-26.zip](https://object.pouta.csc.fi/OPUS-MT-models/tr-es/opus-2020-01-26.zip) +* test set translations: [opus-2020-01-26.test.txt](https://object.pouta.csc.fi/OPUS-MT-models/tr-es/opus-2020-01-26.test.txt) +* test set scores: [opus-2020-01-26.eval.txt](https://object.pouta.csc.fi/OPUS-MT-models/tr-es/opus-2020-01-26.eval.txt) + +## Benchmarks + +| testset | BLEU | chr-F | +|-----------------------|-------|-------| +| Tatoeba.tr.es | 56.3 | 0.722 | + diff --git a/work-spm/eval/scores.txt b/work-spm/eval/scores.txt index fc331200..2b5be290 100644 --- a/work-spm/eval/scores.txt +++ b/work-spm/eval/scores.txt @@ -41,7 +41,37 @@ am sv JW300.am.sv 21.0 0.377 ar de Tatoeba.ar.de 43.0 0.614 ar en Tatoeba.ar.en 49.4 0.661 ar fi JW300.ar.fi 18.4 0.415 -ar fr Tatoeba.ar.fr 43.2 0.600 +ar fr Tatoeba.ar.fr 43.5 0.602 +ar nd JW300.ar.nd 0.0 0.021 +ar ne JW300.ar.ne 1.6 0.103 +ar ng JW300.ar.ng 0.1 0.029 +ar niu JW300.ar.niu 4.7 0.238 +ar nl GlobalVoices.ar.nl 12.1 0.388 +ar nr JW300.ar.nr 0.0 0.006 +ar nso JW300.ar.nso 4.0 0.204 +ar nya JW300.ar.nya 1.2 0.124 +ar ny JW300.ar.ny 3.1 0.178 +ar nyk JW300.ar.nyk 0.0 0.003 +ar nyn JW300.ar.nyn 0.3 0.051 +ar nzi JW300.ar.nzi 0.4 0.045 +ar oke JW300.ar.oke 2.1 0.157 +ar om JW300.ar.om 0.9 0.146 +ar os JW300.ar.os 1.5 0.155 +ar pag JW300.ar.pag 3.1 0.185 +ar pa JW300.ar.pa 1.7 0.131 +ar pap JW300.ar.pap 2.9 0.166 +ar pck bible-uedin.ar.pck 4.2 0.220 +ar pcm JW300.ar.pcm 1.6 0.124 +ar pes bible-uedin.ar.pes 3.5 0.213 +ar pis JW300.ar.pis 3.1 0.190 +ar pl GlobalVoices.ar.pl 2.5 0.176 +ar plt bible-uedin.ar.plt 4.0 0.282 +ar pon JW300.ar.pon 1.9 0.177 +ar prl JW300.ar.prl 0.5 0.139 +ar pso JW300.ar.pso 0.0 0.055 +ar que JW300.ar.que 0.0 0.006 +ar qu JW300.ar.qu 0.2 0.039 +ar quy JW300.ar.quy 0.0 0.048 ar sv GlobalVoices.ar.sv 12.9 0.386 as de JW300.as.de 1.1 0.176 ase de JW300.ase.de 27.2 0.478 @@ -180,6 +210,8 @@ cmn+cn+yue+ze_zh+zh_cn+zh_CN+zh_HK+zh_tw+zh_TW+zh_yue+zhs+zht+zh de Tatoeba.cmn. cmn+cn+yue+ze_zh+zh_cn+zh_CN+zh_HK+zh_tw+zh_TW+zh_yue+zhs+zht+zh fi bible-uedin.cmn.fi 21.6 0.497 cmn+cn+yue+ze_zh+zh_cn+zh_CN+zh_HK+zh_tw+zh_TW+zh_yue+zhs+zht+zh sv Tatoeba.cmn.sv 46.6 0.620 cnh en JW300.cnh.en 6.9 0.240 +cnh es JW300.cnh.es 3.4 0.167 +cnh fr JW300.cnh.fr 4.2 0.183 crp de bible-uedin.crp.de 2.5 0.190 crp es bible-uedin.crp.es 2.8 0.187 crp fi bible-uedin.crp.fi 2.0 0.181 @@ -236,6 +268,7 @@ csn fr JW300.csn.fr 3.5 0.164 csn sv JW300.csn.sv 3.8 0.173 cs sv JW300.cs.sv 30.6 0.527 ctu en JW300.ctu.en 2.9 0.157 +ctu es JW300.ctu.es 4.9 0.210 ctu fr JW300.ctu.fr 3.3 0.166 cv de JW300.cv.de 1.4 0.148 cv en JW300.cv.en 2.6 0.151 @@ -250,7 +283,7 @@ cy es JW300.cy.es 0.0 0.025 cy fi JW300.cy.fi 0.3 0.067 cy fr JW300.cy.fr 8.7 0.266 cy sv JW300.cy.sv 6.6 0.218 -da de Tatoeba.da.de 57.4 0.741 +da de Tatoeba.da.de 57.4 0.740 da en Tatoeba.da.en 63.6 0.769 da es Tatoeba.da.es 53.7 0.715 da fi Tatoeba.da.fi 39.0 0.629 @@ -301,6 +334,7 @@ de csn JW300.de.csn 2.7 0.154 de cs Tatoeba.de.cs 42.2 0.625 de cv JW300.de.cv 2.0 0.144 de cy JW300.de.cy 4.1 0.187 +de da Tatoeba.de.da 57.1 0.731 de de Tatoeba.de.de 40.7 0.616 de dhv JW300.de.dhv 5.6 0.241 de dje bible-uedin.de.dje 4.9 0.223 @@ -359,9 +393,11 @@ de guw JW300.de.guw 27.1 0.472 de gv bible-uedin.de.gv 0.0 0.028 de gym JW300.de.gym 3.4 0.218 de ha JW300.de.ha 20.7 0.417 +de he Tatoeba.de.he 36.6 0.580 de hi JW300.de.hi 4.2 0.162 de hil JW300.de.hil 33.9 0.563 de ho JW300.de.ho 22.6 0.461 +de hr Tatoeba.de.hr 42.6 0.643 de hsb Tatoeba.de.hsb 0.1 0.042 de ht JW300.de.ht 21.8 0.390 de hu Tatoeba.de.hu 34.3 0.588 @@ -462,6 +498,7 @@ de qvi JW300.de.qvi 2.0 0.178 de rar JW300.de.rar 4.6 0.207 dhv de JW300.dhv.de 1.5 0.144 dhv en JW300.dhv.en 4.7 0.190 +dhv es JW300.dhv.es 0.0 0.000 dhv fi JW300.dhv.fi 1.2 0.149 dhv fr JW300.dhv.fr 3.2 0.157 dhv sv JW300.dhv.sv 2.5 0.160 @@ -472,7 +509,9 @@ dje fi bible-uedin.dje.fi 3.0 0.211 dje fr bible-uedin.dje.fr 4.1 0.211 dje sv bible-uedin.dje.sv 4.4 0.214 djk en JW300.djk.en 4.3 0.177 +djk fi JW300.djk.fi 0.1 0.097 djk fr JW300.djk.fr 3.0 0.156 +djk sv JW300.djk.sv 0.3 0.115 ecs es JW300.ecs.es 16.2 0.423 ee de JW300.ee.de 22.3 0.430 ee en JW300.ee.en 39.3 0.556 @@ -855,6 +894,7 @@ es cat JW300.es.cat 3.5 0.175 es ceb JW300.es.ceb 33.9 0.564 es chk JW300.es.chk 17.1 0.378 es cjk JW300.es.cjk 3.5 0.214 +es cnh JW300.es.cnh 4.5 0.210 es crp bible-uedin.es.crp 4.5 0.202 es crs JW300.es.crs 26.4 0.453 es csg JW300.es.csg 91.2 0.937 @@ -868,6 +908,7 @@ es cs newstest2012.es.cs 20.9 0.476 es cs newstest2013.es.cs 23.7 0.502 es csn JW300.es.csn 87.8 0.901 es cs Tatoeba.es.cs 46.4 0.655 +es ctu JW300.es.ctu 8.7 0.291 es cv JW300.es.cv 1.8 0.136 es cy JW300.es.cy 5.2 0.211 es da Tatoeba.es.da 55.7 0.712 @@ -879,11 +920,13 @@ es de newstest2011.es.de 22.4 0.520 es de newstest2012.es.de 23.2 0.523 es de newstest2013.es.de 24.7 0.539 es de Tatoeba.es.de 50.0 0.683 +es dhv JW300.es.dhv 0.0 0.000 es dje bible-uedin.es.dje 4.7 0.227 es djk JW300.es.djk 4.7 0.182 es ecs JW300.es.ecs 17.2 0.426 es ee JW300.es.ee 25.6 0.470 es efi JW300.es.efi 24.6 0.452 +es el Tatoeba.es.el 48.0 0.658 es eo Tatoeba.es.eo 44.7 0.657 es es Tatoeba.es.es 51.7 0.688 es et JW300.es.et 20.7 0.466 @@ -930,6 +973,7 @@ es ilo JW300.es.ilo 31.0 0.544 es ise JW300.es.ise 3.5 0.181 es is JW300.es.is 22.3 0.418 es iso JW300.es.iso 22.4 0.396 +es it Tatoeba.es.it 56.1 0.752 es jap bible-uedin.es.jap 3.9 0.121 es ja Tatoeba.es.ja 1.1 0.352 es jp GlobalVoices.es.jp 0.1 0.022 @@ -1021,6 +1065,7 @@ es plt bible-uedin.es.plt 3.1 0.268 es pon JW300.es.pon 21.6 0.448 es prl JW300.es.prl 92.2 0.950 es pso JW300.es.pso 3.6 0.151 +es pt_br+pt_BR+pt_PT+pt Tatoeba.es.pt 54.7 0.730 es que JW300.es.que 11.8 0.377 es qu JW300.es.qu 12.1 0.339 es quy JW300.es.quy 13.6 0.399 @@ -1126,6 +1171,7 @@ fi am JW300.fi.am 14.8 0.295 fi ar JW300.fi.ar 8.0 0.225 fi ase JW300.fi.ase 1.1 0.111 fi as JW300.fi.as 0.9 0.095 +fi ay JW300.fi.ay 5.6 0.249 fi az_IR+az JW300.fi.az 16.8 0.385 fi ba JW300.fi.ba 0.1 0.034 fi bas JW300.fi.bas 4.6 0.182 @@ -1153,6 +1199,7 @@ fi cv JW300.fi.cv 0.2 0.062 fi cy JW300.fi.cy 1.8 0.115 fi dhv JW300.fi.dhv 5.4 0.241 fi dje bible-uedin.fi.dje 4.7 0.219 +fi djk JW300.fi.djk 0.6 0.152 fi ee JW300.fi.ee 28.0 0.500 fi efi JW300.fi.efi 26.6 0.482 fi el JW300.fi.el 27.1 0.490 @@ -1173,6 +1220,7 @@ fi fa JW300.fi.fa 15.1 0.346 fi fi Tatoeba.fi.fi 23.3 0.554 fi fj JW300.fi.fj 26.6 0.500 fi fon JW300.fi.fon 5.6 0.167 +fi fr Tatoeba.fi.fr 49.9 0.657 fi fse JW300.fi.fse 81.9 0.882 fi gaa JW300.fi.gaa 30.5 0.514 fi gd bible-uedin.fi.gd 0.0 0.110 @@ -1183,10 +1231,13 @@ fi gu JW300.fi.gu 2.7 0.133 fi guw JW300.fi.guw 32.4 0.527 fi gv bible-uedin.fi.gv 0.0 0.045 fi gym JW300.fi.gym 3.3 0.212 +fi ha JW300.fi.ha 24.2 0.461 fi he JW300.fi.he 20.7 0.424 fi hi JW300.fi.hi 15.4 0.302 +fi hil JW300.fi.hil 38.7 0.610 fi ho JW300.fi.ho 25.7 0.496 fi hr JW300.fi.hr 23.5 0.476 +fi ht JW300.fi.ht 27.1 0.453 fi hu Tatoeba.fi.hu 50.4 0.705 fi hy JW300.fi.hy 13.1 0.330 fi ibg JW300.fi.ibg 4.4 0.211 @@ -1206,14 +1257,25 @@ fi kbp JW300.fi.kbp 3.9 0.167 fi kek JW300.fi.kek 0.0 0.003 fi kg JW300.fi.kg 29.5 0.535 fi ki JW300.fi.ki 3.5 0.195 +fi kj JW300.fi.kj 4.2 0.241 fi kl JW300.fi.kl 12.2 0.412 +fi kmb JW300.fi.kmb 4.3 0.205 +fi km JW300.fi.km 0.7 0.073 +fi kn JW300.fi.kn 1.9 0.117 fi ko JW300.fi.ko 16.6 0.316 fi koo JW300.fi.koo 2.5 0.199 fi kqn JW300.fi.kqn 22.3 0.476 +fi kri JW300.fi.kri 6.6 0.202 +fi kvk JW300.fi.kvk 1.4 0.084 fi kwn JW300.fi.kwn 2.9 0.197 +fi kwy JW300.fi.kwy 19.0 0.406 +fi ky JW300.fi.ky 14.7 0.346 +fi la bible-uedin.fi.la 0.8 0.154 fil en GlobalVoices.fil.en 0.3 0.099 fil es GlobalVoices.fil.es 0.2 0.085 fil fr GlobalVoices.fil.fr 0.2 0.093 +fi lg JW300.fi.lg 21.7 0.473 +fi ln JW300.fi.ln 29.2 0.536 fi lt JW300.fi.lt 18.6 0.439 fi lua JW300.fi.lua 26.6 0.505 fi lue JW300.fi.lue 22.4 0.497 @@ -1224,8 +1286,17 @@ fi lus JW300.fi.lus 26.3 0.480 fi lv JW300.fi.lv 20.6 0.439 fi mam JW300.fi.mam 3.3 0.202 fi mco JW300.fi.mco 2.7 0.184 +fi mfe JW300.fi.mfe 22.6 0.426 +fi mfs JW300.fi.mfs 3.1 0.172 +fi mg JW300.fi.mg 21.7 0.468 fi mgr JW300.fi.mgr 2.6 0.197 +fi mh JW300.fi.mh 20.8 0.404 fi mi bible-uedin.fi.mi 5.6 0.238 +fi mk JW300.fi.mk 28.9 0.501 +fi ml JW300.fi.ml 3.5 0.184 +fi mn JW300.fi.mn 10.2 0.254 +fi mos JW300.fi.mos 21.4 0.366 +fi mr JW300.fi.mr 2.6 0.131 fi mt JW300.fi.mt 29.9 0.490 fi my JW300.fi.my 1.6 0.145 fi nb_NO+nb+nn_NO+nn+nog+no_nb+no JW300.fi.no 34.2 0.545 @@ -1243,33 +1314,77 @@ fi nyk JW300.fi.nyk 2.2 0.201 fi nyn JW300.fi.nyn 2.8 0.206 fi nzi JW300.fi.nzi 3.5 0.183 fi oke JW300.fi.oke 5.3 0.213 +fi om JW300.fi.om 14.1 0.404 +fi os JW300.fi.os 15.8 0.361 +fi pag JW300.fi.pag 28.0 0.510 +fi pa JW300.fi.pa 14.3 0.289 +fi pap JW300.fi.pap 27.3 0.478 +fi pck bible-uedin.fi.pck 4.1 0.202 fi pcm JW300.fi.pcm 4.3 0.179 +fi pes bible-uedin.fi.pes 3.1 0.189 +fi pis JW300.fi.pis 27.5 0.493 +fi plt bible-uedin.fi.plt 2.6 0.245 fi pon JW300.fi.pon 23.7 0.475 +fi prl JW300.fi.prl 3.9 0.171 +fi pso JW300.fi.pso 3.1 0.153 fi pt_br+pt_BR+pt_PT+pt JW300.fi.pt 30.5 0.531 +fi que JW300.fi.que 1.8 0.160 +fi qu JW300.fi.qu 8.1 0.261 +fi quy JW300.fi.quy 1.7 0.176 +fi quz JW300.fi.quz 8.7 0.269 +fi qvi JW300.fi.qvi 1.7 0.179 +fi rar JW300.fi.rar 5.6 0.231 +fi rnd JW300.fi.rnd 18.3 0.404 fi ro JW300.fi.ro 27.0 0.490 fi rsl JW300.fi.rsl 3.2 0.151 fi run JW300.fi.run 23.2 0.498 +fi ru Tatoeba.fi.ru 45.9 0.662 fi rw JW300.fi.rw 25.3 0.509 +fi seh JW300.fi.seh 2.9 0.194 +fi sg JW300.fi.sg 29.3 0.480 fi sid JW300.fi.sid 2.2 0.183 fi si JW300.fi.si 6.9 0.196 fi sk JW300.fi.sk 28.1 0.501 fi sl JW300.fi.sl 24.1 0.481 fi sm JW300.fi.sm 24.0 0.443 fi sn JW300.fi.sn 25.3 0.547 +fi so bible-uedin.fi.so 2.1 0.199 +fi sop JW300.fi.sop 2.4 0.212 fi sq JW300.fi.sq 32.0 0.535 fi sr_ME+sr+srp bible-uedin.fi.srp 16.4 0.421 fi srn JW300.fi.srn 29.2 0.491 +fi ss JW300.fi.ss 10.4 0.334 fi ssp JW300.fi.ssp 3.9 0.176 +fi st JW300.fi.st 37.1 0.570 fi sv fiskmo_testset.fi.sv 27.4 0.601 fi sv Tatoeba.fi.sv 55.0 0.706 +fi swc JW300.fi.swc 27.5 0.515 fi sw JW300.fi.sw 29.9 0.548 +fi ta_LK+ta JW300.fi.ta 3.6 0.150 +fi tdt JW300.fi.tdt 14.7 0.328 +fi te JW300.fi.te 1.7 0.134 fi tg_TJ+tg JW300.fi.tg 9.7 0.260 +fi th JW300.fi.th 16.0 0.319 fi ti JW300.fi.ti 16.6 0.303 +fi tiv JW300.fi.tiv 23.6 0.425 +fi tk JW300.fi.tk 9.1 0.257 +fi tll JW300.fi.tll 23.6 0.478 +fi tl_PH+tl JW300.fi.tl 36.5 0.595 fi tn JW300.fi.tn 34.5 0.555 fi tog JW300.fi.tog 3.2 0.216 +fi toi JW300.fi.toi 22.0 0.509 +fi to JW300.fi.to 38.3 0.541 fi top JW300.fi.top 2.0 0.175 +fi tpi JW300.fi.tpi 30.5 0.504 +fi tr Tatoeba.fi.tr 30.5 0.601 fi tsc JW300.fi.tsc 4.4 0.234 +fi ts JW300.fi.ts 33.6 0.563 +fi tt JW300.fi.tt 14.0 0.350 +fi tum JW300.fi.tum 19.7 0.445 fi tvl JW300.fi.tvl 33.6 0.517 +fi tw JW300.fi.tw 29.2 0.504 +fi ty JW300.fi.ty 39.7 0.565 +fi tzo JW300.fi.tzo 8.6 0.290 fi uk JW300.fi.uk 23.3 0.445 fi umb JW300.fi.umb 19.8 0.426 fi urh JW300.fi.urh 4.5 0.192 @@ -1279,11 +1394,16 @@ fi vi_VN+vi JW300.fi.vi 38.8 0.520 fi vmw JW300.fi.vmw 2.2 0.183 fi vsl JW300.fi.vsl 4.5 0.178 fi wal JW300.fi.wal 10.1 0.343 +fi war JW300.fi.war 35.1 0.565 +fi wes JW300.fi.wes 5.2 0.198 +fi wls JW300.fi.wls 24.7 0.466 fi xh JW300.fi.xh 25.3 0.554 fi yao JW300.fi.yao 2.9 0.193 fi yap JW300.fi.yap 25.4 0.445 +fi yo JW300.fi.yo 25.8 0.427 fi yua JW300.fi.yua 12.5 0.341 fi zai JW300.fi.zai 5.9 0.229 +fi zlm JW300.fi.zlm 4.3 0.224 fi zne JW300.fi.zne 22.7 0.464 fi zul+zu JW300.fi.zu 28.1 0.584 fj de JW300.fj.de 18.8 0.377 @@ -1303,6 +1423,7 @@ fr ada JW300.fr.ada 7.6 0.213 fr aed JW300.fr.aed 3.4 0.163 fr af JW300.fr.af 36.0 0.546 fr am GlobalVoices.fr.am 0.5 0.108 +fr ar Tatoeba.fr.ar 15.6 0.469 fr ase JW300.fr.ase 38.5 0.545 fr as JW300.fr.as 1.5 0.127 fr ay JW300.fr.ay 6.4 0.256 @@ -1327,6 +1448,7 @@ fr cat JW300.fr.cat 3.2 0.168 fr ceb JW300.fr.ceb 32.8 0.543 fr chk JW300.fr.chk 19.8 0.397 fr cjk JW300.fr.cjk 3.1 0.214 +fr cnh JW300.fr.cnh 5.0 0.221 fr crp bible-uedin.fr.crp 4.2 0.201 fr crs JW300.fr.crs 31.6 0.492 fr csg JW300.fr.csg 4.0 0.177 @@ -1391,6 +1513,7 @@ fr hmn JW300.fr.hmn 5.1 0.212 fr ho JW300.fr.ho 25.4 0.480 fr hr JW300.fr.hr 20.7 0.442 fr ht JW300.fr.ht 29.2 0.461 +fr hu Tatoeba.fr.hu 41.3 0.629 fr hy JW300.fr.hy 7.8 0.225 fr ia Tatoeba.fr.ia 0.2 0.092 fr iba JW300.fr.iba 6.1 0.245 @@ -1502,6 +1625,7 @@ fr rnd JW300.fr.rnd 21.8 0.431 fr ro Tatoeba.fr.ro 42.1 0.640 fr rsl JW300.fr.rsl 2.9 0.147 fr run JW300.fr.run 23.8 0.482 +fr ru Tatoeba.fr.ru 37.9 0.585 fr rw JW300.fr.rw 25.5 0.483 fr seh JW300.fr.seh 3.2 0.205 fr sg JW300.fr.sg 29.7 0.473 @@ -1519,6 +1643,7 @@ fr srn JW300.fr.srn 27.4 0.459 fr ss JW300.fr.ss 14.1 0.383 fr ssp JW300.fr.ssp 4.1 0.178 fr st JW300.fr.st 34.6 0.540 +fr sv Tatoeba.fr.sv 60.1 0.744 fr swc JW300.fr.swc 28.2 0.499 fr sw GlobalVoices.fr.sw 12.0 0.404 fr ta_LK+ta JW300.fr.ta 2.5 0.116 @@ -1633,8 +1758,10 @@ ha de JW300.ha.de 19.1 0.377 ha en JW300.ha.en 35.0 0.506 ha en Tatoeba.ha.en 39.0 0.497 ha es JW300.ha.es 21.8 0.394 +ha fi JW300.ha.fi 21.9 0.435 ha fr JW300.ha.fr 24.3 0.415 ha sv JW300.ha.sv 25.8 0.438 +he de Tatoeba.he.de 45.5 0.647 he fi JW300.he.fi 23.3 0.492 he sv JW300.he.sv 28.9 0.493 hi de JW300.hi.de 9.5 0.266 @@ -1645,6 +1772,7 @@ hi es GlobalVoices.hi.es 4.2 0.241 hi fr GlobalVoices.hi.fr 1.4 0.163 hil de JW300.hil.de 26.4 0.479 hil en JW300.hil.en 49.2 0.638 +hil fi JW300.hil.fi 29.9 0.547 hil fr JW300.hil.fr 31.2 0.494 hil sv JW300.hil.sv 0.7 0.125 hi sv JW300.hi.sv 11.6 0.255 @@ -1672,6 +1800,7 @@ hu de Tatoeba.hu.de 44.1 0.637 hu en Tatoeba.hu.en 52.9 0.683 hu fi Tatoeba.hu.fi 48.2 0.700 hu fr Tatoeba.hu.fr 50.3 0.660 +hu sv Tatoeba.hu.sv 52.6 0.686 hy de JW300.hy.de 10.1 0.280 hy en Tatoeba.hy.en 29.5 0.466 hy es JW300.hy.es 9.7 0.263 @@ -1732,7 +1861,9 @@ it de Tatoeba.it.de 49.4 0.678 it en newssyscomb2009.it.en 35.3 0.600 it en newstest2009.it.en 34.0 0.594 it en Tatoeba.it.en 70.9 0.808 +it es Tatoeba.it.es 61.2 0.761 it fr Tatoeba.it.fr 67.9 0.792 +it sv Tatoeba.it.sv 56.0 0.707 ja de Tatoeba.ja.de 30.1 0.518 ja en Tatoeba.ja.en 41.7 0.589 ja es Tatoeba.ja.es 34.6 0.553 @@ -1796,6 +1927,7 @@ ki sv JW300.ki.sv 2.3 0.149 kj de JW300.kj.de 1.4 0.141 kj en JW300.kj.en 30.3 0.477 kj es JW300.kj.es 2.3 0.148 +kj fi JW300.kj.fi 1.7 0.173 kj fr JW300.kj.fr 2.4 0.150 kj sv JW300.kj.sv 2.4 0.161 kk en newsdev2019-enkk.kk.en 0.0 0.036 @@ -1810,11 +1942,13 @@ kl sv JW300.kl.sv 19.4 0.372 kmb de JW300.kmb.de 1.5 0.129 kmb en JW300.kmb.en 5.1 0.189 kmb es JW300.kmb.es 2.6 0.140 +kmb fi JW300.kmb.fi 1.3 0.141 kmb fr JW300.kmb.fr 3.4 0.154 kmb sv JW300.kmb.sv 2.5 0.152 km de JW300.km.de 0.6 0.186 km en Tatoeba.km.en 0.1 0.115 km es Tatoeba.km.es 0.1 0.118 +km fi JW300.km.fi 0.6 0.149 km fr GlobalVoices.km.fr 0.1 0.113 km sv JW300.km.sv 0.6 0.130 kn de JW300.kn.de 1.9 0.181 @@ -1881,18 +2015,20 @@ ky sv JW300.ky.sv 15.9 0.323 la de Tatoeba.la.de 0.2 0.074 la en Tatoeba.la.en 2.1 0.144 la es Tatoeba.la.es 0.2 0.052 +la fi bible-uedin.la.fi 1.6 0.165 la fr Tatoeba.la.fr 0.2 0.090 la sv bible-uedin.la.sv 2.5 0.162 lg de JW300.lg.de 17.9 0.367 lg en JW300.lg.en 32.6 0.480 lg en Tatoeba.lg.en 5.4 0.243 lg es JW300.lg.es 22.1 0.393 +lg fi JW300.lg.fi 21.8 0.424 lg fr JW300.lg.fr 23.7 0.406 lg sv JW300.lg.sv 24.5 0.423 ln de JW300.ln.de 23.3 0.428 ln en JW300.ln.en 35.9 0.516 ln es JW300.ln.es 26.5 0.444 -ln fi JW300.ln.fi 0.7 0.128 +ln fi JW300.ln.fi 0.6 0.133 ln fr JW300.ln.fr 28.4 0.456 ln sv JW300.ln.sv 1.0 0.136 lo en JW300.lo.en 3.7 0.150 @@ -1978,12 +2114,14 @@ mfe sv JW300.mfe.sv 2.2 0.169 mfs de JW300.mfs.de 2.2 0.147 mfs en JW300.mfs.en 3.8 0.172 mfs es JW300.mfs.es 88.9 0.910 +mfs fi JW300.mfs.fi 2.3 0.162 mfs fr JW300.mfs.fr 3.7 0.166 mfs sv JW300.mfs.sv 3.4 0.168 mg de GlobalVoices.mg.de 8.8 0.337 mg en GlobalVoices.mg.en 27.6 0.522 mg en Tatoeba.mg.en 50.2 0.607 mg es GlobalVoices.mg.es 23.1 0.480 +mg fi JW300.mg.fi 18.8 0.399 mg fr GlobalVoices.mg.fr 19.5 0.451 mgr de JW300.mgr.de 1.9 0.144 mgr en JW300.mgr.en 5.2 0.184 @@ -1994,6 +2132,7 @@ mg sv GlobalVoices.mg.sv 4.9 0.275 mh de JW300.mh.de 19.2 0.380 mh en JW300.mh.en 36.5 0.505 mh es JW300.mh.es 23.6 0.407 +mh fi JW300.mh.fi 23.3 0.442 mh fr JW300.mh.fr 0.3 0.063 mh sv JW300.mh.sv 0.2 0.031 mi de bible-uedin.mi.de 3.7 0.221 @@ -2005,11 +2144,13 @@ mi sv bible-uedin.mi.sv 4.0 0.207 mk de GlobalVoices.mk.de 16.7 0.450 mk en Tatoeba.mk.en 59.8 0.720 mk es GlobalVoices.mk.es 27.6 0.541 +mk fi JW300.mk.fi 25.9 0.498 mk fr GlobalVoices.mk.fr 22.3 0.492 mk sv GlobalVoices.mk.sv 18.1 0.446 ml de JW300.ml.de 5.2 0.208 ml en Tatoeba.ml.en 43.0 0.601 ml es JW300.ml.es 6.6 0.212 +ml fi JW300.ml.fi 4.5 0.204 ml fr JW300.ml.fr 11.6 0.245 ml sv JW300.ml.sv 5.9 0.194 mn de JW300.mn.de 7.9 0.239 @@ -2022,6 +2163,7 @@ mn sv JW300.mn.sv 9.8 0.249 mos de JW300.mos.de 15.5 0.322 mos en JW300.mos.en 26.1 0.408 mos es JW300.mos.es 16.4 0.329 +mos fi JW300.mos.fi 18.0 0.373 mos fr JW300.mos.fr 0.5 0.160 mos sv JW300.mos.sv 20.0 0.372 mr de JW300.mr.de 1.3 0.179 @@ -2095,6 +2237,7 @@ niu fr JW300.niu.fr 28.1 0.452 niu sv JW300.niu.sv 29.2 0.478 nl es Tatoeba.nl.es 51.6 0.698 nl fi JW300.nl.fi 28.1 0.553 +nl fr Tatoeba.nl.fr 51.3 0.674 nl sv GlobalVoices.nl.sv 25.0 0.518 nr de JW300.nr.de 1.6 0.145 nr en JW300.nr.en 3.8 0.190 @@ -2151,6 +2294,7 @@ oke sv JW300.oke.sv 4.7 0.177 om de JW300.om.de 10.8 0.273 om en JW300.om.en 27.3 0.448 om es JW300.om.es 13.4 0.296 +om fi JW300.om.fi 14.9 0.349 om fr JW300.om.fr 17.0 0.338 om sv JW300.om.sv 0.6 0.142 or en Tatoeba.or.en 1.2 0.093 @@ -2165,22 +2309,26 @@ pa de JW300.pa.de 10.8 0.267 pa en JW300.pa.en 20.6 0.320 pa en Tatoeba.pa.en 29.3 0.464 pa es JW300.pa.es 10.4 0.251 +pa fi JW300.pa.fi 12.6 0.297 pa fr JW300.pa.fr 14.3 0.278 pag de JW300.pag.de 22.8 0.435 pag en JW300.pag.en 42.4 0.580 pag es JW300.pag.es 27.9 0.459 +pag fi JW300.pag.fi 26.7 0.496 pag fr JW300.pag.fr 0.2 0.043 pag sv JW300.pag.sv 29.8 0.492 pap de JW300.pap.de 25.0 0.466 pap en JW300.pap.en 47.3 0.634 pap en Tatoeba.pap.en 63.2 0.684 pap es JW300.pap.es 32.3 0.518 +pap fi JW300.pap.fi 27.7 0.520 pap fr JW300.pap.fr 31.0 0.498 pap sv JW300.pap.sv 1.7 0.142 pa sv JW300.pa.sv 11.8 0.253 pck de bible-uedin.pck.de 3.4 0.207 pck en bible-uedin.pck.en 4.0 0.214 pck es bible-uedin.pck.es 3.5 0.201 +pck fi bible-uedin.pck.fi 2.5 0.191 pck fr bible-uedin.pck.fr 3.2 0.189 pck sv bible-uedin.pck.sv 3.2 0.181 pcm de JW300.pcm.de 2.5 0.161 @@ -2192,21 +2340,24 @@ pcm sv JW300.pcm.sv 4.1 0.168 pes de Tatoeba.pes.de 0.6 0.159 pes en Tatoeba.pes.en 0.7 0.147 pes es bible-uedin.pes.es 3.7 0.216 +pes fi bible-uedin.pes.fi 2.7 0.201 pes fr bible-uedin.pes.fr 0.6 0.126 pes sv bible-uedin.pes.sv 3.8 0.197 pis de JW300.pis.de 19.7 0.391 pis en JW300.pis.en 33.3 0.493 pis es JW300.pis.es 24.1 0.421 +pis fi JW300.pis.fi 21.8 0.439 pis fr JW300.pis.fr 24.9 0.421 pis sv JW300.pis.sv 25.9 0.442 pl de Tatoeba.pl.de 47.8 0.665 pl en Tatoeba.pl.en 54.9 0.701 pl es Tatoeba.pl.es 46.9 0.654 pl fr Tatoeba.pl.fr 49.0 0.659 -pl sv Tatoeba.pl.sv 58.3 0.712 +pl sv Tatoeba.pl.sv 58.9 0.717 plt de bible-uedin.plt.de 2.7 0.193 plt en bible-uedin.plt.en 3.6 0.203 plt es bible-uedin.plt.es 2.9 0.192 +plt fi bible-uedin.plt.fi 2.0 0.176 plt fr bible-uedin.plt.fr 3.0 0.189 plt sv bible-uedin.plt.sv 2.9 0.171 pon de JW300.pon.de 18.9 0.375 @@ -2218,23 +2369,27 @@ pon sv JW300.pon.sv 26.4 0.436 prl de JW300.prl.de 3.1 0.169 prl en JW300.prl.en 4.3 0.175 prl es JW300.prl.es 93.3 0.955 +prl fi JW300.prl.fi 3.2 0.173 prl fr JW300.prl.fr 5.0 0.179 prl sv JW300.prl.sv 4.6 0.169 ps en Tatoeba.ps.en 73.6 0.827 pso de JW300.pso.de 3.4 0.177 pso en JW300.pso.en 4.7 0.189 pso es JW300.pso.es 4.3 0.172 +pso fi JW300.pso.fi 2.5 0.169 pso fr JW300.pso.fr 4.6 0.179 pso sv JW300.pso.sv 5.1 0.180 qu de JW300.qu.de 5.7 0.200 que de JW300.que.de 2.4 0.147 que en JW300.que.en 8.0 0.216 que es JW300.que.es 14.7 0.310 +que fi JW300.que.fi 1.7 0.153 que fr JW300.que.fr 0.6 0.126 qu en JW300.qu.en 8.6 0.218 qu en Tatoeba.qu.en 22.7 0.348 qu es JW300.qu.es 12.1 0.274 que sv JW300.que.sv 0.0 0.020 +qu fi JW300.qu.fi 8.0 0.234 qu fr JW300.qu.fr 6.9 0.208 qug en JW300.qug.en 3.6 0.155 qu sv JW300.qu.sv 0.4 0.086 @@ -2246,20 +2401,24 @@ quy sv JW300.quy.sv 2.4 0.159 quz de JW300.quz.de 6.2 0.204 quz en JW300.quz.en 8.9 0.222 quz es JW300.quz.es 13.4 0.285 +quz fi JW300.quz.fi 8.1 0.231 quz fr JW300.quz.fr 0.5 0.116 quz sv JW300.quz.sv 0.2 0.102 qvi de JW300.qvi.de 1.6 0.130 qvi en JW300.qvi.en 3.1 0.145 qvi es JW300.qvi.es 3.4 0.156 +qvi fi JW300.qvi.fi 1.5 0.129 qvi fr JW300.qvi.fr 2.7 0.143 qvi sv JW300.qvi.sv 2.4 0.134 rar de JW300.rar.de 1.2 0.129 rar en JW300.rar.en 5.0 0.212 rar es JW300.rar.es 2.2 0.144 +rar fi JW300.rar.fi 1.4 0.145 rar fr JW300.rar.fr 2.9 0.150 rar sv JW300.rar.sv 2.0 0.147 rnd en JW300.rnd.en 37.8 0.531 rnd es JW300.rnd.es 15.6 0.319 +rnd fi JW300.rnd.fi 16.4 0.365 rnd fr JW300.rnd.fr 22.1 0.392 rnd sv JW300.rnd.sv 21.2 0.387 ro fi JW300.ro.fi 25.2 0.521 @@ -2286,7 +2445,7 @@ ru es Tatoeba.ru.es 49.4 0.675 ru fi Tatoeba.ru.fi 40.9 0.649 ru fr newstest2012.ru.fr 18.3 0.497 ru fr newstest2013.ru.fr 21.6 0.516 -ru fr Tatoeba.ru.fr 51.1 0.666 +ru fr Tatoeba.ru.fr 51.5 0.670 run en JW300.run.en 42.7 0.583 run es JW300.run.es 26.9 0.452 run fr JW300.run.fr 0.6 0.102 @@ -2303,6 +2462,7 @@ seh fr JW300.seh.fr 3.7 0.167 seh sv JW300.seh.sv 3.0 0.166 sg en JW300.sg.en 32.0 0.477 sg es JW300.sg.es 21.3 0.385 +sg fi JW300.sg.fi 22.7 0.438 sg fr JW300.sg.fr 24.9 0.420 sg sv JW300.sg.sv 25.3 0.428 sid en JW300.sid.en 4.2 0.176 @@ -2370,6 +2530,7 @@ st fi JW300.st.fi 28.8 0.520 st fr JW300.st.fr 30.7 0.490 st sv JW300.st.sv 33.5 0.523 sv ab JW300.sv.ab 1.3 0.119 +sv ach JW300.sv.ach 4.8 0.202 sv ada JW300.sv.ada 7.2 0.208 sv aed JW300.sv.aed 3.4 0.161 sv af JW300.sv.af 44.4 0.623 @@ -2406,11 +2567,13 @@ sv cv JW300.sv.cv 2.2 0.147 sv cy JW300.sv.cy 5.6 0.211 sv dhv JW300.sv.dhv 6.8 0.256 sv dje bible-uedin.sv.dje 5.2 0.229 +sv djk JW300.sv.djk 0.8 0.155 sv ee JW300.sv.ee 29.7 0.508 sv efi JW300.sv.efi 29.4 0.502 sv el GlobalVoices.sv.el 20.8 0.456 sv en Tatoeba.sv.en 65.3 0.769 sv eo Tatoeba.sv.eo 24.0 0.455 +sv es Tatoeba.sv.es 52.1 0.683 sv et JW300.sv.et 23.5 0.497 sv eu bible-uedin.sv.eu 0.2 0.097 sv fa JW300.sv.fa 14.7 0.331 @@ -2418,6 +2581,7 @@ sv fi fiskmo_testset.sv.fi 26.5 0.614 sv fi Tatoeba.sv.fi 44.9 0.669 sv fj JW300.sv.fj 27.8 0.504 sv fon JW300.sv.fon 4.9 0.158 +sv fr Tatoeba.sv.fr 59.7 0.731 sv fse JW300.sv.fse 3.9 0.187 sv gaa JW300.sv.gaa 31.3 0.522 sv gd bible-uedin.sv.gd 0.0 0.023 @@ -2435,6 +2599,7 @@ sv hil JW300.sv.hil 38.2 0.610 sv ho JW300.sv.ho 26.7 0.503 sv hr JW300.sv.hr 25.7 0.498 sv ht JW300.sv.ht 28.0 0.457 +sv hu Tatoeba.sv.hu 44.6 0.660 sv hy JW300.sv.hy 6.7 0.192 sv ibg JW300.sv.ibg 4.7 0.203 sv id JW300.sv.id 35.6 0.581 @@ -2530,6 +2695,7 @@ sv rnd JW300.sv.rnd 20.3 0.433 sv ro JW300.sv.ro 29.5 0.510 sv rsl JW300.sv.rsl 3.7 0.150 sv run JW300.sv.run 24.4 0.502 +sv ru Tatoeba.sv.ru 46.6 0.662 sv rw JW300.sv.rw 26.7 0.514 sv seh JW300.sv.seh 3.2 0.192 sv sg JW300.sv.sg 30.0 0.487 @@ -2618,6 +2784,7 @@ te fr JW300.te.fr 5.9 0.198 te sv JW300.te.sv 2.9 0.177 th en Tatoeba.th.en 47.2 0.636 th es JW300.th.es 15.0 0.302 +th fi JW300.th.fi 11.4 0.280 th fr JW300.th.fr 20.4 0.363 th sv JW300.th.sv 17.8 0.321 ti en JW300.ti.en 30.4 0.461 @@ -2645,7 +2812,7 @@ tn fr JW300.tn.fr 29.0 0.474 tn sv JW300.tn.sv 32.0 0.508 to en JW300.to.en 49.3 0.627 to es JW300.to.es 26.6 0.447 -to fi JW300.to.fi 25.0 0.487 +to fi JW300.to.fi 0.2 0.084 to fr JW300.to.fr 27.9 0.456 tog en JW300.tog.en 4.4 0.171 tog es JW300.tog.es 2.6 0.144 @@ -2668,6 +2835,7 @@ top sv JW300.top.sv 2.0 0.144 to sv JW300.to.sv 30.7 0.493 tpi en JW300.tpi.en 29.1 0.448 tpi es JW300.tpi.es 19.5 0.369 +tpi fr JW300.tpi.fr 0.3 0.079 tpi sv JW300.tpi.sv 21.6 0.396 tr en newsdev2016-entr.tr.en 27.6 0.548 tr en newstest2016-entr.tr.en 25.2 0.532 @@ -2701,7 +2869,7 @@ tvl fi JW300.tvl.fi 22.0 0.439 tvl fr JW300.tvl.fr 24.0 0.410 tvl sv JW300.tvl.sv 24.7 0.427 tw es JW300.tw.es 25.9 0.441 -tw fi JW300.tw.fi 24.6 0.477 +tw fi JW300.tw.fi 25.6 0.488 tw fr JW300.tw.fr 26.7 0.442 tw sv JW300.tw.sv 29.0 0.471 ty es JW300.ty.es 27.3 0.457 @@ -2733,8 +2901,8 @@ urh fr JW300.urh.fr 4.0 0.169 urh sv JW300.urh.sv 3.6 0.163 ve en JW300.ve.en 41.3 0.566 ve es JW300.ve.es 23.1 0.413 -ve fi JW300.ve.fi 23.6 0.465 -ve sv JW300.ve.sv 27.1 0.461 +ve fi JW300.ve.fi 0.1 0.053 +ve fr JW300.ve.fr 0.1 0.034 vmw en JW300.vmw.en 4.3 0.185 vmw es JW300.vmw.es 2.2 0.144 vmw fi JW300.vmw.fi 1.4 0.154 @@ -2747,10 +2915,11 @@ vsl fr JW300.vsl.fr 5.0 0.187 vsl sv JW300.vsl.sv 4.7 0.176 wal en JW300.wal.en 22.5 0.386 wal es JW300.wal.es 8.9 0.243 -wal fi JW300.wal.fi 10.1 0.283 -wal sv JW300.wal.sv 10.9 0.267 +wal fi JW300.wal.fi 4.4 0.227 +wal sv JW300.wal.sv 5.3 0.214 war en Tatoeba.war.en 8.4 0.286 war es JW300.war.es 28.7 0.470 +war fi JW300.war.fi 26.9 0.507 war fr JW300.war.fr 30.2 0.482 war sv JW300.war.sv 31.4 0.505 wes en JW300.wes.en 6.3 0.217 @@ -2760,6 +2929,7 @@ wes fr JW300.wes.fr 3.8 0.167 wes sv JW300.wes.sv 4.2 0.173 wls en JW300.wls.en 31.8 0.471 wls es JW300.wls.es 18.9 0.354 +wls fi JW300.wls.fi 16.9 0.374 wls fr JW300.wls.fr 22.6 0.389 wls sv JW300.wls.sv 23.8 0.408 xh en JW300.xh.en 45.8 0.610