browsermt/bergamot-translator
1
1
Fork 0
mirror of https://github.com/browsermt/bergamot-translator.git synced 2024-08-15 16:40:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Updated wasm test page to pass vocabulary files as bytes

Browse Source
This commit is contained in:
Abhishek Aggarwal 2021-05-10 19:09:44 +02:00 committed by abhi-agg
parent 9f78985e45
commit 5025285e5c
1 changed files with 17 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
wasm/test_page/bergamot.html
View File

@ -113,10 +113,7 @@ shortlist:
`;
*/
const modelConfigWithoutModelAndShortList = `vocabs:
- /${languagePair}/vocab.${vocabLanguagePair}.spm
- /${languagePair}/vocab.${vocabLanguagePair}.spm
beam-size: 1
const modelConfig = `beam-size: 1
normalize: 1.0
word-penalty: 0
max-length-break: 128
@ -136,9 +133,15 @@ gemm-precision: int8shift
// gemm-precision: int8shiftAlphaAll
const modelFile = `models/${languagePair}/model.${languagePair}.intgemm.alphas.bin`;
console.debug("modelFile: ", modelFile);
const shortlistFile = `models/${languagePair}/lex.50.50.${languagePair}.s2t.bin`;
const vocabFiles = [`models/${languagePair}/vocab.${vocabLanguagePair}.spm`,
`models/${languagePair}/vocab.${vocabLanguagePair}.spm`];
const uniqueVocabFiles = new Set(vocabFiles);
console.debug("modelFile: ", modelFile);
console.debug("shortlistFile: ", shortlistFile);
console.debug("No. of unique vocabs: ", uniqueVocabFiles.size);
uniqueVocabFiles.forEach(item => console.debug("unique vocabFile: ", item));
try {
// Download the files as buffers from the given urls
@ -146,16 +149,23 @@ gemm-precision: int8shift
const downloadedBuffers = await Promise.all([downloadAsArrayBuffer(modelFile), downloadAsArrayBuffer(shortlistFile)]);
const modelBuffer = downloadedBuffers[0];
const shortListBuffer = downloadedBuffers[1];
const downloadedVocabBuffers = [];
for (let item of uniqueVocabFiles.values()) {
downloadedVocabBuffers.push(await downloadAsArrayBuffer(item));
}
log(`${languagePair} file download took ${(Date.now() - start) / 1000} secs`);
// Construct AlignedMemory objects with downloaded buffers
var alignedModelMemory = constructAlignedMemoryFromBuffer(modelBuffer, 256);
var alignedShortlistMemory = constructAlignedMemoryFromBuffer(shortListBuffer, 64);
var alignedVocabsMemoryList = new Module.AlignedMemoryList;
downloadedVocabBuffers.forEach(item => alignedVocabsMemoryList.push_back(constructAlignedMemoryFromBuffer(item, 64)));
// Instantiate the TranslationModel
if (translationModel) translationModel.delete();
console.debug("Creating TranslationModel with config:", modelConfigWithoutModelAndShortList);
translationModel = new Module.TranslationModel(modelConfigWithoutModelAndShortList, alignedModelMemory, alignedShortlistMemory);
console.debug("Creating TranslationModel with config:", modelConfig);
translationModel = new Module.TranslationModel(modelConfig, alignedModelMemory, alignedShortlistMemory, alignedVocabsMemoryList);
} catch (error) {
log(error);
}