mirror of
https://github.com/browsermt/bergamot-translator.git
synced 2024-10-26 05:43:59 +03:00
Updated wasm test page to pass vocabulary files as bytes
This commit is contained in:
parent
9f78985e45
commit
5025285e5c
@ -113,10 +113,7 @@ shortlist:
|
|||||||
`;
|
`;
|
||||||
*/
|
*/
|
||||||
|
|
||||||
const modelConfigWithoutModelAndShortList = `vocabs:
|
const modelConfig = `beam-size: 1
|
||||||
- /${languagePair}/vocab.${vocabLanguagePair}.spm
|
|
||||||
- /${languagePair}/vocab.${vocabLanguagePair}.spm
|
|
||||||
beam-size: 1
|
|
||||||
normalize: 1.0
|
normalize: 1.0
|
||||||
word-penalty: 0
|
word-penalty: 0
|
||||||
max-length-break: 128
|
max-length-break: 128
|
||||||
@ -136,9 +133,15 @@ gemm-precision: int8shift
|
|||||||
// gemm-precision: int8shiftAlphaAll
|
// gemm-precision: int8shiftAlphaAll
|
||||||
|
|
||||||
const modelFile = `models/${languagePair}/model.${languagePair}.intgemm.alphas.bin`;
|
const modelFile = `models/${languagePair}/model.${languagePair}.intgemm.alphas.bin`;
|
||||||
console.debug("modelFile: ", modelFile);
|
|
||||||
const shortlistFile = `models/${languagePair}/lex.50.50.${languagePair}.s2t.bin`;
|
const shortlistFile = `models/${languagePair}/lex.50.50.${languagePair}.s2t.bin`;
|
||||||
|
const vocabFiles = [`models/${languagePair}/vocab.${vocabLanguagePair}.spm`,
|
||||||
|
`models/${languagePair}/vocab.${vocabLanguagePair}.spm`];
|
||||||
|
|
||||||
|
const uniqueVocabFiles = new Set(vocabFiles);
|
||||||
|
console.debug("modelFile: ", modelFile);
|
||||||
console.debug("shortlistFile: ", shortlistFile);
|
console.debug("shortlistFile: ", shortlistFile);
|
||||||
|
console.debug("No. of unique vocabs: ", uniqueVocabFiles.size);
|
||||||
|
uniqueVocabFiles.forEach(item => console.debug("unique vocabFile: ", item));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Download the files as buffers from the given urls
|
// Download the files as buffers from the given urls
|
||||||
@ -146,16 +149,23 @@ gemm-precision: int8shift
|
|||||||
const downloadedBuffers = await Promise.all([downloadAsArrayBuffer(modelFile), downloadAsArrayBuffer(shortlistFile)]);
|
const downloadedBuffers = await Promise.all([downloadAsArrayBuffer(modelFile), downloadAsArrayBuffer(shortlistFile)]);
|
||||||
const modelBuffer = downloadedBuffers[0];
|
const modelBuffer = downloadedBuffers[0];
|
||||||
const shortListBuffer = downloadedBuffers[1];
|
const shortListBuffer = downloadedBuffers[1];
|
||||||
|
|
||||||
|
const downloadedVocabBuffers = [];
|
||||||
|
for (let item of uniqueVocabFiles.values()) {
|
||||||
|
downloadedVocabBuffers.push(await downloadAsArrayBuffer(item));
|
||||||
|
}
|
||||||
log(`${languagePair} file download took ${(Date.now() - start) / 1000} secs`);
|
log(`${languagePair} file download took ${(Date.now() - start) / 1000} secs`);
|
||||||
|
|
||||||
// Construct AlignedMemory objects with downloaded buffers
|
// Construct AlignedMemory objects with downloaded buffers
|
||||||
var alignedModelMemory = constructAlignedMemoryFromBuffer(modelBuffer, 256);
|
var alignedModelMemory = constructAlignedMemoryFromBuffer(modelBuffer, 256);
|
||||||
var alignedShortlistMemory = constructAlignedMemoryFromBuffer(shortListBuffer, 64);
|
var alignedShortlistMemory = constructAlignedMemoryFromBuffer(shortListBuffer, 64);
|
||||||
|
var alignedVocabsMemoryList = new Module.AlignedMemoryList;
|
||||||
|
downloadedVocabBuffers.forEach(item => alignedVocabsMemoryList.push_back(constructAlignedMemoryFromBuffer(item, 64)));
|
||||||
|
|
||||||
// Instantiate the TranslationModel
|
// Instantiate the TranslationModel
|
||||||
if (translationModel) translationModel.delete();
|
if (translationModel) translationModel.delete();
|
||||||
console.debug("Creating TranslationModel with config:", modelConfigWithoutModelAndShortList);
|
console.debug("Creating TranslationModel with config:", modelConfig);
|
||||||
translationModel = new Module.TranslationModel(modelConfigWithoutModelAndShortList, alignedModelMemory, alignedShortlistMemory);
|
translationModel = new Module.TranslationModel(modelConfig, alignedModelMemory, alignedShortlistMemory, alignedVocabsMemoryList);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
log(error);
|
log(error);
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user