marian-nmt/marian
1
1
Fork 0
mirror of https://github.com/marian-nmt/marian.git synced 2024-09-11 06:15:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add comment about 'dim-vocabs'

Browse Source
This commit is contained in:
Roman Grundkiewicz 2018-10-19 13:28:06 +01:00
parent 0bdff22deb
commit acf66bc969
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/data/corpus_base.cpp
View File

@ -79,6 +79,9 @@ CorpusBase::CorpusBase(Ptr<Options> options, bool translate)
for(size_t i = 0; i < paths_.size(); ++i) {
Ptr<Vocab> vocab = New<Vocab>(options_, i);
int vocSize = vocab->loadOrCreate("", paths_[i], maxVocabs[i]);
// TODO: this is not nice as it modifies the option object and needs to expose the changes
// outside the corpus as models need to know about the vocabulary size; extract the vocab
// creation functionality from the class.
options_->get()["dim-vocabs"][i] = vocSize;
options_->get()["vocabs"].push_back(paths_[i] + ".yml");
@ -92,6 +95,9 @@ CorpusBase::CorpusBase(Ptr<Options> options, bool translate)
for(size_t i = 0; i < vocabPaths.size(); ++i) {
Ptr<Vocab> vocab = New<Vocab>(options_, i);
int vocSize = vocab->loadOrCreate(vocabPaths[i], paths_[i], maxVocabs[i]);
// TODO: this is not nice as it modifies the option object and needs to expose the changes
// outside the corpus as models need to know about the vocabulary size; extract the vocab
// creation functionality from the class.
options_->get()["dim-vocabs"][i] = vocSize;
vocabs_.emplace_back(vocab);