mirror of
https://github.com/marian-nmt/marian.git
synced 2024-09-11 06:15:56 +03:00
made SentencePiece changes compile under Windows
This commit is contained in:
parent
d231d0af71
commit
dd05dde278
5
src/data/default_vocab.cpp
Normal file → Executable file
5
src/data/default_vocab.cpp
Normal file → Executable file
@ -106,7 +106,7 @@ public:
|
||||
ABORT_IF(line.empty(),
|
||||
"DefaultVocabulary file {} must not contain empty lines",
|
||||
vocabPath);
|
||||
vocab.insert({line, vocab.size()});
|
||||
vocab.insert({line, (Word)vocab.size()});
|
||||
}
|
||||
ABORT_IF(in.bad(), "DefaultVocabulary file {} could not be read", vocabPath);
|
||||
}
|
||||
@ -308,5 +308,4 @@ Ptr<VocabBase> createDefaultVocab() {
|
||||
return New<DefaultVocab>();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
2
src/data/sentencepiece_vocab.cpp
Normal file → Executable file
2
src/data/sentencepiece_vocab.cpp
Normal file → Executable file
@ -133,6 +133,7 @@ Ptr<VocabBase> createSentencePieceVocab(const std::string& vocabPath, Ptr<Option
|
||||
#ifdef USE_SENTENCEPIECE
|
||||
return New<SentencePieceVocab>(options, batchIndex);
|
||||
#else
|
||||
batchIndex;
|
||||
ABORT("*.spm suffix in path {} reserved for SentencePiece models, "
|
||||
"but support for SentencePiece is not compiled into Marian. "
|
||||
"Try to recompile after `cmake .. -DUSE_SENTENCEPIECE=on [...]`",
|
||||
@ -144,4 +145,3 @@ Ptr<VocabBase> createSentencePieceVocab(const std::string& vocabPath, Ptr<Option
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
@ -132,6 +132,8 @@
|
||||
<ClCompile Include="..\src\common\config.cpp" />
|
||||
<ClCompile Include="..\src\common\config_parser.cpp" />
|
||||
<ClCompile Include="..\src\data\alignment.cpp" />
|
||||
<ClCompile Include="..\src\data\default_vocab.cpp" />
|
||||
<ClCompile Include="..\src\data\sentencepiece_vocab.cpp" />
|
||||
<ClCompile Include="..\src\data\vocab.cpp" />
|
||||
<ClCompile Include="..\src\data\corpus_base.cpp" />
|
||||
<ClCompile Include="..\src\data\corpus.cpp" />
|
||||
|
@ -259,6 +259,12 @@
|
||||
<ClCompile Include="..\src\tensors\rand.cpp">
|
||||
<Filter>tensors</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\src\data\default_vocab.cpp">
|
||||
<Filter>data</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\src\data\sentencepiece_vocab.cpp">
|
||||
<Filter>data</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\src\marian.h" />
|
||||
|
Loading…
Reference in New Issue
Block a user