mirror of
https://github.com/marian-nmt/marian.git
synced 2024-09-11 06:15:56 +03:00
Merged PR 10415: Fix windows build errors
1. Added template definition for 'uint64_t', Windows has different definition of 'long' type. 2. Fixed warnings on windows.
This commit is contained in:
parent
9a4f784390
commit
5fb31b28d2
@ -23,7 +23,7 @@ struct Convert {
|
||||
// specialization for translating from string, @TODO check if this is required at all, mostly for compilation now.
|
||||
template <typename To>
|
||||
struct Convert<To, std::string> {
|
||||
static inline To apply(const std::string& from) {
|
||||
static inline To apply(const std::string& /* from */) {
|
||||
ABORT("Not implemented");
|
||||
}
|
||||
};
|
||||
@ -84,7 +84,10 @@ std::vector<T> As<std::vector<T>>::apply(const FastOpt& node) {
|
||||
// specializations for simple vector types
|
||||
template struct As<std::vector<bool>>;
|
||||
template struct As<std::vector<int>>;
|
||||
template struct As<std::vector<unsigned long>>;
|
||||
// Windows and Unix based OS have different type definitions for 'unsigned long'.
|
||||
// So, we need an explicit definition for uint64_t. Otherwise, there's a linking error on windows.
|
||||
// https://software.intel.com/en-us/articles/size-of-long-integer-type-on-different-architecture-and-os/
|
||||
template struct As<std::vector<uint64_t>>;
|
||||
template struct As<std::vector<float>>;
|
||||
template struct As<std::vector<double>>;
|
||||
template struct As<std::vector<std::string>>;
|
||||
|
@ -84,19 +84,19 @@ CorpusBase::CorpusBase(Ptr<Options> options, bool translate)
|
||||
"Vocabularies will be built separately for each file.");
|
||||
|
||||
std::vector<int> vocabDims(paths_.size(), 0);
|
||||
std::vector<std::string> vocabPaths(paths_.size());
|
||||
std::vector<std::string> vocabPaths1(paths_.size());
|
||||
// Create vocabs if not provided
|
||||
for(size_t i = 0; i < paths_.size(); ++i) {
|
||||
Ptr<Vocab> vocab = New<Vocab>(options_, i);
|
||||
std::vector<std::string> trainPaths = { paths_[i] };
|
||||
vocabDims[i] = vocab->loadOrCreate("", trainPaths, maxVocabs[i]);
|
||||
vocabPaths[i] = paths_[i] + ".yml";
|
||||
vocabDims[i] = (int) vocab->loadOrCreate("", trainPaths, maxVocabs[i]);
|
||||
vocabPaths1[i] = paths_[i] + ".yml";
|
||||
vocabs_.emplace_back(vocab);
|
||||
}
|
||||
// TODO: this is not nice as it modifies the option object and needs to expose the changes
|
||||
// outside the corpus as models need to know about the vocabulary size; extract the vocab
|
||||
// creation functionality from the class.
|
||||
options_->set("dim-vocabs", vocabDims, "vocabs", vocabPaths);
|
||||
options_->set("dim-vocabs", vocabDims, "vocabs", vocabPaths1);
|
||||
} else {
|
||||
// Load all vocabs
|
||||
size_t numVocs = vocabPaths.size();
|
||||
@ -128,7 +128,7 @@ CorpusBase::CorpusBase(Ptr<Options> options, bool translate)
|
||||
// it wild not be created again, but just correctly loaded.
|
||||
auto pathsAndSize = groupVocab[vocabPaths[i]];
|
||||
std::vector<std::string> groupedPaths(pathsAndSize.paths.begin(), pathsAndSize.paths.end());
|
||||
vocabDims[i] = vocab->loadOrCreate(vocabPaths[i], groupedPaths, pathsAndSize.size);
|
||||
vocabDims[i] = (int) vocab->loadOrCreate(vocabPaths[i], groupedPaths, pathsAndSize.size);
|
||||
vocabs_.emplace_back(vocab);
|
||||
}
|
||||
// TODO: this is not nice as it modifies the option object and needs to expose the changes
|
||||
@ -150,7 +150,7 @@ CorpusBase::CorpusBase(Ptr<Options> options, bool translate)
|
||||
vocabDims.resize(numVocs, 0);
|
||||
for(size_t i = 0; i + 1 < numVocs; ++i) {
|
||||
Ptr<Vocab> vocab = New<Vocab>(options_, i);
|
||||
vocabDims[i] = vocab->load(vocabPaths[i], maxVocabs[i]);
|
||||
vocabDims[i] = (int) vocab->load(vocabPaths[i], maxVocabs[i]);
|
||||
vocabs_.emplace_back(vocab);
|
||||
}
|
||||
// TODO: As above, this is not nice as it modifies the option object and needs to expose the changes
|
||||
|
@ -531,7 +531,7 @@ public:
|
||||
auto defaultParams = std::dynamic_pointer_cast<MappedParameters>(it->second);
|
||||
if(!defaultParams) {
|
||||
// but it's not mapped, so delete it and replace it with a mapped version
|
||||
auto defaultParams = New<MappedParameters>(defaultElementType_);
|
||||
defaultParams = New<MappedParameters>(defaultElementType_);
|
||||
defaultParams->init(backend_);
|
||||
paramsByElementType_[defaultElementType_] = defaultParams;
|
||||
}
|
||||
@ -540,8 +540,8 @@ public:
|
||||
|
||||
// pre-populate parameters by type
|
||||
for(auto& item : items) {
|
||||
auto it = paramsByElementType_.find(item.type);
|
||||
if(it == paramsByElementType_.end()) {
|
||||
auto it1 = paramsByElementType_.find(item.type);
|
||||
if(it1 == paramsByElementType_.end()) {
|
||||
auto params = New<MappedParameters>(item.type);
|
||||
params->init(backend_);
|
||||
paramsByElementType_.insert({item.type, params});
|
||||
|
@ -130,8 +130,8 @@ private:
|
||||
Ptr<Backend> backend_;
|
||||
|
||||
public:
|
||||
MappedParameters(Type acceptedElementType_) : Parameters(acceptedElementType_) {
|
||||
LOG(debug, "Created mapped parameter object of type {}", acceptedElementType_);
|
||||
MappedParameters(Type acceptedElementType) : Parameters(acceptedElementType) {
|
||||
LOG(debug, "Created mapped parameter object of type {}", acceptedElementType);
|
||||
}
|
||||
|
||||
virtual void init(Ptr<Backend> backend) override { backend_ = backend; }
|
||||
|
@ -196,7 +196,7 @@ Ptr<DecoderState> EncoderDecoder::step(Ptr<ExpressionGraph> graph,
|
||||
state = hypIndices.empty() ? state : state->select(hypIndices, batchIndices, beamSize);
|
||||
|
||||
// Fill state with embeddings based on last prediction
|
||||
decoders_[0]->embeddingsFromPrediction(graph, state, words, batchIndices.size(), beamSize);
|
||||
decoders_[0]->embeddingsFromPrediction(graph, state, words, (int) batchIndices.size(), beamSize);
|
||||
auto nextState = decoders_[0]->step(graph, state);
|
||||
|
||||
return nextState;
|
||||
|
@ -235,7 +235,7 @@ public:
|
||||
|
||||
if(PURGE_BATCH)
|
||||
if(newBeam.empty() && !beam.empty()) { // previous beam had hyps, but all were finished in this step, newBeam will now stay empty
|
||||
for(int i = beamIdx + 1; i < beams.size(); ++i) // for all entries above this beam
|
||||
for(size_t i = beamIdx + 1; i < beams.size(); ++i) // for all entries above this beam
|
||||
batchIdxMap[i] = batchIdxMap[i] - 1; // make them look at one batch index below, as the current entry will be removed from the batch.
|
||||
}
|
||||
|
||||
@ -406,7 +406,7 @@ public:
|
||||
}
|
||||
}
|
||||
if(factorGroup == 0)
|
||||
currentDimBatch = batchIndices.size(); // keep batch size constant for all factor groups in a time step
|
||||
currentDimBatch = (IndexType) batchIndices.size(); // keep batch size constant for all factor groups in a time step
|
||||
prevPathScores = graph->constant({(int)maxBeamSize, 1, (int)currentDimBatch, 1}, inits::fromVector(prevScores));
|
||||
}
|
||||
if (!anyCanExpand) // all words cannot expand this factor: skip
|
||||
|
@ -586,6 +586,10 @@
|
||||
<ClCompile Include="..\src\3rd_party\pathie-cpp\src\pathie_ifstream.cpp" />
|
||||
<ClCompile Include="..\src\3rd_party\pathie-cpp\src\pathie_ofstream.cpp" />
|
||||
<ClCompile Include="..\src\3rd_party\pathie-cpp\src\temp.cpp" />
|
||||
<ClCompile Include="..\src\3rd_party\phf\phf.cc">
|
||||
<WarningLevel Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">TurnOffAllWarnings</WarningLevel>
|
||||
<WarningLevel Condition="'$(Configuration)|$(Platform)'=='Release|x64'">TurnOffAllWarnings</WarningLevel>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\src\3rd_party\sentencepiece\src\bpe_model.cc">
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
|
||||
@ -962,6 +966,7 @@
|
||||
<ClInclude Include="..\src\3rd_party\pathie-cpp\include\pathie_ifstream.hpp" />
|
||||
<ClInclude Include="..\src\3rd_party\pathie-cpp\include\pathie_ofstream.hpp" />
|
||||
<ClInclude Include="..\src\3rd_party\pathie-cpp\include\temp.hpp" />
|
||||
<ClInclude Include="..\src\3rd_party\phf\phf.h" />
|
||||
<ClInclude Include="..\src\3rd_party\sentencepiece\src\bpe_model.h">
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
|
||||
@ -1094,6 +1099,7 @@
|
||||
<ClCompile Include="..\src\common\cli_helper.cpp" />
|
||||
<ClCompile Include="..\src\common\cli_wrapper.cpp" />
|
||||
<ClCompile Include="..\src\common\config_validator.cpp" />
|
||||
<ClCompile Include="..\src\common\fastopt.cpp" />
|
||||
<ClCompile Include="..\src\common\filesystem.cpp" />
|
||||
<ClCompile Include="..\src\common\file_stream.cpp" />
|
||||
<ClCompile Include="..\src\common\io.cpp" />
|
||||
@ -1251,6 +1257,7 @@
|
||||
<ClInclude Include="..\src\common\cli_helper.h" />
|
||||
<ClInclude Include="..\src\common\cli_wrapper.h" />
|
||||
<ClInclude Include="..\src\common\config_validator.h" />
|
||||
<ClInclude Include="..\src\common\fastopt.h" />
|
||||
<ClInclude Include="..\src\common\filesystem.h" />
|
||||
<ClInclude Include="..\src\common\hash.h" />
|
||||
<ClInclude Include="..\src\common\io.h" />
|
||||
|
@ -733,6 +733,12 @@
|
||||
<ClCompile Include="..\src\common\types.cpp">
|
||||
<Filter>common</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\src\common\fastopt.cpp">
|
||||
<Filter>common</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\src\3rd_party\phf\phf.cc">
|
||||
<Filter>3rd_party\phf</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\src\marian.h" />
|
||||
@ -2050,6 +2056,12 @@
|
||||
<ClInclude Include="..\src\graph\expression_graph_packable.h">
|
||||
<Filter>graph</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\src\common\fastopt.h">
|
||||
<Filter>common</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\src\3rd_party\phf\phf.h">
|
||||
<Filter>3rd_party\phf</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Filter Include="3rd_party">
|
||||
@ -2283,6 +2295,9 @@
|
||||
<Filter Include="3rd_party\half_float">
|
||||
<UniqueIdentifier>{defd3aec-3c56-4d70-a4bb-90ba9003d98d}</UniqueIdentifier>
|
||||
</Filter>
|
||||
<Filter Include="3rd_party\phf">
|
||||
<UniqueIdentifier>{352ac0e9-daed-437a-bc36-fb85ecd037eb}</UniqueIdentifier>
|
||||
</Filter>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="..\src\3rd_party\nccl\src\bootstrap.cu">
|
||||
|
Loading…
Reference in New Issue
Block a user