marian-nmt/marian
1
1
Fork 0
mirror of https://github.com/marian-nmt/marian.git synced 2024-09-11 06:15:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merged PR 10415: Fix windows build errors

Browse Source 
1. Added template definition for 'uint64_t', Windows has different definition of 'long' type.
2. Fixed warnings on windows.
This commit is contained in:
Young Jin Kim 2019-11-13 07:37:46 +00:00
parent 9a4f784390
commit 5fb31b28d2
8 changed files with 41 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/common/fastopt.cpp
View File

@ -23,7 +23,7 @@ struct Convert {
// specialization for translating from string, @TODO check if this is required at all, mostly for compilation now.
template <typename To>
struct Convert<To, std::string> {
static inline To apply(const std::string& from) {
static inline To apply(const std::string& /* from */) {
ABORT("Not implemented");
}
};
@ -84,7 +84,10 @@ std::vector<T> As<std::vector<T>>::apply(const FastOpt& node) {
// specializations for simple vector types
template struct As<std::vector<bool>>;
template struct As<std::vector<int>>;
template struct As<std::vector<unsigned long>>;
// Windows and Unix based OS have different type definitions for 'unsigned long'.
// So, we need an explicit definition for uint64_t. Otherwise, there's a linking error on windows.
// https://software.intel.com/en-us/articles/size-of-long-integer-type-on-different-architecture-and-os/
template struct As<std::vector<uint64_t>>;
template struct As<std::vector<float>>;
template struct As<std::vector<double>>;
template struct As<std::vector<std::string>>;

12
src/data/corpus_base.cpp
View File

@ -84,19 +84,19 @@ CorpusBase::CorpusBase(Ptr<Options> options, bool translate)
"Vocabularies will be built separately for each file.");
std::vector<int> vocabDims(paths_.size(), 0);
std::vector<std::string> vocabPaths(paths_.size());
std::vector<std::string> vocabPaths1(paths_.size());
// Create vocabs if not provided
for(size_t i = 0; i < paths_.size(); ++i) {
Ptr<Vocab> vocab = New<Vocab>(options_, i);
std::vector<std::string> trainPaths = { paths_[i] };
vocabDims[i] = vocab->loadOrCreate("", trainPaths, maxVocabs[i]);
vocabPaths[i] = paths_[i] + ".yml";
vocabDims[i] = (int) vocab->loadOrCreate("", trainPaths, maxVocabs[i]);
vocabPaths1[i] = paths_[i] + ".yml";
vocabs_.emplace_back(vocab);
}
// TODO: this is not nice as it modifies the option object and needs to expose the changes
// outside the corpus as models need to know about the vocabulary size; extract the vocab
// creation functionality from the class.
options_->set("dim-vocabs", vocabDims, "vocabs", vocabPaths);
options_->set("dim-vocabs", vocabDims, "vocabs", vocabPaths1);
} else {
// Load all vocabs
size_t numVocs = vocabPaths.size();
@ -128,7 +128,7 @@ CorpusBase::CorpusBase(Ptr<Options> options, bool translate)
// it wild not be created again, but just correctly loaded.
auto pathsAndSize = groupVocab[vocabPaths[i]];
std::vector<std::string> groupedPaths(pathsAndSize.paths.begin(), pathsAndSize.paths.end());
vocabDims[i] = vocab->loadOrCreate(vocabPaths[i], groupedPaths, pathsAndSize.size);
vocabDims[i] = (int) vocab->loadOrCreate(vocabPaths[i], groupedPaths, pathsAndSize.size);
vocabs_.emplace_back(vocab);
}
// TODO: this is not nice as it modifies the option object and needs to expose the changes
@ -150,7 +150,7 @@ CorpusBase::CorpusBase(Ptr<Options> options, bool translate)
vocabDims.resize(numVocs, 0);
for(size_t i = 0; i + 1 < numVocs; ++i) {
Ptr<Vocab> vocab = New<Vocab>(options_, i);
vocabDims[i] = vocab->load(vocabPaths[i], maxVocabs[i]);
vocabDims[i] = (int) vocab->load(vocabPaths[i], maxVocabs[i]);
vocabs_.emplace_back(vocab);
}
// TODO: As above, this is not nice as it modifies the option object and needs to expose the changes

6
src/graph/expression_graph.h
View File

@ -531,7 +531,7 @@ public:
auto defaultParams = std::dynamic_pointer_cast<MappedParameters>(it->second);
if(!defaultParams) {
// but it's not mapped, so delete it and replace it with a mapped version
auto defaultParams = New<MappedParameters>(defaultElementType_);
defaultParams = New<MappedParameters>(defaultElementType_);
defaultParams->init(backend_);
paramsByElementType_[defaultElementType_] = defaultParams;
}
@ -540,8 +540,8 @@ public:
// pre-populate parameters by type
for(auto& item : items) {
auto it = paramsByElementType_.find(item.type);
if(it == paramsByElementType_.end()) {
auto it1 = paramsByElementType_.find(item.type);
if(it1 == paramsByElementType_.end()) {
auto params = New<MappedParameters>(item.type);
params->init(backend_);
paramsByElementType_.insert({item.type, params});

4
src/graph/parameters.h
View File

@ -130,8 +130,8 @@ private:
Ptr<Backend> backend_;
public:
MappedParameters(Type acceptedElementType_) : Parameters(acceptedElementType_) {
LOG(debug, "Created mapped parameter object of type {}", acceptedElementType_);
MappedParameters(Type acceptedElementType) : Parameters(acceptedElementType) {
LOG(debug, "Created mapped parameter object of type {}", acceptedElementType);
}
virtual void init(Ptr<Backend> backend) override { backend_ = backend; }

2
src/models/encoder_decoder.cpp
View File

@ -196,7 +196,7 @@ Ptr<DecoderState> EncoderDecoder::step(Ptr<ExpressionGraph> graph,
state = hypIndices.empty() ? state : state->select(hypIndices, batchIndices, beamSize);
// Fill state with embeddings based on last prediction
decoders_[0]->embeddingsFromPrediction(graph, state, words, batchIndices.size(), beamSize);
decoders_[0]->embeddingsFromPrediction(graph, state, words, (int) batchIndices.size(), beamSize);
auto nextState = decoders_[0]->step(graph, state);
return nextState;

4
src/translator/beam_search.h
View File

@ -235,7 +235,7 @@ public:
if(PURGE_BATCH)
if(newBeam.empty() && !beam.empty()) { // previous beam had hyps, but all were finished in this step, newBeam will now stay empty
for(int i = beamIdx + 1; i < beams.size(); ++i) // for all entries above this beam
for(size_t i = beamIdx + 1; i < beams.size(); ++i) // for all entries above this beam
batchIdxMap[i] = batchIdxMap[i] - 1; // make them look at one batch index below, as the current entry will be removed from the batch.
}
@ -406,7 +406,7 @@ public:
}
}
if(factorGroup == 0)
currentDimBatch = batchIndices.size(); // keep batch size constant for all factor groups in a time step
currentDimBatch = (IndexType) batchIndices.size(); // keep batch size constant for all factor groups in a time step
prevPathScores = graph->constant({(int)maxBeamSize, 1, (int)currentDimBatch, 1}, inits::fromVector(prevScores));
}
if (!anyCanExpand) // all words cannot expand this factor: skip

7
vs/Marian.vcxproj
View File

@ -586,6 +586,10 @@
<ClCompile Include="..\src\3rd_party\pathie-cpp\src\pathie_ifstream.cpp" />
<ClCompile Include="..\src\3rd_party\pathie-cpp\src\pathie_ofstream.cpp" />
<ClCompile Include="..\src\3rd_party\pathie-cpp\src\temp.cpp" />
<ClCompile Include="..\src\3rd_party\phf\phf.cc">
<WarningLevel Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">TurnOffAllWarnings</WarningLevel>
<WarningLevel Condition="'$(Configuration)|$(Platform)'=='Release|x64'">TurnOffAllWarnings</WarningLevel>
</ClCompile>
<ClCompile Include="..\src\3rd_party\sentencepiece\src\bpe_model.cc">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
@ -962,6 +966,7 @@
<ClInclude Include="..\src\3rd_party\pathie-cpp\include\pathie_ifstream.hpp" />
<ClInclude Include="..\src\3rd_party\pathie-cpp\include\pathie_ofstream.hpp" />
<ClInclude Include="..\src\3rd_party\pathie-cpp\include\temp.hpp" />
<ClInclude Include="..\src\3rd_party\phf\phf.h" />
<ClInclude Include="..\src\3rd_party\sentencepiece\src\bpe_model.h">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
@ -1094,6 +1099,7 @@
<ClCompile Include="..\src\common\cli_helper.cpp" />
<ClCompile Include="..\src\common\cli_wrapper.cpp" />
<ClCompile Include="..\src\common\config_validator.cpp" />
<ClCompile Include="..\src\common\fastopt.cpp" />
<ClCompile Include="..\src\common\filesystem.cpp" />
<ClCompile Include="..\src\common\file_stream.cpp" />
<ClCompile Include="..\src\common\io.cpp" />
@ -1251,6 +1257,7 @@
<ClInclude Include="..\src\common\cli_helper.h" />
<ClInclude Include="..\src\common\cli_wrapper.h" />
<ClInclude Include="..\src\common\config_validator.h" />
<ClInclude Include="..\src\common\fastopt.h" />
<ClInclude Include="..\src\common\filesystem.h" />
<ClInclude Include="..\src\common\hash.h" />
<ClInclude Include="..\src\common\io.h" />

15
vs/Marian.vcxproj.filters
View File

@ -733,6 +733,12 @@
<ClCompile Include="..\src\common\types.cpp">
<Filter>common</Filter>
</ClCompile>
<ClCompile Include="..\src\common\fastopt.cpp">
<Filter>common</Filter>
</ClCompile>
<ClCompile Include="..\src\3rd_party\phf\phf.cc">
<Filter>3rd_party\phf</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\src\marian.h" />
@ -2050,6 +2056,12 @@
<ClInclude Include="..\src\graph\expression_graph_packable.h">
<Filter>graph</Filter>
</ClInclude>
<ClInclude Include="..\src\common\fastopt.h">
<Filter>common</Filter>
</ClInclude>
<ClInclude Include="..\src\3rd_party\phf\phf.h">
<Filter>3rd_party\phf</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Filter Include="3rd_party">
@ -2283,6 +2295,9 @@
<Filter Include="3rd_party\half_float">
<UniqueIdentifier>{defd3aec-3c56-4d70-a4bb-90ba9003d98d}</UniqueIdentifier>
</Filter>
<Filter Include="3rd_party\phf">
<UniqueIdentifier>{352ac0e9-daed-437a-bc36-fb85ecd037eb}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<None Include="..\src\3rd_party\nccl\src\bootstrap.cu">