marian-nmt/marian
1
1
Fork 0
mirror of https://github.com/marian-nmt/marian.git synced 2024-09-11 06:15:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

fixed (eliminated) all TABs that had snuck in so far; fixed a few warnings on Windows

Browse Source
This commit is contained in:
Frank Seide 2018-10-15 13:27:29 -07:00
parent 07b85c0101
commit e8443b0875
15 changed files with 9664 additions and 9668 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/3rd_party/catch.hpp vendored Normal file → Executable file
View File

@ -927,8 +927,8 @@ namespace Catch {
template<typename T> STATIC_ASSERT_Expression_Too_Complex_Please_Rewrite_As_Binary_Comparison& operator && ( T const& );
template<typename T> STATIC_ASSERT_Expression_Too_Complex_Please_Rewrite_As_Binary_Comparison& operator || ( T const& );
private:
DecomposedExpression& operator = (DecomposedExpression const&);
private:
DecomposedExpression& operator = (DecomposedExpression const&);
};
struct AssertionInfo
@ -3490,16 +3490,16 @@ return @ desc; \
#include <crtdbg.h>
class LeakDetector {
public:
LeakDetector() {
int flag = _CrtSetDbgFlag(_CRTDBG_REPORT_FLAG);
flag |= _CRTDBG_LEAK_CHECK_DF;
flag |= _CRTDBG_ALLOC_MEM_DF;
_CrtSetDbgFlag(flag);
_CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG);
_CrtSetReportFile(_CRT_WARN, _CRTDBG_FILE_STDERR);
// Change this to leaking allocation's number to break there
_CrtSetBreakAlloc(-1);
}
LeakDetector() {
int flag = _CrtSetDbgFlag(_CRTDBG_REPORT_FLAG);
flag |= _CRTDBG_LEAK_CHECK_DF;
flag |= _CRTDBG_ALLOC_MEM_DF;
_CrtSetDbgFlag(flag);
_CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG);
_CrtSetReportFile(_CRT_WARN, _CRTDBG_FILE_STDERR);
// Change this to leaking allocation's number to break there
_CrtSetBreakAlloc(-1);
}
};
#else
class LeakDetector {};
@ -11303,7 +11303,7 @@ extern "C" int wmain (int argc, wchar_t * argv[], wchar_t * []) {
int main (int argc, char * argv[]) {
#endif
int result = Catch::Session().run( argc, argv );
int result = Catch::Session().run(argc, argv);
return ( result < 0xff ? result : 0xff );
}

2
src/3rd_party/exception.h vendored Normal file → Executable file
View File

@ -137,7 +137,7 @@ class ErrnoException : public Exception {
// file wasn't there, or couldn't be open for some reason
class FileOpenException : public Exception {
public:
FileOpenException() throw() {}
FileOpenException() throw() {}
~FileOpenException() throw() {}
};

18854
src/3rd_party/spdlog/tests/catch.hpp vendored Normal file → Executable file
View File

File diff suppressed because it is too large Load Diff

8
src/3rd_party/svd/defs_and_types.h vendored Normal file → Executable file
View File

@ -35,19 +35,19 @@
typedef struct {
float x, y, z;
float x, y, z;
} fcoords;
typedef struct {
long x, y, z;
long x, y, z;
} lcoords;
typedef struct {
int x, y, z;
int x, y, z;
} icoords;
typedef struct {
float min, max;
float min, max;
} lims;

4
src/3rd_party/yaml-cpp/dll.h vendored Normal file → Executable file
View File

@ -20,10 +20,10 @@
// manually)
#ifdef yaml_cpp_EXPORTS // Building YAML-CPP DLL (definition created by CMake
// or defined manually)
// #pragma message( "Defining YAML_CPP_API for DLL export" )
// #pragma message( "Defining YAML_CPP_API for DLL export" )
#define YAML_CPP_API __declspec(dllexport)
#else // yaml_cpp_EXPORTS
// #pragma message( "Defining YAML_CPP_API for DLL import" )
// #pragma message( "Defining YAML_CPP_API for DLL import" )
#define YAML_CPP_API __declspec(dllimport)
#endif // yaml_cpp_EXPORTS
#else // YAML_CPP_DLL

8
src/3rd_party/yaml-cpp/scanner.cpp vendored Normal file → Executable file
View File

@ -34,10 +34,10 @@ Token& Scanner::peek() {
// if it's empty before peeking.
#if 0
static Token *pLast = 0;
if(pLast != &m_tokens.front())
std::cerr << "peek: " << m_tokens.front() << "\n";
pLast = &m_tokens.front();
static Token *pLast = 0;
if (pLast != &m_tokens.front())
std::cerr << "peek: " << m_tokens.front() << "\n";
pLast = &m_tokens.front();
#endif
return m_tokens.front();

2
src/3rd_party/yaml-cpp/scantoken.cpp vendored Normal file → Executable file
View File

@ -320,7 +320,7 @@ void Scanner::ScanPlainScalar() {
// finally, check and see if we ended on an illegal character
// if(Exp::IllegalCharInScalar.Matches(INPUT))
// throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_SCALAR);
// throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_SCALAR);
Token token(Token::PLAIN_SCALAR, mark);
token.value = scalar;

26
src/common/config_parser.cpp
View File

@ -366,30 +366,30 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) {
true);
// support for universal encoder ULR https://arxiv.org/pdf/1802.05368.pdf
cli.add<bool>("--ulr-enabled",
"Is ULR (Universal Language Representation) enabled?",
false);
"Is ULR (Universal Language Representation) enabled?",
false);
// reading pre-trained universal embedings for multi-sources
// note that source and target here is relative to ULR not the translation langs
//queries: EQ in Fig2 : is the unified embbedins projected to one space.
//"Path to file with universal sources embeddings from projection into universal space")
cli.add<bool>("--ulr-query-vectors",
"Path to file with universal sources embeddings from projection into universal space",
"");
"Path to file with universal sources embeddings from projection into universal space",
"");
//keys: EK in Fig2 : is the keys of the target embbedins projected to unified space (i.e. ENU in multi-lingual case)
cli.add<std::string>("--ulr-keys-vectors",
"Path to file with universal sources embeddings of traget keys from projection into universal space",
"");
"Path to file with universal sources embeddings of traget keys from projection into universal space",
"");
cli.add<bool>("--ulr-trainable-transformation",
"Is Query Transformation Matrix A trainable ?",
false);
"Is Query Transformation Matrix A trainable ?",
false);
cli.add<int>("--ulr-dim-emb",
"ULR mono embed dim");
"ULR mono embed dim");
cli.add<float>("--ulr-dropout",
"ULR dropout on embeddings attentions: default is no dropuout",
0.0f);
"ULR dropout on embeddings attentions: default is no dropuout",
0.0f);
cli.add<float>("--ulr-softmax-temperature",
"ULR softmax temperature to control randomness of predictions- deafult is 1.0: no temperature ",
1.0f);
"ULR softmax temperature to control randomness of predictions- deafult is 1.0: no temperature ",
1.0f);
// clang-format on
}

12
src/common/config_validator.cpp
View File

@ -137,13 +137,11 @@ void ConfigValidator::validateDevices(cli::mode mode) const {
+ ")' for option '--devices' is invalid. "
+ help);
// validate ULR options
UTIL_THROW_IF2(
(get<bool>("ulr-enabled") == true &&
(get<std::string>("ulr-query-vectors") == ""
|| get<std::string>("ulr-keys-vectors") == "")),
"ULR enablign requires query and keys vectors specified with "
"--ulr-query-vectors and --ulr-keys-vectors option");
UTIL_THROW_IF2((get<bool>("ulr-enabled") == true &&
(get<std::string>("ulr-query-vectors") == ""
|| get<std::string>("ulr-keys-vectors") == "")),
"ULR enablign requires query and keys vectors specified with "
"--ulr-query-vectors and --ulr-keys-vectors option");
}
} // namespace marian

135
src/layers/generic.h
View File

@ -173,83 +173,80 @@ struct EmbeddingFactory : public Factory {
bool fixed = opt<bool>("fixed", false);
NodeInitializer initFunc = inits::glorot_uniform;
if (options_->has("embFile")) {
std::string file = opt<std::string>("embFile");
if (!file.empty()) {
bool norm = opt<bool>("normalization", false);
initFunc = inits::from_word2vec(file, dimVoc, dimEmb, norm);
}
}
if (options_->has("embFile")) {
std::string file = opt<std::string>("embFile");
if (!file.empty()) {
bool norm = opt<bool>("normalization", false);
initFunc = inits::from_word2vec(file, dimVoc, dimEmb, norm);
}
}
return graph_->param(name, {dimVoc, dimEmb}, initFunc, fixed);
}
};
struct ULREmbeddingFactory : public Factory {
ULREmbeddingFactory(Ptr<ExpressionGraph> graph) : Factory(graph) {}
ULREmbeddingFactory(Ptr<ExpressionGraph> graph) : Factory(graph) {}
std::vector<Expr> construct() {
std::string name = "url_embed"; //opt<std::string>("prefix");
int dimKeys = opt<int>("dimTgtVoc");
int dimQueries = opt<int>("dimSrcVoc");
int dimEmb = opt<int>("dimEmb");
int dimUlrEmb = opt<int>("dimUlrEmb"); // ULR mono embed size
bool fixed = opt<bool>("fixed", false);
std::vector<Expr> ulrEmbeds;
NodeInitializer initFunc = inits::glorot_uniform;
std::string queryFile = opt<std::string>("ulrQueryFile");
std::string keyFile = opt<std::string>("ulrKeysFile");
bool norm = opt<bool>("normalization", false);
bool trainTrans = opt<bool>("ulrTrainTransform", false);
if (!queryFile.empty() && !keyFile.empty()) {
initFunc = inits::from_word2vec(queryFile, dimQueries, dimUlrEmb, false);
name = "ulr_query";
fixed = true;
auto query_embed = graph_->param(name, { dimQueries, dimUlrEmb }, initFunc, fixed);
ulrEmbeds.push_back(query_embed);
// keys embeds
initFunc = inits::from_word2vec(keyFile, dimKeys, dimUlrEmb, false);
name = "ulr_keys";
fixed = true;
auto key_embed = graph_->param(name, { dimKeys, dimUlrEmb }, initFunc, fixed);
ulrEmbeds.push_back(key_embed);
// actual trainable embedding
initFunc = inits::glorot_uniform;
name = "ulr_embed";
fixed = false;
auto ulr_embed = graph_->param(name, {dimKeys , dimEmb }, initFunc, fixed); // note the reverse dim
ulrEmbeds.push_back(ulr_embed);
// init trainable src embedding
name = "ulr_src_embed";
auto ulr_src_embed = graph_->param(name, { dimQueries, dimEmb }, initFunc, fixed);
ulrEmbeds.push_back(ulr_src_embed);
// ulr transformation matrix
//initFunc = inits::diag(1.f); // identity matrix - is it ok to init wiht identity or shall we make this to the fixed case only
if (trainTrans) {
initFunc = inits::glorot_uniform;
fixed = false;
}
else
{
initFunc = inits::diag(1.f); // identity matrix
fixed = true;
}
name = "ulr_transform";
auto ulr_transform = graph_->param(name, { dimUlrEmb, dimUlrEmb }, initFunc, fixed);
ulrEmbeds.push_back(ulr_transform);
initFunc = inits::from_value(1.f); // TBD: we should read sharable flags here - 1 means all sharable - 0 means no universal embeddings - should be zero for top freq only
fixed = true;
name = "ulr_shared";
auto share_embed = graph_->param(name, { dimQueries, 1 }, initFunc, fixed);
ulrEmbeds.push_back(share_embed);
std::vector<Expr> construct() {
std::string name = "url_embed"; //opt<std::string>("prefix");
int dimKeys = opt<int>("dimTgtVoc");
int dimQueries = opt<int>("dimSrcVoc");
int dimEmb = opt<int>("dimEmb");
int dimUlrEmb = opt<int>("dimUlrEmb"); // ULR mono embed size
bool fixed = opt<bool>("fixed", false);
std::vector<Expr> ulrEmbeds;
NodeInitializer initFunc = inits::glorot_uniform;
std::string queryFile = opt<std::string>("ulrQueryFile");
std::string keyFile = opt<std::string>("ulrKeysFile");
bool trainTrans = opt<bool>("ulrTrainTransform", false);
}
return ulrEmbeds;
}
if (!queryFile.empty() && !keyFile.empty()) {
initFunc = inits::from_word2vec(queryFile, dimQueries, dimUlrEmb, false);
name = "ulr_query";
fixed = true;
auto query_embed = graph_->param(name, { dimQueries, dimUlrEmb }, initFunc, fixed);
ulrEmbeds.push_back(query_embed);
// keys embeds
initFunc = inits::from_word2vec(keyFile, dimKeys, dimUlrEmb, false);
name = "ulr_keys";
fixed = true;
auto key_embed = graph_->param(name, { dimKeys, dimUlrEmb }, initFunc, fixed);
ulrEmbeds.push_back(key_embed);
// actual trainable embedding
initFunc = inits::glorot_uniform;
name = "ulr_embed";
fixed = false;
auto ulr_embed = graph_->param(name, {dimKeys , dimEmb }, initFunc, fixed); // note the reverse dim
ulrEmbeds.push_back(ulr_embed);
// init trainable src embedding
name = "ulr_src_embed";
auto ulr_src_embed = graph_->param(name, { dimQueries, dimEmb }, initFunc, fixed);
ulrEmbeds.push_back(ulr_src_embed);
// ulr transformation matrix
//initFunc = inits::diag(1.f); // identity matrix - is it ok to init wiht identity or shall we make this to the fixed case only
if (trainTrans) {
initFunc = inits::glorot_uniform;
fixed = false;
}
else
{
initFunc = inits::diag(1.f); // identity matrix
fixed = true;
}
name = "ulr_transform";
auto ulr_transform = graph_->param(name, { dimUlrEmb, dimUlrEmb }, initFunc, fixed);
ulrEmbeds.push_back(ulr_transform);
initFunc = inits::from_value(1.f); // TBD: we should read sharable flags here - 1 means all sharable - 0 means no universal embeddings - should be zero for top freq only
fixed = true;
name = "ulr_shared";
auto share_embed = graph_->param(name, { dimQueries, 1 }, initFunc, fixed);
ulrEmbeds.push_back(share_embed);
}
return ulrEmbeds;
}
};
typedef Accumulator<EmbeddingFactory> embedding;

100
src/models/encoder.h
View File

@ -16,62 +16,60 @@ protected:
// virtual
std::tuple<Expr, Expr> lookup(Ptr<ExpressionGraph> graph,
Expr srcEmbeddings,
Ptr<data::CorpusBatch> batch) const {
auto subBatch = (*batch)[batchIndex_];
int dimBatch = (int)subBatch->batchSize();
int dimEmb = srcEmbeddings->shape()[-1];
int dimWords = (int)subBatch->batchWidth();
auto chosenEmbeddings = rows(srcEmbeddings, subBatch->data());
auto batchEmbeddings
= reshape(chosenEmbeddings, {dimWords, dimBatch, dimEmb});
auto batchMask = graph->constant({dimWords, dimBatch, 1},
inits::from_vector(subBatch->mask()));
Ptr<data::CorpusBatch> batch) const {
auto subBatch = (*batch)[batchIndex_];
int dimBatch = (int)subBatch->batchSize();
int dimEmb = srcEmbeddings->shape()[-1];
int dimWords = (int)subBatch->batchWidth();
auto chosenEmbeddings = rows(srcEmbeddings, subBatch->data());
auto batchEmbeddings = reshape(chosenEmbeddings, { dimWords, dimBatch, dimEmb });
auto batchMask = graph->constant({ dimWords, dimBatch, 1 },
inits::from_vector(subBatch->mask()));
return std::make_tuple(batchEmbeddings, batchMask);
return std::make_tuple(batchEmbeddings, batchMask);
}
std::tuple<Expr, Expr> ulrLookup(Ptr<ExpressionGraph> graph,
std::vector<Expr> urlEmbeddings,
Ptr<data::CorpusBatch> batch) const {
auto subBatch = (*batch)[batchIndex_];
// is their a better way to do this?
assert(urlEmbeddings.size() == 6);
auto queryEmbed = urlEmbeddings[0]; //Q : dimQueries*dimUlrEmb
auto keyEmbed = urlEmbeddings[1]; // K : dimKeys*dimUlrEmb
auto uniEmbed = urlEmbeddings[2]; // E : dimQueries*dimEmb
auto srcEmbed = urlEmbeddings[3]; // I : dimQueries*dimEmb
auto ulrTransform = urlEmbeddings[4]; //A : dimUlrEmb *dimUlrEmb
auto ulrSharable = urlEmbeddings[5]; //alpha : dimQueries*1
int dimBatch = (int)subBatch->batchSize();
int dimEmb = uniEmbed->shape()[-1];
int dimWords = (int)subBatch->batchWidth();
// D = K.A.QT
// dimm(K) = univ_tok_vocab*uni_embed_size
// dim A = uni_embed_size*uni_embed_size
// dim Q: uni_embed_size * total_merged_vocab_size
// dim D = univ_tok_vocab * total_merged_vocab_size
// note all above can be precombuted and serialized if A is not trainiabale and during decoding (TBD)
// here we need to handle the mini-batch
// extract raws corresponding to Xs in this mini batch from Q
auto queryEmbeddings = rows(queryEmbed, subBatch->data());
auto srcEmbeddings = rows(srcEmbed, subBatch->data()); // extract trainable src embeddings
auto alpha = rows(ulrSharable, subBatch->data()); // extract sharable flags
auto qt = dot(queryEmbeddings, ulrTransform,false, false); //A: transform embeddings based on similarity A : dimUlrEmb *dimUlrEmb
auto z = dot(qt, keyEmbed, false, true); // query-key similarity
std::vector<Expr> urlEmbeddings,
Ptr<data::CorpusBatch> batch) const {
auto subBatch = (*batch)[batchIndex_];
// is their a better way to do this?
assert(urlEmbeddings.size() == 6);
auto queryEmbed = urlEmbeddings[0]; //Q : dimQueries*dimUlrEmb
auto keyEmbed = urlEmbeddings[1]; // K : dimKeys*dimUlrEmb
auto uniEmbed = urlEmbeddings[2]; // E : dimQueries*dimEmb
auto srcEmbed = urlEmbeddings[3]; // I : dimQueries*dimEmb
auto ulrTransform = urlEmbeddings[4]; //A : dimUlrEmb *dimUlrEmb
auto ulrSharable = urlEmbeddings[5]; //alpha : dimQueries*1
int dimBatch = (int)subBatch->batchSize();
int dimEmb = uniEmbed->shape()[-1];
int dimWords = (int)subBatch->batchWidth();
// D = K.A.QT
// dimm(K) = univ_tok_vocab*uni_embed_size
// dim A = uni_embed_size*uni_embed_size
// dim Q: uni_embed_size * total_merged_vocab_size
// dim D = univ_tok_vocab * total_merged_vocab_size
// note all above can be precombuted and serialized if A is not trainiabale and during decoding (TBD)
// here we need to handle the mini-batch
// extract raws corresponding to Xs in this mini batch from Q
auto queryEmbeddings = rows(queryEmbed, subBatch->data());
auto srcEmbeddings = rows(srcEmbed, subBatch->data()); // extract trainable src embeddings
auto alpha = rows(ulrSharable, subBatch->data()); // extract sharable flags
auto qt = dot(queryEmbeddings, ulrTransform, false, false); //A: transform embeddings based on similarity A : dimUlrEmb *dimUlrEmb
auto z = dot(qt, keyEmbed, false, true); // query-key similarity
float dropProb = this->options_->get<float>("ulr-dropout", 0.0f); // default no dropout
z = dropout(z, dropProb);
float tau = this->options_->get<float>("ulr-softmax-temperature", 1.0f); // default no temperature
// temperature in softmax is to control randomness of predictions
// high temperature Softmax outputs are more close to each other
// low temperatures the softmax become more similar to "hardmax"
auto weights = softmax(z/tau); // assume default is dim=-1, what about temprature? - scaler ??
auto chosenEmbeddings = dot(weights, uniEmbed); // THIS IS WRONG - IT SHOULD BE AVERAGE
auto chosenEmbeddings_mix = srcEmbeddings + alpha * chosenEmbeddings; // this should be elementwise broadcast
auto batchEmbeddings
= reshape(chosenEmbeddings_mix, { dimWords, dimBatch, dimEmb });
auto batchMask = graph->constant({ dimWords, dimBatch, 1 },
inits::from_vector(subBatch->mask()));
return std::make_tuple(batchEmbeddings, batchMask);
z = dropout(z, dropProb);
float tau = this->options_->get<float>("ulr-softmax-temperature", 1.0f); // default no temperature
// temperature in softmax is to control randomness of predictions
// high temperature Softmax outputs are more close to each other
// low temperatures the softmax become more similar to "hardmax"
auto weights = softmax(z / tau); // assume default is dim=-1, what about temprature? - scaler ??
auto chosenEmbeddings = dot(weights, uniEmbed); // THIS IS WRONG - IT SHOULD BE AVERAGE
auto chosenEmbeddings_mix = srcEmbeddings + alpha * chosenEmbeddings; // this should be elementwise broadcast
auto batchEmbeddings = reshape(chosenEmbeddings_mix, { dimWords, dimBatch, dimEmb });
auto batchMask = graph->constant({ dimWords, dimBatch, 1 },
inits::from_vector(subBatch->mask()));
return std::make_tuple(batchEmbeddings, batchMask);
}
public:
EncoderBase(Ptr<Options> options)

85
src/models/transformer.h
View File

@ -481,38 +481,37 @@ public:
// returns the embedding matrix based on options
// and based on batchIndex_.
std::vector<Expr> ULREmbeddings(size_t subBatchIndex) const {
// standard encoder word embeddings
int dimSrcVoc = opt<std::vector<int>>("dim-vocabs")[0]; //ULR multi-lingual src
int dimTgtVoc = opt<std::vector<int>>("dim-vocabs")[1]; //ULR monon tgt
int dimEmb = opt<int>("dim-emb");
int dimUlrEmb = opt<int>("ulr-dim-emb");
auto embFactory = ulr_embedding(graph_)("dimSrcVoc", dimSrcVoc)("dimTgtVoc", dimTgtVoc)
("dimUlrEmb", dimUlrEmb)("dimEmb", dimEmb)
("ulrTrainTransform", opt<bool>("ulr-trainable-transformation"))
("ulrQueryFile", opt<std::string>("ulr-query-vectors"))
("ulrKeysFile" , opt<std::string>("ulr-keys-vectors"));
return embFactory.construct();
std::vector<Expr> ULREmbeddings() const {
// standard encoder word embeddings
int dimSrcVoc = opt<std::vector<int>>("dim-vocabs")[0]; //ULR multi-lingual src
int dimTgtVoc = opt<std::vector<int>>("dim-vocabs")[1]; //ULR monon tgt
int dimEmb = opt<int>("dim-emb");
int dimUlrEmb = opt<int>("ulr-dim-emb");
auto embFactory = ulr_embedding(graph_)("dimSrcVoc", dimSrcVoc)("dimTgtVoc", dimTgtVoc)
("dimUlrEmb", dimUlrEmb)("dimEmb", dimEmb)
("ulrTrainTransform", opt<bool>("ulr-trainable-transformation"))
("ulrQueryFile", opt<std::string>("ulr-query-vectors"))
("ulrKeysFile", opt<std::string>("ulr-keys-vectors"));
return embFactory.construct();
}
Expr wordEmbeddings(size_t subBatchIndex) const {
// standard encoder word embeddings
int dimVoc = opt<std::vector<int>>("dim-vocabs")[subBatchIndex];
int dimEmb = opt<int>("dim-emb");
auto embFactory = embedding(graph_)("dimVocab", dimVoc)("dimEmb", dimEmb);
if(opt<bool>("tied-embeddings-src") || opt<bool>("tied-embeddings-all"))
embFactory("prefix", "Wemb");
else
embFactory("prefix", prefix_ + "_Wemb");
if(options_->has("embedding-fix-src"))
embFactory("fixed", opt<bool>("embedding-fix-src"));
if(options_->has("embedding-vectors")) {
auto embFiles = opt<std::vector<std::string>>("embedding-vectors");
embFactory
("embFile", embFiles[subBatchIndex])
("normalization", opt<bool>("embedding-normalization"));
}
return embFactory.construct();
// standard encoder word embeddings
int dimVoc = opt<std::vector<int>>("dim-vocabs")[subBatchIndex];
int dimEmb = opt<int>("dim-emb");
auto embFactory = embedding(graph_)("dimVocab", dimVoc)("dimEmb", dimEmb);
if (opt<bool>("tied-embeddings-src") || opt<bool>("tied-embeddings-all"))
embFactory("prefix", "Wemb");
else
embFactory("prefix", prefix_ + "_Wemb");
if (options_->has("embedding-fix-src"))
embFactory("fixed", opt<bool>("embedding-fix-src"));
if (options_->has("embedding-vectors")) {
auto embFiles = opt<std::vector<std::string>>("embedding-vectors");
embFactory("embFile", embFiles[subBatchIndex])
("normalization", opt<bool>("embedding-normalization"));
}
return embFactory.construct();
}
Ptr<EncoderState> build(Ptr<ExpressionGraph> graph,
@ -525,20 +524,20 @@ public:
int dimEmb = opt<int>("dim-emb");
int dimBatch = (int)batch->size();
int dimSrcWords = (int)(*batch)[batchIndex_]->batchWidth();
Expr batchEmbeddings, batchMask;
if (options_->has("ulr-enabled") && options_->get<bool>("ulr-enabled") == true) {
auto embeddings = ULREmbeddings(batchIndex_); // embedding matrix, considering tying and some other options
// embed the source words in the batch using ULR
std::tie(batchEmbeddings, batchMask)
= EncoderBase::ulrLookup(graph_, embeddings, batch);
}
else
{
auto embeddings = wordEmbeddings(batchIndex_); // embedding matrix, considering tying and some other options
// embed the source words in the batch
std::tie(batchEmbeddings, batchMask)
= EncoderBase::lookup(graph_, embeddings, batch);
}
// create the embedding matrix, considering tying and some other options
// embed the source words in the batch
Expr batchEmbeddings, batchMask;
if (options_->has("ulr-enabled") && options_->get<bool>("ulr-enabled") == true) {
auto embeddings = ULREmbeddings(); // embedding uses ULR
std::tie(batchEmbeddings, batchMask)
= EncoderBase::ulrLookup(graph_, embeddings, batch);
}
else
{
auto embeddings = wordEmbeddings(batchIndex_);
std::tie(batchEmbeddings, batchMask)
= EncoderBase::lookup(graph_, embeddings, batch);
}
// apply dropout over source words
float dropoutSrc = inference_ ? 0 : opt<float>("dropout-src");
if(dropoutSrc) {

64
src/tensors/gpu/cuda_helpers.h
View File

@ -47,36 +47,36 @@ void CudaCopy(const T* start, const T* end, T* dest) {
}
// void cusparseStatus(cusparseStatus_t status){
// switch(status){
// case CUSPARSE_STATUS_INVALID_VALUE:
// printf("invalid value");
// break;
// case CUSPARSE_STATUS_NOT_INITIALIZED:
// printf("not initialized");
// break;
// case CUSPARSE_STATUS_ARCH_MISMATCH:
// printf("arch mismatch");
// break;
// case CUSPARSE_STATUS_EXECUTION_FAILED:
// printf("exe failed");
// break;
// case CUSPARSE_STATUS_INTERNAL_ERROR:
// printf("internal error");
// break;
// case CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED:
// printf("not supported");
// break;
// case CUSPARSE_STATUS_ALLOC_FAILED:
// printf("alloc failed");
// break;
// case CUSPARSE_STATUS_MAPPING_ERROR :
// printf("map error");
// break;
// case CUSPARSE_STATUS_SUCCESS:
// printf("success\n");
// break;
// default:
// printf("unknown status\n");
// break;
// }
// switch(status){
// case CUSPARSE_STATUS_INVALID_VALUE:
// printf("invalid value");
// break;
// case CUSPARSE_STATUS_NOT_INITIALIZED:
// printf("not initialized");
// break;
// case CUSPARSE_STATUS_ARCH_MISMATCH:
// printf("arch mismatch");
// break;
// case CUSPARSE_STATUS_EXECUTION_FAILED:
// printf("exe failed");
// break;
// case CUSPARSE_STATUS_INTERNAL_ERROR:
// printf("internal error");
// break;
// case CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED:
// printf("not supported");
// break;
// case CUSPARSE_STATUS_ALLOC_FAILED:
// printf("alloc failed");
// break;
// case CUSPARSE_STATUS_MAPPING_ERROR :
// printf("map error");
// break;
// case CUSPARSE_STATUS_SUCCESS:
// printf("success\n");
// break;
// default:
// printf("unknown status\n");
// break;
// }
//}

2
src/translator/nth_element.cpp
View File

@ -107,6 +107,8 @@ GetNBestListFn createGetNBestListFn(size_t beamSize, size_t dimBatch, DeviceId d
#ifdef CUDA_FOUND
if(deviceId.type == DeviceType::gpu)
return createGetNBestListGPUFn(beamSize, dimBatch, deviceId);
#else
deviceId; // (unused)
#endif
auto nth = New<NthElementCPU>(beamSize, dimBatch);
return [nth](const std::vector<size_t>& beamSizes,

4
vs/Marian.vcxproj.filters
View File

@ -1045,7 +1045,6 @@
<ClInclude Include="..\src\command\marian_decoder.cpp">
<Filter>command</Filter>
</ClInclude>
<ClInclude Include="..\src\command\marian_conv.cpp" />
<ClInclude Include="..\src\3rd_party\ExceptionWithCallStack.h">
<Filter>3rd_party</Filter>
</ClInclude>
@ -1073,6 +1072,9 @@
<ClInclude Include="..\src\microsoft\quicksand.h">
<Filter>microsoft</Filter>
</ClInclude>
<ClInclude Include="..\src\command\marian_conv.cpp">
<Filter>command</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Filter Include="3rd_party">