mirror of
https://github.com/marian-nmt/marian.git
synced 2024-09-11 06:15:56 +03:00
fixed (eliminated) all TABs that had snuck in so far; fixed a few warnings on Windows
This commit is contained in:
parent
07b85c0101
commit
e8443b0875
26
src/3rd_party/catch.hpp
vendored
Normal file → Executable file
26
src/3rd_party/catch.hpp
vendored
Normal file → Executable file
@ -927,8 +927,8 @@ namespace Catch {
|
||||
template<typename T> STATIC_ASSERT_Expression_Too_Complex_Please_Rewrite_As_Binary_Comparison& operator && ( T const& );
|
||||
template<typename T> STATIC_ASSERT_Expression_Too_Complex_Please_Rewrite_As_Binary_Comparison& operator || ( T const& );
|
||||
|
||||
private:
|
||||
DecomposedExpression& operator = (DecomposedExpression const&);
|
||||
private:
|
||||
DecomposedExpression& operator = (DecomposedExpression const&);
|
||||
};
|
||||
|
||||
struct AssertionInfo
|
||||
@ -3490,16 +3490,16 @@ return @ desc; \
|
||||
#include <crtdbg.h>
|
||||
class LeakDetector {
|
||||
public:
|
||||
LeakDetector() {
|
||||
int flag = _CrtSetDbgFlag(_CRTDBG_REPORT_FLAG);
|
||||
flag |= _CRTDBG_LEAK_CHECK_DF;
|
||||
flag |= _CRTDBG_ALLOC_MEM_DF;
|
||||
_CrtSetDbgFlag(flag);
|
||||
_CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG);
|
||||
_CrtSetReportFile(_CRT_WARN, _CRTDBG_FILE_STDERR);
|
||||
// Change this to leaking allocation's number to break there
|
||||
_CrtSetBreakAlloc(-1);
|
||||
}
|
||||
LeakDetector() {
|
||||
int flag = _CrtSetDbgFlag(_CRTDBG_REPORT_FLAG);
|
||||
flag |= _CRTDBG_LEAK_CHECK_DF;
|
||||
flag |= _CRTDBG_ALLOC_MEM_DF;
|
||||
_CrtSetDbgFlag(flag);
|
||||
_CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG);
|
||||
_CrtSetReportFile(_CRT_WARN, _CRTDBG_FILE_STDERR);
|
||||
// Change this to leaking allocation's number to break there
|
||||
_CrtSetBreakAlloc(-1);
|
||||
}
|
||||
};
|
||||
#else
|
||||
class LeakDetector {};
|
||||
@ -11303,7 +11303,7 @@ extern "C" int wmain (int argc, wchar_t * argv[], wchar_t * []) {
|
||||
int main (int argc, char * argv[]) {
|
||||
#endif
|
||||
|
||||
int result = Catch::Session().run( argc, argv );
|
||||
int result = Catch::Session().run(argc, argv);
|
||||
return ( result < 0xff ? result : 0xff );
|
||||
}
|
||||
|
||||
|
2
src/3rd_party/exception.h
vendored
Normal file → Executable file
2
src/3rd_party/exception.h
vendored
Normal file → Executable file
@ -137,7 +137,7 @@ class ErrnoException : public Exception {
|
||||
// file wasn't there, or couldn't be open for some reason
|
||||
class FileOpenException : public Exception {
|
||||
public:
|
||||
FileOpenException() throw() {}
|
||||
FileOpenException() throw() {}
|
||||
~FileOpenException() throw() {}
|
||||
};
|
||||
|
||||
|
18854
src/3rd_party/spdlog/tests/catch.hpp
vendored
Normal file → Executable file
18854
src/3rd_party/spdlog/tests/catch.hpp
vendored
Normal file → Executable file
File diff suppressed because it is too large
Load Diff
8
src/3rd_party/svd/defs_and_types.h
vendored
Normal file → Executable file
8
src/3rd_party/svd/defs_and_types.h
vendored
Normal file → Executable file
@ -35,19 +35,19 @@
|
||||
|
||||
|
||||
typedef struct {
|
||||
float x, y, z;
|
||||
float x, y, z;
|
||||
} fcoords;
|
||||
|
||||
typedef struct {
|
||||
long x, y, z;
|
||||
long x, y, z;
|
||||
} lcoords;
|
||||
|
||||
typedef struct {
|
||||
int x, y, z;
|
||||
int x, y, z;
|
||||
} icoords;
|
||||
|
||||
typedef struct {
|
||||
float min, max;
|
||||
float min, max;
|
||||
} lims;
|
||||
|
||||
|
||||
|
4
src/3rd_party/yaml-cpp/dll.h
vendored
Normal file → Executable file
4
src/3rd_party/yaml-cpp/dll.h
vendored
Normal file → Executable file
@ -20,10 +20,10 @@
|
||||
// manually)
|
||||
#ifdef yaml_cpp_EXPORTS // Building YAML-CPP DLL (definition created by CMake
|
||||
// or defined manually)
|
||||
// #pragma message( "Defining YAML_CPP_API for DLL export" )
|
||||
// #pragma message( "Defining YAML_CPP_API for DLL export" )
|
||||
#define YAML_CPP_API __declspec(dllexport)
|
||||
#else // yaml_cpp_EXPORTS
|
||||
// #pragma message( "Defining YAML_CPP_API for DLL import" )
|
||||
// #pragma message( "Defining YAML_CPP_API for DLL import" )
|
||||
#define YAML_CPP_API __declspec(dllimport)
|
||||
#endif // yaml_cpp_EXPORTS
|
||||
#else // YAML_CPP_DLL
|
||||
|
8
src/3rd_party/yaml-cpp/scanner.cpp
vendored
Normal file → Executable file
8
src/3rd_party/yaml-cpp/scanner.cpp
vendored
Normal file → Executable file
@ -34,10 +34,10 @@ Token& Scanner::peek() {
|
||||
// if it's empty before peeking.
|
||||
|
||||
#if 0
|
||||
static Token *pLast = 0;
|
||||
if(pLast != &m_tokens.front())
|
||||
std::cerr << "peek: " << m_tokens.front() << "\n";
|
||||
pLast = &m_tokens.front();
|
||||
static Token *pLast = 0;
|
||||
if (pLast != &m_tokens.front())
|
||||
std::cerr << "peek: " << m_tokens.front() << "\n";
|
||||
pLast = &m_tokens.front();
|
||||
#endif
|
||||
|
||||
return m_tokens.front();
|
||||
|
2
src/3rd_party/yaml-cpp/scantoken.cpp
vendored
Normal file → Executable file
2
src/3rd_party/yaml-cpp/scantoken.cpp
vendored
Normal file → Executable file
@ -320,7 +320,7 @@ void Scanner::ScanPlainScalar() {
|
||||
|
||||
// finally, check and see if we ended on an illegal character
|
||||
// if(Exp::IllegalCharInScalar.Matches(INPUT))
|
||||
// throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_SCALAR);
|
||||
// throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_SCALAR);
|
||||
|
||||
Token token(Token::PLAIN_SCALAR, mark);
|
||||
token.value = scalar;
|
||||
|
@ -366,30 +366,30 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) {
|
||||
true);
|
||||
// support for universal encoder ULR https://arxiv.org/pdf/1802.05368.pdf
|
||||
cli.add<bool>("--ulr-enabled",
|
||||
"Is ULR (Universal Language Representation) enabled?",
|
||||
false);
|
||||
"Is ULR (Universal Language Representation) enabled?",
|
||||
false);
|
||||
// reading pre-trained universal embedings for multi-sources
|
||||
// note that source and target here is relative to ULR not the translation langs
|
||||
//queries: EQ in Fig2 : is the unified embbedins projected to one space.
|
||||
//"Path to file with universal sources embeddings from projection into universal space")
|
||||
cli.add<bool>("--ulr-query-vectors",
|
||||
"Path to file with universal sources embeddings from projection into universal space",
|
||||
"");
|
||||
"Path to file with universal sources embeddings from projection into universal space",
|
||||
"");
|
||||
//keys: EK in Fig2 : is the keys of the target embbedins projected to unified space (i.e. ENU in multi-lingual case)
|
||||
cli.add<std::string>("--ulr-keys-vectors",
|
||||
"Path to file with universal sources embeddings of traget keys from projection into universal space",
|
||||
"");
|
||||
"Path to file with universal sources embeddings of traget keys from projection into universal space",
|
||||
"");
|
||||
cli.add<bool>("--ulr-trainable-transformation",
|
||||
"Is Query Transformation Matrix A trainable ?",
|
||||
false);
|
||||
"Is Query Transformation Matrix A trainable ?",
|
||||
false);
|
||||
cli.add<int>("--ulr-dim-emb",
|
||||
"ULR mono embed dim");
|
||||
"ULR mono embed dim");
|
||||
cli.add<float>("--ulr-dropout",
|
||||
"ULR dropout on embeddings attentions: default is no dropuout",
|
||||
0.0f);
|
||||
"ULR dropout on embeddings attentions: default is no dropuout",
|
||||
0.0f);
|
||||
cli.add<float>("--ulr-softmax-temperature",
|
||||
"ULR softmax temperature to control randomness of predictions- deafult is 1.0: no temperature ",
|
||||
1.0f);
|
||||
"ULR softmax temperature to control randomness of predictions- deafult is 1.0: no temperature ",
|
||||
1.0f);
|
||||
// clang-format on
|
||||
}
|
||||
|
||||
|
@ -137,13 +137,11 @@ void ConfigValidator::validateDevices(cli::mode mode) const {
|
||||
+ ")' for option '--devices' is invalid. "
|
||||
+ help);
|
||||
// validate ULR options
|
||||
UTIL_THROW_IF2(
|
||||
(get<bool>("ulr-enabled") == true &&
|
||||
(get<std::string>("ulr-query-vectors") == ""
|
||||
|| get<std::string>("ulr-keys-vectors") == "")),
|
||||
"ULR enablign requires query and keys vectors specified with "
|
||||
"--ulr-query-vectors and --ulr-keys-vectors option");
|
||||
|
||||
UTIL_THROW_IF2((get<bool>("ulr-enabled") == true &&
|
||||
(get<std::string>("ulr-query-vectors") == ""
|
||||
|| get<std::string>("ulr-keys-vectors") == "")),
|
||||
"ULR enablign requires query and keys vectors specified with "
|
||||
"--ulr-query-vectors and --ulr-keys-vectors option");
|
||||
}
|
||||
|
||||
} // namespace marian
|
||||
|
@ -173,83 +173,80 @@ struct EmbeddingFactory : public Factory {
|
||||
bool fixed = opt<bool>("fixed", false);
|
||||
|
||||
NodeInitializer initFunc = inits::glorot_uniform;
|
||||
if (options_->has("embFile")) {
|
||||
std::string file = opt<std::string>("embFile");
|
||||
if (!file.empty()) {
|
||||
bool norm = opt<bool>("normalization", false);
|
||||
initFunc = inits::from_word2vec(file, dimVoc, dimEmb, norm);
|
||||
}
|
||||
}
|
||||
|
||||
if (options_->has("embFile")) {
|
||||
std::string file = opt<std::string>("embFile");
|
||||
if (!file.empty()) {
|
||||
bool norm = opt<bool>("normalization", false);
|
||||
initFunc = inits::from_word2vec(file, dimVoc, dimEmb, norm);
|
||||
}
|
||||
}
|
||||
|
||||
return graph_->param(name, {dimVoc, dimEmb}, initFunc, fixed);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
struct ULREmbeddingFactory : public Factory {
|
||||
ULREmbeddingFactory(Ptr<ExpressionGraph> graph) : Factory(graph) {}
|
||||
ULREmbeddingFactory(Ptr<ExpressionGraph> graph) : Factory(graph) {}
|
||||
|
||||
std::vector<Expr> construct() {
|
||||
std::string name = "url_embed"; //opt<std::string>("prefix");
|
||||
int dimKeys = opt<int>("dimTgtVoc");
|
||||
int dimQueries = opt<int>("dimSrcVoc");
|
||||
int dimEmb = opt<int>("dimEmb");
|
||||
int dimUlrEmb = opt<int>("dimUlrEmb"); // ULR mono embed size
|
||||
bool fixed = opt<bool>("fixed", false);
|
||||
std::vector<Expr> ulrEmbeds;
|
||||
NodeInitializer initFunc = inits::glorot_uniform;
|
||||
std::string queryFile = opt<std::string>("ulrQueryFile");
|
||||
std::string keyFile = opt<std::string>("ulrKeysFile");
|
||||
bool norm = opt<bool>("normalization", false);
|
||||
bool trainTrans = opt<bool>("ulrTrainTransform", false);
|
||||
|
||||
if (!queryFile.empty() && !keyFile.empty()) {
|
||||
initFunc = inits::from_word2vec(queryFile, dimQueries, dimUlrEmb, false);
|
||||
name = "ulr_query";
|
||||
fixed = true;
|
||||
auto query_embed = graph_->param(name, { dimQueries, dimUlrEmb }, initFunc, fixed);
|
||||
ulrEmbeds.push_back(query_embed);
|
||||
// keys embeds
|
||||
initFunc = inits::from_word2vec(keyFile, dimKeys, dimUlrEmb, false);
|
||||
name = "ulr_keys";
|
||||
fixed = true;
|
||||
auto key_embed = graph_->param(name, { dimKeys, dimUlrEmb }, initFunc, fixed);
|
||||
ulrEmbeds.push_back(key_embed);
|
||||
// actual trainable embedding
|
||||
initFunc = inits::glorot_uniform;
|
||||
name = "ulr_embed";
|
||||
fixed = false;
|
||||
auto ulr_embed = graph_->param(name, {dimKeys , dimEmb }, initFunc, fixed); // note the reverse dim
|
||||
ulrEmbeds.push_back(ulr_embed);
|
||||
// init trainable src embedding
|
||||
name = "ulr_src_embed";
|
||||
auto ulr_src_embed = graph_->param(name, { dimQueries, dimEmb }, initFunc, fixed);
|
||||
ulrEmbeds.push_back(ulr_src_embed);
|
||||
// ulr transformation matrix
|
||||
//initFunc = inits::diag(1.f); // identity matrix - is it ok to init wiht identity or shall we make this to the fixed case only
|
||||
if (trainTrans) {
|
||||
initFunc = inits::glorot_uniform;
|
||||
fixed = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
initFunc = inits::diag(1.f); // identity matrix
|
||||
fixed = true;
|
||||
}
|
||||
name = "ulr_transform";
|
||||
auto ulr_transform = graph_->param(name, { dimUlrEmb, dimUlrEmb }, initFunc, fixed);
|
||||
ulrEmbeds.push_back(ulr_transform);
|
||||
|
||||
initFunc = inits::from_value(1.f); // TBD: we should read sharable flags here - 1 means all sharable - 0 means no universal embeddings - should be zero for top freq only
|
||||
fixed = true;
|
||||
name = "ulr_shared";
|
||||
auto share_embed = graph_->param(name, { dimQueries, 1 }, initFunc, fixed);
|
||||
ulrEmbeds.push_back(share_embed);
|
||||
std::vector<Expr> construct() {
|
||||
std::string name = "url_embed"; //opt<std::string>("prefix");
|
||||
int dimKeys = opt<int>("dimTgtVoc");
|
||||
int dimQueries = opt<int>("dimSrcVoc");
|
||||
int dimEmb = opt<int>("dimEmb");
|
||||
int dimUlrEmb = opt<int>("dimUlrEmb"); // ULR mono embed size
|
||||
bool fixed = opt<bool>("fixed", false);
|
||||
std::vector<Expr> ulrEmbeds;
|
||||
NodeInitializer initFunc = inits::glorot_uniform;
|
||||
std::string queryFile = opt<std::string>("ulrQueryFile");
|
||||
std::string keyFile = opt<std::string>("ulrKeysFile");
|
||||
bool trainTrans = opt<bool>("ulrTrainTransform", false);
|
||||
|
||||
}
|
||||
|
||||
return ulrEmbeds;
|
||||
}
|
||||
if (!queryFile.empty() && !keyFile.empty()) {
|
||||
initFunc = inits::from_word2vec(queryFile, dimQueries, dimUlrEmb, false);
|
||||
name = "ulr_query";
|
||||
fixed = true;
|
||||
auto query_embed = graph_->param(name, { dimQueries, dimUlrEmb }, initFunc, fixed);
|
||||
ulrEmbeds.push_back(query_embed);
|
||||
// keys embeds
|
||||
initFunc = inits::from_word2vec(keyFile, dimKeys, dimUlrEmb, false);
|
||||
name = "ulr_keys";
|
||||
fixed = true;
|
||||
auto key_embed = graph_->param(name, { dimKeys, dimUlrEmb }, initFunc, fixed);
|
||||
ulrEmbeds.push_back(key_embed);
|
||||
// actual trainable embedding
|
||||
initFunc = inits::glorot_uniform;
|
||||
name = "ulr_embed";
|
||||
fixed = false;
|
||||
auto ulr_embed = graph_->param(name, {dimKeys , dimEmb }, initFunc, fixed); // note the reverse dim
|
||||
ulrEmbeds.push_back(ulr_embed);
|
||||
// init trainable src embedding
|
||||
name = "ulr_src_embed";
|
||||
auto ulr_src_embed = graph_->param(name, { dimQueries, dimEmb }, initFunc, fixed);
|
||||
ulrEmbeds.push_back(ulr_src_embed);
|
||||
// ulr transformation matrix
|
||||
//initFunc = inits::diag(1.f); // identity matrix - is it ok to init wiht identity or shall we make this to the fixed case only
|
||||
if (trainTrans) {
|
||||
initFunc = inits::glorot_uniform;
|
||||
fixed = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
initFunc = inits::diag(1.f); // identity matrix
|
||||
fixed = true;
|
||||
}
|
||||
name = "ulr_transform";
|
||||
auto ulr_transform = graph_->param(name, { dimUlrEmb, dimUlrEmb }, initFunc, fixed);
|
||||
ulrEmbeds.push_back(ulr_transform);
|
||||
|
||||
initFunc = inits::from_value(1.f); // TBD: we should read sharable flags here - 1 means all sharable - 0 means no universal embeddings - should be zero for top freq only
|
||||
fixed = true;
|
||||
name = "ulr_shared";
|
||||
auto share_embed = graph_->param(name, { dimQueries, 1 }, initFunc, fixed);
|
||||
ulrEmbeds.push_back(share_embed);
|
||||
}
|
||||
return ulrEmbeds;
|
||||
}
|
||||
};
|
||||
|
||||
typedef Accumulator<EmbeddingFactory> embedding;
|
||||
|
@ -16,62 +16,60 @@ protected:
|
||||
// virtual
|
||||
std::tuple<Expr, Expr> lookup(Ptr<ExpressionGraph> graph,
|
||||
Expr srcEmbeddings,
|
||||
Ptr<data::CorpusBatch> batch) const {
|
||||
auto subBatch = (*batch)[batchIndex_];
|
||||
int dimBatch = (int)subBatch->batchSize();
|
||||
int dimEmb = srcEmbeddings->shape()[-1];
|
||||
int dimWords = (int)subBatch->batchWidth();
|
||||
auto chosenEmbeddings = rows(srcEmbeddings, subBatch->data());
|
||||
auto batchEmbeddings
|
||||
= reshape(chosenEmbeddings, {dimWords, dimBatch, dimEmb});
|
||||
auto batchMask = graph->constant({dimWords, dimBatch, 1},
|
||||
inits::from_vector(subBatch->mask()));
|
||||
Ptr<data::CorpusBatch> batch) const {
|
||||
auto subBatch = (*batch)[batchIndex_];
|
||||
int dimBatch = (int)subBatch->batchSize();
|
||||
int dimEmb = srcEmbeddings->shape()[-1];
|
||||
int dimWords = (int)subBatch->batchWidth();
|
||||
auto chosenEmbeddings = rows(srcEmbeddings, subBatch->data());
|
||||
auto batchEmbeddings = reshape(chosenEmbeddings, { dimWords, dimBatch, dimEmb });
|
||||
auto batchMask = graph->constant({ dimWords, dimBatch, 1 },
|
||||
inits::from_vector(subBatch->mask()));
|
||||
|
||||
return std::make_tuple(batchEmbeddings, batchMask);
|
||||
return std::make_tuple(batchEmbeddings, batchMask);
|
||||
}
|
||||
|
||||
std::tuple<Expr, Expr> ulrLookup(Ptr<ExpressionGraph> graph,
|
||||
std::vector<Expr> urlEmbeddings,
|
||||
Ptr<data::CorpusBatch> batch) const {
|
||||
auto subBatch = (*batch)[batchIndex_];
|
||||
// is their a better way to do this?
|
||||
assert(urlEmbeddings.size() == 6);
|
||||
auto queryEmbed = urlEmbeddings[0]; //Q : dimQueries*dimUlrEmb
|
||||
auto keyEmbed = urlEmbeddings[1]; // K : dimKeys*dimUlrEmb
|
||||
auto uniEmbed = urlEmbeddings[2]; // E : dimQueries*dimEmb
|
||||
auto srcEmbed = urlEmbeddings[3]; // I : dimQueries*dimEmb
|
||||
auto ulrTransform = urlEmbeddings[4]; //A : dimUlrEmb *dimUlrEmb
|
||||
auto ulrSharable = urlEmbeddings[5]; //alpha : dimQueries*1
|
||||
int dimBatch = (int)subBatch->batchSize();
|
||||
int dimEmb = uniEmbed->shape()[-1];
|
||||
int dimWords = (int)subBatch->batchWidth();
|
||||
// D = K.A.QT
|
||||
// dimm(K) = univ_tok_vocab*uni_embed_size
|
||||
// dim A = uni_embed_size*uni_embed_size
|
||||
// dim Q: uni_embed_size * total_merged_vocab_size
|
||||
// dim D = univ_tok_vocab * total_merged_vocab_size
|
||||
// note all above can be precombuted and serialized if A is not trainiabale and during decoding (TBD)
|
||||
// here we need to handle the mini-batch
|
||||
// extract raws corresponding to Xs in this mini batch from Q
|
||||
auto queryEmbeddings = rows(queryEmbed, subBatch->data());
|
||||
auto srcEmbeddings = rows(srcEmbed, subBatch->data()); // extract trainable src embeddings
|
||||
auto alpha = rows(ulrSharable, subBatch->data()); // extract sharable flags
|
||||
auto qt = dot(queryEmbeddings, ulrTransform,false, false); //A: transform embeddings based on similarity A : dimUlrEmb *dimUlrEmb
|
||||
auto z = dot(qt, keyEmbed, false, true); // query-key similarity
|
||||
std::vector<Expr> urlEmbeddings,
|
||||
Ptr<data::CorpusBatch> batch) const {
|
||||
auto subBatch = (*batch)[batchIndex_];
|
||||
// is their a better way to do this?
|
||||
assert(urlEmbeddings.size() == 6);
|
||||
auto queryEmbed = urlEmbeddings[0]; //Q : dimQueries*dimUlrEmb
|
||||
auto keyEmbed = urlEmbeddings[1]; // K : dimKeys*dimUlrEmb
|
||||
auto uniEmbed = urlEmbeddings[2]; // E : dimQueries*dimEmb
|
||||
auto srcEmbed = urlEmbeddings[3]; // I : dimQueries*dimEmb
|
||||
auto ulrTransform = urlEmbeddings[4]; //A : dimUlrEmb *dimUlrEmb
|
||||
auto ulrSharable = urlEmbeddings[5]; //alpha : dimQueries*1
|
||||
int dimBatch = (int)subBatch->batchSize();
|
||||
int dimEmb = uniEmbed->shape()[-1];
|
||||
int dimWords = (int)subBatch->batchWidth();
|
||||
// D = K.A.QT
|
||||
// dimm(K) = univ_tok_vocab*uni_embed_size
|
||||
// dim A = uni_embed_size*uni_embed_size
|
||||
// dim Q: uni_embed_size * total_merged_vocab_size
|
||||
// dim D = univ_tok_vocab * total_merged_vocab_size
|
||||
// note all above can be precombuted and serialized if A is not trainiabale and during decoding (TBD)
|
||||
// here we need to handle the mini-batch
|
||||
// extract raws corresponding to Xs in this mini batch from Q
|
||||
auto queryEmbeddings = rows(queryEmbed, subBatch->data());
|
||||
auto srcEmbeddings = rows(srcEmbed, subBatch->data()); // extract trainable src embeddings
|
||||
auto alpha = rows(ulrSharable, subBatch->data()); // extract sharable flags
|
||||
auto qt = dot(queryEmbeddings, ulrTransform, false, false); //A: transform embeddings based on similarity A : dimUlrEmb *dimUlrEmb
|
||||
auto z = dot(qt, keyEmbed, false, true); // query-key similarity
|
||||
float dropProb = this->options_->get<float>("ulr-dropout", 0.0f); // default no dropout
|
||||
z = dropout(z, dropProb);
|
||||
float tau = this->options_->get<float>("ulr-softmax-temperature", 1.0f); // default no temperature
|
||||
// temperature in softmax is to control randomness of predictions
|
||||
// high temperature Softmax outputs are more close to each other
|
||||
// low temperatures the softmax become more similar to "hardmax"
|
||||
auto weights = softmax(z/tau); // assume default is dim=-1, what about temprature? - scaler ??
|
||||
auto chosenEmbeddings = dot(weights, uniEmbed); // THIS IS WRONG - IT SHOULD BE AVERAGE
|
||||
auto chosenEmbeddings_mix = srcEmbeddings + alpha * chosenEmbeddings; // this should be elementwise broadcast
|
||||
auto batchEmbeddings
|
||||
= reshape(chosenEmbeddings_mix, { dimWords, dimBatch, dimEmb });
|
||||
auto batchMask = graph->constant({ dimWords, dimBatch, 1 },
|
||||
inits::from_vector(subBatch->mask()));
|
||||
return std::make_tuple(batchEmbeddings, batchMask);
|
||||
z = dropout(z, dropProb);
|
||||
float tau = this->options_->get<float>("ulr-softmax-temperature", 1.0f); // default no temperature
|
||||
// temperature in softmax is to control randomness of predictions
|
||||
// high temperature Softmax outputs are more close to each other
|
||||
// low temperatures the softmax become more similar to "hardmax"
|
||||
auto weights = softmax(z / tau); // assume default is dim=-1, what about temprature? - scaler ??
|
||||
auto chosenEmbeddings = dot(weights, uniEmbed); // THIS IS WRONG - IT SHOULD BE AVERAGE
|
||||
auto chosenEmbeddings_mix = srcEmbeddings + alpha * chosenEmbeddings; // this should be elementwise broadcast
|
||||
auto batchEmbeddings = reshape(chosenEmbeddings_mix, { dimWords, dimBatch, dimEmb });
|
||||
auto batchMask = graph->constant({ dimWords, dimBatch, 1 },
|
||||
inits::from_vector(subBatch->mask()));
|
||||
return std::make_tuple(batchEmbeddings, batchMask);
|
||||
}
|
||||
public:
|
||||
EncoderBase(Ptr<Options> options)
|
||||
|
@ -481,38 +481,37 @@ public:
|
||||
// returns the embedding matrix based on options
|
||||
// and based on batchIndex_.
|
||||
|
||||
std::vector<Expr> ULREmbeddings(size_t subBatchIndex) const {
|
||||
// standard encoder word embeddings
|
||||
int dimSrcVoc = opt<std::vector<int>>("dim-vocabs")[0]; //ULR multi-lingual src
|
||||
int dimTgtVoc = opt<std::vector<int>>("dim-vocabs")[1]; //ULR monon tgt
|
||||
int dimEmb = opt<int>("dim-emb");
|
||||
int dimUlrEmb = opt<int>("ulr-dim-emb");
|
||||
auto embFactory = ulr_embedding(graph_)("dimSrcVoc", dimSrcVoc)("dimTgtVoc", dimTgtVoc)
|
||||
("dimUlrEmb", dimUlrEmb)("dimEmb", dimEmb)
|
||||
("ulrTrainTransform", opt<bool>("ulr-trainable-transformation"))
|
||||
("ulrQueryFile", opt<std::string>("ulr-query-vectors"))
|
||||
("ulrKeysFile" , opt<std::string>("ulr-keys-vectors"));
|
||||
return embFactory.construct();
|
||||
std::vector<Expr> ULREmbeddings() const {
|
||||
// standard encoder word embeddings
|
||||
int dimSrcVoc = opt<std::vector<int>>("dim-vocabs")[0]; //ULR multi-lingual src
|
||||
int dimTgtVoc = opt<std::vector<int>>("dim-vocabs")[1]; //ULR monon tgt
|
||||
int dimEmb = opt<int>("dim-emb");
|
||||
int dimUlrEmb = opt<int>("ulr-dim-emb");
|
||||
auto embFactory = ulr_embedding(graph_)("dimSrcVoc", dimSrcVoc)("dimTgtVoc", dimTgtVoc)
|
||||
("dimUlrEmb", dimUlrEmb)("dimEmb", dimEmb)
|
||||
("ulrTrainTransform", opt<bool>("ulr-trainable-transformation"))
|
||||
("ulrQueryFile", opt<std::string>("ulr-query-vectors"))
|
||||
("ulrKeysFile", opt<std::string>("ulr-keys-vectors"));
|
||||
return embFactory.construct();
|
||||
}
|
||||
|
||||
Expr wordEmbeddings(size_t subBatchIndex) const {
|
||||
// standard encoder word embeddings
|
||||
int dimVoc = opt<std::vector<int>>("dim-vocabs")[subBatchIndex];
|
||||
int dimEmb = opt<int>("dim-emb");
|
||||
auto embFactory = embedding(graph_)("dimVocab", dimVoc)("dimEmb", dimEmb);
|
||||
if(opt<bool>("tied-embeddings-src") || opt<bool>("tied-embeddings-all"))
|
||||
embFactory("prefix", "Wemb");
|
||||
else
|
||||
embFactory("prefix", prefix_ + "_Wemb");
|
||||
if(options_->has("embedding-fix-src"))
|
||||
embFactory("fixed", opt<bool>("embedding-fix-src"));
|
||||
if(options_->has("embedding-vectors")) {
|
||||
auto embFiles = opt<std::vector<std::string>>("embedding-vectors");
|
||||
embFactory
|
||||
("embFile", embFiles[subBatchIndex])
|
||||
("normalization", opt<bool>("embedding-normalization"));
|
||||
}
|
||||
return embFactory.construct();
|
||||
// standard encoder word embeddings
|
||||
int dimVoc = opt<std::vector<int>>("dim-vocabs")[subBatchIndex];
|
||||
int dimEmb = opt<int>("dim-emb");
|
||||
auto embFactory = embedding(graph_)("dimVocab", dimVoc)("dimEmb", dimEmb);
|
||||
if (opt<bool>("tied-embeddings-src") || opt<bool>("tied-embeddings-all"))
|
||||
embFactory("prefix", "Wemb");
|
||||
else
|
||||
embFactory("prefix", prefix_ + "_Wemb");
|
||||
if (options_->has("embedding-fix-src"))
|
||||
embFactory("fixed", opt<bool>("embedding-fix-src"));
|
||||
if (options_->has("embedding-vectors")) {
|
||||
auto embFiles = opt<std::vector<std::string>>("embedding-vectors");
|
||||
embFactory("embFile", embFiles[subBatchIndex])
|
||||
("normalization", opt<bool>("embedding-normalization"));
|
||||
}
|
||||
return embFactory.construct();
|
||||
}
|
||||
|
||||
Ptr<EncoderState> build(Ptr<ExpressionGraph> graph,
|
||||
@ -525,20 +524,20 @@ public:
|
||||
int dimEmb = opt<int>("dim-emb");
|
||||
int dimBatch = (int)batch->size();
|
||||
int dimSrcWords = (int)(*batch)[batchIndex_]->batchWidth();
|
||||
Expr batchEmbeddings, batchMask;
|
||||
if (options_->has("ulr-enabled") && options_->get<bool>("ulr-enabled") == true) {
|
||||
auto embeddings = ULREmbeddings(batchIndex_); // embedding matrix, considering tying and some other options
|
||||
// embed the source words in the batch using ULR
|
||||
std::tie(batchEmbeddings, batchMask)
|
||||
= EncoderBase::ulrLookup(graph_, embeddings, batch);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto embeddings = wordEmbeddings(batchIndex_); // embedding matrix, considering tying and some other options
|
||||
// embed the source words in the batch
|
||||
std::tie(batchEmbeddings, batchMask)
|
||||
= EncoderBase::lookup(graph_, embeddings, batch);
|
||||
}
|
||||
// create the embedding matrix, considering tying and some other options
|
||||
// embed the source words in the batch
|
||||
Expr batchEmbeddings, batchMask;
|
||||
if (options_->has("ulr-enabled") && options_->get<bool>("ulr-enabled") == true) {
|
||||
auto embeddings = ULREmbeddings(); // embedding uses ULR
|
||||
std::tie(batchEmbeddings, batchMask)
|
||||
= EncoderBase::ulrLookup(graph_, embeddings, batch);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto embeddings = wordEmbeddings(batchIndex_);
|
||||
std::tie(batchEmbeddings, batchMask)
|
||||
= EncoderBase::lookup(graph_, embeddings, batch);
|
||||
}
|
||||
// apply dropout over source words
|
||||
float dropoutSrc = inference_ ? 0 : opt<float>("dropout-src");
|
||||
if(dropoutSrc) {
|
||||
|
@ -47,36 +47,36 @@ void CudaCopy(const T* start, const T* end, T* dest) {
|
||||
}
|
||||
|
||||
// void cusparseStatus(cusparseStatus_t status){
|
||||
// switch(status){
|
||||
// case CUSPARSE_STATUS_INVALID_VALUE:
|
||||
// printf("invalid value");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_NOT_INITIALIZED:
|
||||
// printf("not initialized");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_ARCH_MISMATCH:
|
||||
// printf("arch mismatch");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_EXECUTION_FAILED:
|
||||
// printf("exe failed");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_INTERNAL_ERROR:
|
||||
// printf("internal error");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED:
|
||||
// printf("not supported");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_ALLOC_FAILED:
|
||||
// printf("alloc failed");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_MAPPING_ERROR :
|
||||
// printf("map error");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_SUCCESS:
|
||||
// printf("success\n");
|
||||
// break;
|
||||
// default:
|
||||
// printf("unknown status\n");
|
||||
// break;
|
||||
// }
|
||||
// switch(status){
|
||||
// case CUSPARSE_STATUS_INVALID_VALUE:
|
||||
// printf("invalid value");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_NOT_INITIALIZED:
|
||||
// printf("not initialized");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_ARCH_MISMATCH:
|
||||
// printf("arch mismatch");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_EXECUTION_FAILED:
|
||||
// printf("exe failed");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_INTERNAL_ERROR:
|
||||
// printf("internal error");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED:
|
||||
// printf("not supported");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_ALLOC_FAILED:
|
||||
// printf("alloc failed");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_MAPPING_ERROR :
|
||||
// printf("map error");
|
||||
// break;
|
||||
// case CUSPARSE_STATUS_SUCCESS:
|
||||
// printf("success\n");
|
||||
// break;
|
||||
// default:
|
||||
// printf("unknown status\n");
|
||||
// break;
|
||||
// }
|
||||
//}
|
||||
|
@ -107,6 +107,8 @@ GetNBestListFn createGetNBestListFn(size_t beamSize, size_t dimBatch, DeviceId d
|
||||
#ifdef CUDA_FOUND
|
||||
if(deviceId.type == DeviceType::gpu)
|
||||
return createGetNBestListGPUFn(beamSize, dimBatch, deviceId);
|
||||
#else
|
||||
deviceId; // (unused)
|
||||
#endif
|
||||
auto nth = New<NthElementCPU>(beamSize, dimBatch);
|
||||
return [nth](const std::vector<size_t>& beamSizes,
|
||||
|
@ -1045,7 +1045,6 @@
|
||||
<ClInclude Include="..\src\command\marian_decoder.cpp">
|
||||
<Filter>command</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\src\command\marian_conv.cpp" />
|
||||
<ClInclude Include="..\src\3rd_party\ExceptionWithCallStack.h">
|
||||
<Filter>3rd_party</Filter>
|
||||
</ClInclude>
|
||||
@ -1073,6 +1072,9 @@
|
||||
<ClInclude Include="..\src\microsoft\quicksand.h">
|
||||
<Filter>microsoft</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\src\command\marian_conv.cpp">
|
||||
<Filter>command</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Filter Include="3rd_party">
|
||||
|
Loading…
Reference in New Issue
Block a user