marian-nmt/marian
1
1
Fork 0
mirror of https://github.com/marian-nmt/marian.git synced 2024-11-04 14:04:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

reluplus node for testing, clean up

Browse Source
This commit is contained in:
Marcin Junczys-Dowmunt 2016-09-19 22:43:12 +02:00
parent 0f64f338cb
commit 8797b5ffd3
6 changed files with 60 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scripts/train_test_model_multi.py
View File

@ -53,9 +53,12 @@ def baseline_model(pixels_count, classes_count):
# model.add(Dropout(0.2, input_shape=(pixels_count,)))
model.add(Dense(2048, input_dim=pixels_count, init='uniform', activation='relu'))
# model.add(Dense(2048, init='uniform', activation='relu'))
model.add(Dropout(0.5))
# model.add(Dropout(0.5))
model.add(Dense(2048, init='uniform', activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(2048, init='uniform', activation='relu'))
model.add(Dense(2048, init='uniform', activation='relu'))
model.add(Dense(2048, init='uniform', activation='relu'))
# model.add(Dropout(0.5))
model.add(Dense(classes_count, init='uniform', activation='softmax'))
opt = Adam(lr=0.0002);

4
src/expression_operators.cu
View File

@ -88,6 +88,10 @@ Expr dot(Expr a, Expr b) {
return Expr(a.graph(), new DotNodeOp(a, b));
}
Expr reluplus(Expr a, Expr b) {
return Expr(a.graph(), new ReLUPlusNodeOp(a, b));
}
Expr cross_entropy(Expr a, Expr b) {
return Expr(a.graph(), new CrossEntropyNodeOp(a, b));
}

3
src/expression_operators.h
View File

@ -53,6 +53,9 @@ Expr operator/(Expr a, Expr b);
Expr dot(Expr a, Expr b);
Expr reluplus(Expr a, Expr b);
/*********************************************************/
// inefficient

5
src/mnist_benchmark.cu
View File

@ -33,7 +33,8 @@ ExpressionGraph build_graph(const std::vector<int>& dims) {
layers.emplace_back(x);
}
else {
layers.emplace_back(relu(dot(layers.back(), weights.back()) + biases.back()));
layers.emplace_back(reluplus(dot(layers.back(), weights.back()), biases.back()));
//layers.emplace_back(relu(dot(layers.back(), weights.back()) + biases.back()));
}
weights.emplace_back(
@ -114,7 +115,7 @@ int main(int argc, char** argv) {
std::vector<float> testLabels = datasets::mnist::ReadLabels("../examples/mnist/t10k-labels-idx1-ubyte", testRows, LABEL_SIZE);
std::cerr << "Done." << std::endl;
ExpressionGraph g = build_graph({IMAGE_SIZE, 2048, 2048, LABEL_SIZE});
ExpressionGraph g = build_graph({IMAGE_SIZE, 2048, 2048, 2048, 2048, 2048, LABEL_SIZE});
std::cout << g.graphviz() << std::endl;
Tensor xt({BATCH_SIZE, IMAGE_SIZE});

30
src/node_operators.h
View File

@ -448,6 +448,34 @@ struct PlusNodeOp : public BinaryNodeOp {
};
struct ReLUPlusNodeOp : public BinaryNodeOp {
template <typename ...Args>
ReLUPlusNodeOp(ChainPtr a, ChainPtr b, Args ...args)
: BinaryNodeOp(a, b, keywords::shape=a->shape(), args...) { }
void forward() {
Element(_1 = ReLU(_2 + _3),
val_, a_->val(), b_->val());
}
void backward() {
Element(_1 += _2 * ReLUback(_3 + _4),
a_->grad(), adj_, a_->val(), b_->val());
Element(_1 += _2 * ReLUback(_3 + _4),
b_->grad(), adj_, a_->val(), b_->val());
}
virtual std::string graphviz() {
std::stringstream ss;
ss << "\"" << this << "\" [shape=\"box\", label=" << label("ReLU<br/>+")
<< ", style=\"filled\", fillcolor=\"yellow\"]" << std::endl;
ss << "\"" << a_ << "\" -> \"" << this << "\"" << std::endl;
ss << "\"" << b_ << "\" -> \"" << this << "\"" << std::endl << std::endl;
return ss.str();
};
};
struct MinusNodeOp : public BinaryNodeOp {
template <typename ...Args>
MinusNodeOp(ChainPtr a, ChainPtr b, Args ...args)
@ -590,7 +618,7 @@ struct CrossEntropyNodeOp : public BinaryNodeOp {
virtual std::string graphviz() {
std::stringstream ss;
ss << "\"" << this << "\" [shape=\"box\", label=" << label("x-ent")
<< ", style=\"filled\", fillcolor=\"yellow\"]" << std::endl;
<< ", style=\"filled\", fillcolor=\"orange\"]" << std::endl;
ss << "\"" << a_ << "\" -> \"" << this << "\"" << std::endl << std::endl;
ss << "\"" << b_ << "\" -> \"" << this << "\"" << std::endl << std::endl;
return ss.str();

150
src/tensor_operators.h
View File

@ -29,140 +29,6 @@ using namespace thrust::placeholders;
#define MAX_THREADS 512
#define MAX_BLOCKS 65535
//template <class Functor>
//__global__ void gElement(Functor functor, Float* out,
// size_t rows, size_t cols) {
// for(int bid = 0; bid < rows; bid += gridDim.x) {
// int j = bid + blockIdx.x;
// if(j < rows) {
// Float* rowOut = out + j * cols;
// for(int tid = 0; tid < cols; tid += blockDim.x) {
// int i = tid + threadIdx.x;
// if(i < cols)
// rowOut[i] = functor(rowOut[i]);;
// }
// }
// }
//}
//
//template <class Functor>
//__global__ void gElement(Functor functor,
// Float* out, const Float* in,
// size_t rows, size_t cols) {
// for(int bid = 0; bid < rows; bid += gridDim.x) {
// int j = bid + blockIdx.x;
// if(j < rows) {
// Float* rowOut = out + j * cols;
// const Float* rowIn = in + j * cols;
//
// for(int tid = 0; tid < cols; tid += blockDim.x) {
// int i = tid + threadIdx.x;
// if(i < cols)
// rowOut[i] = functor(rowOut[i], rowIn[i]);;
// }
// }
// }
//}
//
//template <class Functor>
//__global__ void gElement(Functor functor,
// Float* out, const Float* in1, const Float* in2,
// size_t rows, size_t cols) {
// for(int bid = 0; bid < rows; bid += gridDim.x) {
// int j = bid + blockIdx.x;
// if(j < rows) {
// Float* rowOut = out + j * cols;
// const Float* rowIn1 = in1 + j * cols;
// const Float* rowIn2 = in2 + j * cols;
//
// for(int tid = 0; tid < cols; tid += blockDim.x) {
// int i = tid + threadIdx.x;
// if(i < cols)
// rowOut[i] = functor(rowOut[i], rowIn1[i], rowIn2[i]);
// }
// }
// }
//}
//
//template <class Functor>
//__global__ void gElement(Functor functor,
// Float* out, const Float* in1,
// const Float* in2, const Float* in3,
// size_t rows, size_t cols) {
// for(int bid = 0; bid < rows; bid += gridDim.x) {
// int j = bid + blockIdx.x;
// if(j < rows) {
// Float* rowOut = out + j * cols;
// const Float* rowIn1 = in1 + j * cols;
// const Float* rowIn2 = in2 + j * cols;
// const Float* rowIn3 = in3 + j * cols;
//
// for(int tid = 0; tid < cols; tid += blockDim.x) {
// int i = tid + threadIdx.x;
// if(i < cols)
// rowOut[i] = functor(rowOut[i], rowIn1[i], rowIn2[i], rowIn3[i]);
// }
// }
// }
//}
// @TODO add broadcasting
//template <class Functor>
//void Element(Functor functor, Tensor Out) {
// Float* d_out = Out.data();
// int blocks = std::min(MAX_BLOCKS, (int)Out.shape()[0]);
// int threads = std::min(MAX_THREADS, (int)Out.shape()[1]);
// gElement<<<blocks, threads>>>(functor, d_out,
// Out.shape()[0], Out.shape()[1]);
// cudaStreamSynchronize(0);
//}
//template <class Functor>
//void Element(Functor functor,
// Tensor Out, const Tensor In) {
// Float* d_out = Out.data();
// const Float* d_in = In.data();
//
// int blocks = std::min(MAX_BLOCKS, (int)Out.shape()[0]);
// int threads = std::min(MAX_THREADS, (int)Out.shape()[1]);
// gElement<<<blocks, threads>>>(functor, d_out, d_in,
// Out.shape()[0], Out.shape()[1]);
// cudaStreamSynchronize(0);
//}
//template <class Functor>
//void Element(Functor functor,
// Tensor Out, const Tensor In1, const Tensor In2) {
//
// Float* d_out = Out.data();
// const Float* d_in1 = In1.data();
// const Float* d_in2 = In2.data();
//
// int blocks = std::min(MAX_BLOCKS, (int)Out.shape()[0]);
// int threads = std::min(MAX_THREADS, (int)Out.shape()[1]);
// gElement<<<blocks, threads>>>(functor, d_out, d_in1, d_in2,
// Out.shape()[0], Out.shape()[1]);
// cudaStreamSynchronize(0);
//}
//
//template <class Functor>
//void Element(Functor functor,
// Tensor Out, const Tensor In1,
// const Tensor In2, const Tensor In3) {
//
// Float* d_out = Out.data();
// const Float* d_in1 = In1.data();
// const Float* d_in2 = In2.data();
// const Float* d_in3 = In3.data();
//
// int blocks = std::min(MAX_BLOCKS, (int)Out.shape()[0]);
// int threads = std::min(MAX_THREADS, (int)Out.shape()[1]);
// gElement<<<blocks, threads>>>(functor, d_out, d_in1, d_in2, d_in3,
// Out.shape()[0], Out.shape()[1]);
// cudaStreamSynchronize(0);
//}
class TensorView {
private:
float* data_;
@ -192,6 +58,22 @@ class TensorView {
}
};
//template <class Functor>
//__global__ void gElement(Functor functor) {
// int rows = out.rows();
// int cols = out.cols();
// for(int bid = 0; bid < rows; bid += gridDim.x) {
// int i = bid + blockIdx.x;
// if(i < rows) {
// for(int tid = 0; tid < cols; tid += blockDim.x) {
// int j = tid + threadIdx.x;
// if(j < cols)
// functor(i, j);
// }
// }
// }
//}
template <class Functor>
__global__ void gElement(Functor functor,
TensorView out) {