From 6dbe5bb1d093dd3eb61daf7aa53a493c1ecace0b Mon Sep 17 00:00:00 2001 From: Andre Martins Date: Tue, 13 Sep 2016 16:06:53 +0100 Subject: [PATCH 1/3] Fixed bug affecting softmax (was always returning a scalar output). --- src/expression_operators.h | 17 +++++++++++- src/graph_operators.h | 12 ++++++-- src/test.cu | 57 +++++++++++++++++++++++++------------- 3 files changed, 62 insertions(+), 24 deletions(-) diff --git a/src/expression_operators.h b/src/expression_operators.h index 3c3dc031..8eabbd04 100644 --- a/src/expression_operators.h +++ b/src/expression_operators.h @@ -134,6 +134,7 @@ inline Expr sum(Expr a, Args ...args) { else if(ax == 1) { auto lshape = [n]() -> Shape { int cols = n->val().shape()[1]; + //std::cerr << "Shape will be " << cols << " by 1." << std::endl; return {cols, 1}; }; Expr one = ones(shape={n->shape()[1], 1}, @@ -153,6 +154,20 @@ inline Expr sum(Expr a, Args ...args) { template inline Expr softmax(Expr a, Args ...args) { Expr e = exp(a); +#if 0 + ChainPtr n = a.node(); + auto print_shape = [n]() -> Shape { + std::cerr << "Shape: "; + for (auto val : n->val().shape()) { + std::cerr << val << " "; + } + std::cerr << std::endl; + return {1,1}; + }; + using namespace keywords; + Expr one = ones(shape={1, 1}, lazy_shape=print_shape); +#endif + return e / sum(e, args...); } @@ -187,4 +202,4 @@ inline Expr mean(Expr a, Args ...args) { } } -} \ No newline at end of file +} diff --git a/src/graph_operators.h b/src/graph_operators.h index d07c4b38..30456153 100644 --- a/src/graph_operators.h +++ b/src/graph_operators.h @@ -118,9 +118,15 @@ struct LogNodeOp : public UnaryNodeOp { struct ExpNodeOp : public UnaryNodeOp { template - ExpNodeOp(Args ...args) - : UnaryNodeOp(args...) { } + ExpNodeOp(ChainPtr a, Args ...args) + : UnaryNodeOp(a, keywords::shape=newShape(a), + args...) { } + Shape newShape(ChainPtr a) { + Shape shape = a->shape(); + return shape; + } + void forward() { Element(_1 = Exp(_2), val_, a_->val()); } @@ -289,4 +295,4 @@ struct DivNodeOp : public BroadcastingNodeOp { } }; -} \ No newline at end of file +} diff --git a/src/test.cu b/src/test.cu index 56156fee..48b996f8 100644 --- a/src/test.cu +++ b/src/test.cu @@ -12,7 +12,8 @@ int main(int argc, char** argv) { auto w = param(shape={784, 10}, name="W0"); auto b = param(shape={1, 10}, name="b0"); - auto lr = softmax(dot(x, w) + b, axis=1, name="pred"); + auto scores = dot(x, w) + b; + auto lr = softmax(scores, axis=1, name="pred"); auto graph = -mean(sum(y * log(lr), axis=1), axis=0, name="cost"); Tensor tx({500, 784}, 1); @@ -22,28 +23,44 @@ int main(int argc, char** argv) { y = ty; graph.forward(500); + std::cerr << "Result: "; + for (auto val : scores.val().shape()) { + std::cerr << val << " "; + } + std::cerr << std::endl; + std::cerr << "Result: "; + for (auto val : lr.val().shape()) { + std::cerr << val << " "; + } + std::cerr << std::endl; + std::cerr << "Log-likelihood: "; + for (auto val : graph.val().shape()) { + std::cerr << val << " "; + } + std::cerr << std::endl; + //std::cerr << graph["pred"].val()[0] << std::endl; - - //hook0(graph); - //graph.autodiff(); - //std::cerr << graph["cost"].val()[0] << std::endl; +#if 0 + hook0(graph); + graph.autodiff(); + std::cerr << graph["cost"].val()[0] << std::endl; //hook1(graph); - //for(auto p : graph.params()) { - // auto update = _1 = _1 - alpha * _2; - // Element(update, p.val(), p.grad()); - //} - //hook2(graph); - // - //auto opt = adadelta(cost_function=cost, - // eta=0.9, gamma=0.1, - // set_batch=set, - // before_update=before, - // after_update=after, - // set_valid=valid, - // validation_freq=100, - // verbose=1, epochs=3, early_stopping=10); - //opt.run(); + for(auto p : graph.params()) { + auto update = _1 = _1 - alpha * _2; + Element(update, p.val(), p.grad()); + } + hook2(graph); + auto opt = adadelta(cost_function=cost, + eta=0.9, gamma=0.1, + set_batch=set, + before_update=before, + after_update=after, + set_valid=valid, + validation_freq=100, + verbose=1, epochs=3, early_stopping=10); + opt.run(); +#endif return 0; } \ No newline at end of file From cc7a48310f19423f269f19330c32090b866d3c90 Mon Sep 17 00:00:00 2001 From: romang Date: Tue, 13 Sep 2016 18:08:45 +0200 Subject: [PATCH 2/3] add functions loading MNIST dataset --- .gitignore | 1 + examples/mnist/Makefile | 7 ++- src/mnist.h | 94 +++++++++++++++++++++++++++++++++++++++++ src/test.cu | 4 ++ 4 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 src/mnist.h diff --git a/.gitignore b/.gitignore index 4dfd397b..53468680 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ build # Examples examples/*/*.gz +examples/mnist/*ubyte diff --git a/examples/mnist/Makefile b/examples/mnist/Makefile index 7e4e812f..26f65554 100644 --- a/examples/mnist/Makefile +++ b/examples/mnist/Makefile @@ -2,9 +2,12 @@ all: download -download: train-images-idx3-ubyte.gz train-labels-idx1-ubyte.gz t10k-images-idx3-ubyte.gz t10k-labels-idx3-ubyte.gz +download: train-images-idx3-ubyte train-labels-idx1-ubyte t10k-images-idx3-ubyte t10k-labels-idx1-ubyte -%.gz: +%-ubyte: %-ubyte.gz + gzip -d < $^ > $@ + +%-ubyte.gz: wget http://yann.lecun.com/exdb/mnist/$*.gz -O $@ clean: diff --git a/src/mnist.h b/src/mnist.h new file mode 100644 index 00000000..7727bacc --- /dev/null +++ b/src/mnist.h @@ -0,0 +1,94 @@ +#pragma once + +#include +#include +#include +#include + +namespace datasets { +namespace mnist { + +typedef unsigned char uchar; + +auto reverseInt = [](int i) { + unsigned char c1, c2, c3, c4; + c1 = i & 255, c2 = (i >> 8) & 255, c3 = (i >> 16) & 255, c4 = (i >> 24) & 255; + return ((int)c1 << 24) + ((int)c2 << 16) + ((int)c3 << 8) + c4; +}; + +std::vector> ReadImages(const std::string& full_path) { + std::ifstream file(full_path); + + if (! file.is_open()) + throw std::runtime_error("Cannot open file `" + full_path + "`!"); + + int magic_number = 0, n_rows = 0, n_cols = 0; + + file.read((char *)&magic_number, sizeof(magic_number)); + magic_number = reverseInt(magic_number); + + if (magic_number != 2051) + throw std::runtime_error("Invalid MNIST image file!"); + + int number_of_images = 0; + file.read((char *)&number_of_images, sizeof(number_of_images)), number_of_images = reverseInt(number_of_images); + file.read((char *)&n_rows, sizeof(n_rows)), n_rows = reverseInt(n_rows); + file.read((char *)&n_cols, sizeof(n_cols)), n_cols = reverseInt(n_cols); + + int image_size = n_rows * n_cols; + std::vector> _dataset(number_of_images, std::vector(image_size)); + unsigned char pixel = 0; + + for (int i = 0; i < number_of_images; i++) { + for (int j = 0; j < image_size; j++) { + file.read((char*)&pixel, sizeof(pixel)); + _dataset[i][j] = pixel / 255.0f; + } + } + return _dataset; +} + +std::vector ReadLabels(const std::string& full_path) { + std::ifstream file(full_path); + + if (! file.is_open()) + throw std::runtime_error("Cannot open file `" + full_path + "`!"); + + int magic_number = 0; + file.read((char *)&magic_number, sizeof(magic_number)); + magic_number = reverseInt(magic_number); + + if (magic_number != 2049) + throw std::runtime_error("Invalid MNIST label file!"); + + int number_of_labels = 0; + file.read((char *)&number_of_labels, sizeof(number_of_labels)), number_of_labels = reverseInt(number_of_labels); + + std::vector _dataset(number_of_labels); + for (int i = 0; i < number_of_labels; i++) { + file.read((char*)&_dataset[i], 1); + } + + return _dataset; +} + +} // namespace mnist +} // namespace datasets + + +//int main(int argc, const char *argv[]) { + //auto images = datasets::mnist::ReadImages("t10k-images-idx3-ubyte"); + //auto labels = datasets::mnist::ReadLabels("t10k-labels-idx1-ubyte"); + + //std::cout + //<< "Number of images: " << images.size() << std::endl + //<< "Image size: " << images[0].size() << std::endl; + + //for (int i = 0; i < 3; i++) { + //for (int j = 0; j < images[i].size(); j++) { + //std::cout << images[i][j] << ","; + //} + //std::cout << " label=" << (int)labels[i] << std::endl; + //} + //return 0; +//} diff --git a/src/test.cu b/src/test.cu index 4a2445fd..c2b0d62e 100644 --- a/src/test.cu +++ b/src/test.cu @@ -1,9 +1,13 @@ #include "marian.h" +#include "mnist.h" using namespace std; int main(int argc, char** argv) { + /*auto images = datasets::mnist::ReadImages("../examples/mnist/t10k-images-idx3-ubyte");*/ + /*auto labels = datasets::mnist::ReadLabels("../examples/mnist/t10k-labels-idx1-ubyte");*/ + /*std::cerr << images.size() << " " << images[0].size() << std::endl;*/ using namespace marian; using namespace keywords; From d45f88af6db0fa7fca3877448c810533347ba6f9 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Tue, 13 Sep 2016 18:20:44 +0200 Subject: [PATCH 3/3] fix unpacking data --- examples/mnist/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/mnist/Makefile b/examples/mnist/Makefile index 26f65554..051d5a60 100644 --- a/examples/mnist/Makefile +++ b/examples/mnist/Makefile @@ -8,7 +8,7 @@ download: train-images-idx3-ubyte train-labels-idx1-ubyte t10k-images-idx3-ubyte gzip -d < $^ > $@ %-ubyte.gz: - wget http://yann.lecun.com/exdb/mnist/$*.gz -O $@ + wget http://yann.lecun.com/exdb/mnist/$*-ubyte.gz -O $@ clean: - rm -f *.gz + rm -f *.gz *-ubyte