From 149e789cc34a20086849c4e2b609a021437a0af5 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Wed, 14 Sep 2016 13:13:32 +0100 Subject: [PATCH 1/5] debug --- src/test.cu | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/src/test.cu b/src/test.cu index 90dffa04..6a4ef4bf 100644 --- a/src/test.cu +++ b/src/test.cu @@ -46,21 +46,9 @@ int main(int argc, char** argv) { graph.forward(500); - std::cerr << "Result: "; - for (auto val : scores.val().shape()) { - std::cerr << val << " "; - } - std::cerr << std::endl; - std::cerr << "Result: "; - for (auto val : lr.val().shape()) { - std::cerr << val << " "; - } - std::cerr << std::endl; - std::cerr << "Log-likelihood: "; - for (auto val : graph.val().shape()) { - std::cerr << val << " "; - } - std::cerr << std::endl; + std::cerr << "scores: " << Debug(scores.val().shape()) << endl; + std::cerr << "lr: " << Debug(lr.val().shape()) << endl; + std::cerr << "Log-likelihood: " << Debug(graph.val().shape()) << endl ; graph.backward(); From 2c39bad1d603af02bc2dc74647a60fcb2457a2f3 Mon Sep 17 00:00:00 2001 From: Maximiliana Behnke Date: Wed, 14 Sep 2016 14:27:08 +0200 Subject: [PATCH 2/5] Add test model training script --- scripts/train_test_model.py | 91 +++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100755 scripts/train_test_model.py diff --git a/scripts/train_test_model.py b/scripts/train_test_model.py new file mode 100755 index 00000000..4f3236a9 --- /dev/null +++ b/scripts/train_test_model.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python + +import sys +import os +import numpy as np +from keras.datasets import mnist +from keras.utils import np_utils +from keras.models import Sequential +from keras.layers import Dense +from keras.layers import Dropout + +def softmax(x): + return np.exp(x) / np.sum(np.exp(x), axis=1)[:, None] + + +def baseline_model(pixels_count, classes_count): + model = Sequential() + # model.add(Dense(pixels_count, input_dim=pixels_count, init='normal', activation='relu')) + model.add(Dense(classes_count, input_dim=pixels_count, init='normal', activation='softmax')) + model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) + return model + + +if __name__ == "__main__": + ### Load trainset from mnist + + (X_train, y_train), (X_test, y_test) = mnist.load_data() + + ### Flatten pictures into vectors + + pixels_count = X_train.shape[1] * X_train.shape[2] + X_train = X_train.reshape(X_train.shape[0], pixels_count).astype('float32') + print "X shape: ", X_train.shape + + X_test = X_test.reshape(X_test.shape[0], pixels_count).astype('float32') + + ### Normalize data to (0, 1) + + X_train = X_train / 255 + X_test = X_test / 255 + + ### Change classes to one hot encoding matrixes + + y_train = np_utils.to_categorical(y_train) + classes_count = y_train.shape[1] + print "Y shape: ", y_train.shape + + y_test = np_utils.to_categorical(y_test) + + # Train weight matrix + + # Build the model + model = baseline_model(pixels_count, classes_count) + # Fit the model + model.fit(X_train, y_train, validation_data=(X_test, y_test), nb_epoch=10, batch_size=200, verbose=2) + # Final evaluation of the model + scores = model.evaluate(X_test, y_test, verbose=0) + print("Baseline Error: %.2f%%" % (100-scores[1]*100)) + + ### Weight and bias matrixes - we extract them from the model + + # weights_ones = np.ones((pixels_count, classes_count)) + # print weights_ones.shape + + weights, bias = model.get_weights() + print weights.shape + print bias.shape + print bias + + ### We calculate lr using softmax! + + dot_out = np.dot(X_train, weights) + print "dot_out shape: ", dot_out.shape + # print dot_out[:10] + + add_out = np.add(bias, dot_out) + print "add_out shape: ", add_out.shape + # print add_out[:10] + + # lr = np.around(softmax(add_out), decimals = 6) + lr = softmax(add_out) + print "lr shape: ", lr.shape + # print lr[:10] + # print np.count_nonzero(lr)i + + ### Save model to npz files + if not os.path.exists("test_model"): + os.makedirs("test_model") + np.savez("test_model/model", weights = weights, bias = bias) + + print "Model saved! Check test_model directory" From 9643f52aa5dae8f9c1d8435b777ceeaaeb76ea57 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Wed, 14 Sep 2016 14:30:23 +0200 Subject: [PATCH 3/5] debug --- src/tensor.h | 6 ++++-- src/test.cu | 14 ++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/tensor.h b/src/tensor.h index 02e1645f..499e1ae0 100644 --- a/src/tensor.h +++ b/src/tensor.h @@ -172,11 +172,13 @@ class TensorImpl { strm << "shape=" << marian::Debug(shape_) << std::endl; // values - /* size_t totSize = GetTotalSize(shape()); std::vector values(totSize); thrust::copy(data_.begin(), data_.end(), values.begin()); - */ + + for (size_t i = 0; i < totSize; ++i) { + strm << values[i] << " "; + } return strm.str(); } }; diff --git a/src/test.cu b/src/test.cu index 6a4ef4bf..dec93919 100644 --- a/src/test.cu +++ b/src/test.cu @@ -21,10 +21,10 @@ int main(int argc, char** argv) { Expr w = param(shape={IMAGE_SIZE, LABEL_SIZE}, name="W0"); Expr b = param(shape={1, LABEL_SIZE}, name="b0"); - auto scores = dot(x, w) + b; - auto lr = softmax(scores, axis=1, name="pred"); - auto graph = -mean(sum(y * log(lr), axis=1), axis=0, name="cost"); - cerr << "lr=" << lr.Debug() << endl; + Expr scores = dot(x, w) + b; + Expr lr = softmax(scores, axis=1, name="pred"); + Expr graph = -mean(sum(y * log(lr), axis=1), axis=0, name="cost"); + cerr << "lr=" << Debug(lr.val().shape()) << endl; int numofdata; vector images = datasets::mnist::ReadImages("../examples/mnist/t10k-images-idx3-ubyte", numofdata, IMAGE_SIZE); @@ -38,8 +38,8 @@ int main(int argc, char** argv) { tx.Load(images); ty.Load(labels); - cerr << "tx=" << tx.Debug() << endl; - cerr << "ty=" << ty.Debug() << endl; + cerr << "tx=" << Debug(tx.shape()) << endl; + cerr << "ty=" << Debug(ty.shape()) << endl; x = tx; y = ty; @@ -50,6 +50,8 @@ int main(int argc, char** argv) { std::cerr << "lr: " << Debug(lr.val().shape()) << endl; std::cerr << "Log-likelihood: " << Debug(graph.val().shape()) << endl ; + std::cerr << "scores=" << scores.val().Debug() << endl; + graph.backward(); //std::cerr << graph["pred"].val()[0] << std::endl; From 04a620db191e933dd8edb78e3386aefa4225a9c0 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Wed, 14 Sep 2016 13:42:59 +0100 Subject: [PATCH 4/5] debug --- src/test.cu | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test.cu b/src/test.cu index dec93919..bd417cee 100644 --- a/src/test.cu +++ b/src/test.cu @@ -24,7 +24,7 @@ int main(int argc, char** argv) { Expr scores = dot(x, w) + b; Expr lr = softmax(scores, axis=1, name="pred"); Expr graph = -mean(sum(y * log(lr), axis=1), axis=0, name="cost"); - cerr << "lr=" << Debug(lr.val().shape()) << endl; + //cerr << "lr=" << Debug(lr.val().shape()) << endl; int numofdata; vector images = datasets::mnist::ReadImages("../examples/mnist/t10k-images-idx3-ubyte", numofdata, IMAGE_SIZE); @@ -50,7 +50,8 @@ int main(int argc, char** argv) { std::cerr << "lr: " << Debug(lr.val().shape()) << endl; std::cerr << "Log-likelihood: " << Debug(graph.val().shape()) << endl ; - std::cerr << "scores=" << scores.val().Debug() << endl; + //std::cerr << "scores=" << scores.val().Debug() << endl; + std::cerr << "lr=" << lr.val().Debug() << endl; graph.backward(); From f05d17e7ae179ac5d94bf12e443c28121ec7ab23 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Wed, 14 Sep 2016 14:51:49 +0200 Subject: [PATCH 5/5] output tensors in shape --- src/tensor.h | 9 +++++++-- src/test.cu | 6 +++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/tensor.h b/src/tensor.h index 499e1ae0..83965508 100644 --- a/src/tensor.h +++ b/src/tensor.h @@ -176,8 +176,13 @@ class TensorImpl { std::vector values(totSize); thrust::copy(data_.begin(), data_.end(), values.begin()); - for (size_t i = 0; i < totSize; ++i) { - strm << values[i] << " "; + size_t ind = 0; + for (size_t i = 0; i < shape()[0]; ++i) { + for (size_t j = 0; j < shape()[1]; ++j) { + strm << values[ind] << " "; + ++ind; + } + strm << std::endl; } return strm.str(); } diff --git a/src/test.cu b/src/test.cu index bd417cee..9eb9b498 100644 --- a/src/test.cu +++ b/src/test.cu @@ -21,8 +21,8 @@ int main(int argc, char** argv) { Expr w = param(shape={IMAGE_SIZE, LABEL_SIZE}, name="W0"); Expr b = param(shape={1, LABEL_SIZE}, name="b0"); - Expr scores = dot(x, w) + b; - Expr lr = softmax(scores, axis=1, name="pred"); + Expr z = dot(x, w) + b; + Expr lr = softmax(z, axis=1, name="pred"); Expr graph = -mean(sum(y * log(lr), axis=1), axis=0, name="cost"); //cerr << "lr=" << Debug(lr.val().shape()) << endl; @@ -46,7 +46,7 @@ int main(int argc, char** argv) { graph.forward(500); - std::cerr << "scores: " << Debug(scores.val().shape()) << endl; + std::cerr << "z: " << Debug(z.val().shape()) << endl; std::cerr << "lr: " << Debug(lr.val().shape()) << endl; std::cerr << "Log-likelihood: " << Debug(graph.val().shape()) << endl ;