mirror of
https://github.com/marian-nmt/marian.git
synced 2024-09-11 06:15:56 +03:00
Merged.
This commit is contained in:
commit
76cda34544
@ -26,99 +26,9 @@
|
||||
</natures>
|
||||
<linkedResources>
|
||||
<link>
|
||||
<name>CMakeLists.txt</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/CMakeLists.txt</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>compile_time_crc32.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/compile_time_crc32.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>definitions.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/definitions.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>exception.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/exception.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>exception.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/exception.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>expression_operators.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/expression_operators.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>expressions.cu</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/expressions.cu</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>expressions.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/expressions.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>graph.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/graph.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>graph_operators.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/graph_operators.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>keywords.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/keywords.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>marian.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/marian.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>mnist.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/mnist.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>tensor.cu</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/tensor.cu</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>tensor.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/tensor.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>tensor_operators.cu</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/tensor_operators.cu</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>tensor_operators.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/tensor_operators.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>test.cu</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/test.cu</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>thrust_functions.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src/thrust_functions.h</locationURI>
|
||||
<name>src</name>
|
||||
<type>2</type>
|
||||
<locationURI>PARENT-1-PROJECT_LOC/src</locationURI>
|
||||
</link>
|
||||
</linkedResources>
|
||||
</projectDescription>
|
||||
|
12
src/mnist.h
12
src/mnist.h
@ -11,8 +11,6 @@ namespace mnist {
|
||||
|
||||
typedef unsigned char uchar;
|
||||
|
||||
const size_t IMAGE_SIZE = 784;
|
||||
const size_t LABEL_SIZE = 10;
|
||||
|
||||
const size_t IMAGE_MAGIC_NUMBER = 2051;
|
||||
const size_t LABEL_MAGIC_NUMBER = 2049;
|
||||
@ -23,7 +21,7 @@ auto reverseInt = [](int i) {
|
||||
return ((int)c1 << 24) + ((int)c2 << 16) + ((int)c3 << 8) + c4;
|
||||
};
|
||||
|
||||
std::vector<float> ReadImages(const std::string& full_path, int& number_of_images) {
|
||||
std::vector<float> ReadImages(const std::string& full_path, int& number_of_images, int imgSize) {
|
||||
std::ifstream file(full_path);
|
||||
|
||||
if (! file.is_open())
|
||||
@ -42,9 +40,9 @@ std::vector<float> ReadImages(const std::string& full_path, int& number_of_image
|
||||
file.read((char *)&n_rows, sizeof(n_rows)), n_rows = reverseInt(n_rows);
|
||||
file.read((char *)&n_cols, sizeof(n_cols)), n_cols = reverseInt(n_cols);
|
||||
|
||||
assert(n_rows * n_cols == IMAGE_SIZE);
|
||||
assert(n_rows * n_cols == imgSize);
|
||||
|
||||
int n = number_of_images * IMAGE_SIZE;
|
||||
int n = number_of_images * imgSize;
|
||||
std::vector<float> _dataset(n);
|
||||
|
||||
for (int i = 0; i < n; i++) {
|
||||
@ -55,7 +53,7 @@ std::vector<float> ReadImages(const std::string& full_path, int& number_of_image
|
||||
return _dataset;
|
||||
}
|
||||
|
||||
std::vector<float> ReadLabels(const std::string& full_path, int& number_of_labels) {
|
||||
std::vector<float> ReadLabels(const std::string& full_path, int& number_of_labels, int label_size) {
|
||||
std::ifstream file(full_path);
|
||||
|
||||
if (! file.is_open())
|
||||
@ -70,7 +68,7 @@ std::vector<float> ReadLabels(const std::string& full_path, int& number_of_label
|
||||
|
||||
file.read((char *)&number_of_labels, sizeof(number_of_labels)), number_of_labels = reverseInt(number_of_labels);
|
||||
|
||||
int n = number_of_labels * LABEL_SIZE;
|
||||
int n = number_of_labels * label_size;
|
||||
std::vector<float> _dataset(n, 0.0f);
|
||||
|
||||
for (int i = 0; i < number_of_labels; i++) {
|
||||
|
@ -155,7 +155,7 @@ class TensorImpl {
|
||||
void set(const std::vector<Float> &values) {
|
||||
size_t totSize = std::accumulate(shape().begin(), shape().end(),
|
||||
1, std::multiplies<int>());
|
||||
std::cerr << "totSize=" << totSize << " " << values.size() << std::endl;
|
||||
std::cerr << "tensor size=" << totSize << " vector size=" << values.size() << std::endl;
|
||||
assert(totSize == values.size());
|
||||
thrust::copy(values.begin(), values.end(), data_.begin());
|
||||
}
|
||||
|
31
src/test.cu
31
src/test.cu
@ -12,31 +12,38 @@ int main(int argc, char** argv) {
|
||||
using namespace marian;
|
||||
using namespace keywords;
|
||||
|
||||
const size_t IMAGE_SIZE = 784;
|
||||
const size_t LABEL_SIZE = 10;
|
||||
|
||||
Expr x = input(shape={whatevs, 784}, name="X");
|
||||
Expr y = input(shape={whatevs, 10}, name="Y");
|
||||
Expr x = input(shape={whatevs, IMAGE_SIZE}, name="X");
|
||||
Expr y = input(shape={whatevs, LABEL_SIZE}, name="Y");
|
||||
|
||||
Expr w = param(shape={784, 10}, name="W0");
|
||||
Expr b = param(shape={1, 10}, name="b0");
|
||||
Expr w = param(shape={IMAGE_SIZE, LABEL_SIZE}, name="W0");
|
||||
Expr b = param(shape={1, LABEL_SIZE}, name="b0");
|
||||
|
||||
auto scores = dot(x, w) + b;
|
||||
auto lr = softmax_fast(scores, axis=1, name="pred");
|
||||
auto graph = -mean(sum(y * log(lr), axis=1), axis=0, name="cost");
|
||||
cerr << "lr=" << lr.Debug() << endl;
|
||||
|
||||
|
||||
Tensor tx({500, 784}, 1);
|
||||
Tensor ty({500, 10}, 1);
|
||||
|
||||
#if 0
|
||||
int numImg, imgSize;
|
||||
vector<float> images = datasets::mnist::ReadImages("../examples/mnist/t10k-images-idx3-ubyte", numImg, imgSize);
|
||||
vector<int> labels = datasets::mnist::ReadLabels("../examples/mnist/t10k-labels-idx1-ubyte");
|
||||
int numofdata;
|
||||
vector<float> images = datasets::mnist::ReadImages("../examples/mnist/t10k-images-idx3-ubyte", numofdata, IMAGE_SIZE);
|
||||
vector<float> labels = datasets::mnist::ReadLabels("../examples/mnist/t10k-labels-idx1-ubyte", numofdata, LABEL_SIZE);
|
||||
cerr << "images=" << images.size() << " labels=" << labels.size() << endl;
|
||||
cerr << "numofdata=" << numofdata << endl;
|
||||
|
||||
Tensor tx({numofdata, IMAGE_SIZE}, 1);
|
||||
Tensor ty({numofdata, LABEL_SIZE}, 1);
|
||||
|
||||
tx.Load(images);
|
||||
//ty.Load(labels);
|
||||
ty.Load(labels);
|
||||
|
||||
cerr << "tx=" << tx.Debug() << endl;
|
||||
cerr << "ty=" << ty.Debug() << endl;
|
||||
#else
|
||||
Tensor tx({500, 784}, 1);
|
||||
Tensor ty({500, 10}, 1);
|
||||
#endif
|
||||
|
||||
x = tx;
|
||||
|
Loading…
Reference in New Issue
Block a user