From 601c9ac9807b5ffcbed298952435d9a17d954575 Mon Sep 17 00:00:00 2001 From: Graeme Nail Date: Tue, 15 Feb 2022 13:22:49 +0000 Subject: [PATCH] Detect fortran_order in npz (#911) * Fix fortran_order parsing * Abort on non row-major NPZ entries * Update CHANGELOG * Update VERSION Co-authored-by: Roman Grundkiewicz --- CHANGELOG.md | 4 +++- VERSION | 2 +- src/3rd_party/cnpy/cnpy.cpp | 2 +- src/common/io.cpp | 3 +++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a1dabf9..721ffd06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,12 +12,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Scripts using PyYAML now use `safe_load`; see https://msg.pyyaml.org/load +- Fixed check for `fortran_ordering` in cnpy ### Changed -- Make guided-alignment faster via sparse memory layout, add alignment points for EOS, remove losses other than ce. +- Make guided-alignment faster via sparse memory layout, add alignment points for EOS, remove losses other than ce - Changed minimal C++ standard to C++-17 - Faster LSH top-k search on CPU - Updated intgemm to the latest upstream version +- Parameters in npz files are no longer implicitly assumed to be row-ordered. Non row-ordered parameters will result in an abort ## [1.11.0] - 2022-02-08 diff --git a/VERSION b/VERSION index f5f1545d..62e1a502 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.11.4 +v1.11.5 diff --git a/src/3rd_party/cnpy/cnpy.cpp b/src/3rd_party/cnpy/cnpy.cpp index 7ab102f9..1bda4f48 100644 --- a/src/3rd_party/cnpy/cnpy.cpp +++ b/src/3rd_party/cnpy/cnpy.cpp @@ -82,7 +82,7 @@ void cnpy::parse_npy_header(FILE* fp, char& type, unsigned int& word_size, unsig //fortran order loc1 = (int)header.find("fortran_order")+16; - fortran_order = (header.substr(loc1,5) == "True" ? true : false); + fortran_order = (header.substr(loc1,4) == "True" ? true : false); //shape loc1 = (int)header.find("("); diff --git a/src/common/io.cpp b/src/common/io.cpp index e0b3f39a..6a7be6a3 100644 --- a/src/common/io.cpp +++ b/src/common/io.cpp @@ -90,6 +90,9 @@ void addMetaToItems(const std::string& meta, void loadItemsFromNpz(const std::string& fileName, std::vector& items) { auto numpy = cnpy::npz_load(fileName); for(auto it : numpy) { + ABORT_IF( + it.second->fortran_order, "Numpy item '{}' is not stored in row-major order", it.first); + Shape shape; shape.resize(it.second->shape.size()); for(size_t i = 0; i < it.second->shape.size(); ++i)