mirror of
https://github.com/marian-nmt/marian.git
synced 2024-09-17 09:47:34 +03:00
Detect fortran_order in npz (#911)
* Fix fortran_order parsing * Abort on non row-major NPZ entries * Update CHANGELOG * Update VERSION Co-authored-by: Roman Grundkiewicz <rgrundkiewicz@gmail.com>
This commit is contained in:
parent
58c4576e5d
commit
601c9ac980
@ -12,12 +12,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
|
|||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
- Scripts using PyYAML now use `safe_load`; see https://msg.pyyaml.org/load
|
- Scripts using PyYAML now use `safe_load`; see https://msg.pyyaml.org/load
|
||||||
|
- Fixed check for `fortran_ordering` in cnpy
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
- Make guided-alignment faster via sparse memory layout, add alignment points for EOS, remove losses other than ce.
|
- Make guided-alignment faster via sparse memory layout, add alignment points for EOS, remove losses other than ce
|
||||||
- Changed minimal C++ standard to C++-17
|
- Changed minimal C++ standard to C++-17
|
||||||
- Faster LSH top-k search on CPU
|
- Faster LSH top-k search on CPU
|
||||||
- Updated intgemm to the latest upstream version
|
- Updated intgemm to the latest upstream version
|
||||||
|
- Parameters in npz files are no longer implicitly assumed to be row-ordered. Non row-ordered parameters will result in an abort
|
||||||
|
|
||||||
## [1.11.0] - 2022-02-08
|
## [1.11.0] - 2022-02-08
|
||||||
|
|
||||||
|
2
src/3rd_party/cnpy/cnpy.cpp
vendored
2
src/3rd_party/cnpy/cnpy.cpp
vendored
@ -82,7 +82,7 @@ void cnpy::parse_npy_header(FILE* fp, char& type, unsigned int& word_size, unsig
|
|||||||
|
|
||||||
//fortran order
|
//fortran order
|
||||||
loc1 = (int)header.find("fortran_order")+16;
|
loc1 = (int)header.find("fortran_order")+16;
|
||||||
fortran_order = (header.substr(loc1,5) == "True" ? true : false);
|
fortran_order = (header.substr(loc1,4) == "True" ? true : false);
|
||||||
|
|
||||||
//shape
|
//shape
|
||||||
loc1 = (int)header.find("(");
|
loc1 = (int)header.find("(");
|
||||||
|
@ -90,6 +90,9 @@ void addMetaToItems(const std::string& meta,
|
|||||||
void loadItemsFromNpz(const std::string& fileName, std::vector<Item>& items) {
|
void loadItemsFromNpz(const std::string& fileName, std::vector<Item>& items) {
|
||||||
auto numpy = cnpy::npz_load(fileName);
|
auto numpy = cnpy::npz_load(fileName);
|
||||||
for(auto it : numpy) {
|
for(auto it : numpy) {
|
||||||
|
ABORT_IF(
|
||||||
|
it.second->fortran_order, "Numpy item '{}' is not stored in row-major order", it.first);
|
||||||
|
|
||||||
Shape shape;
|
Shape shape;
|
||||||
shape.resize(it.second->shape.size());
|
shape.resize(it.second->shape.size());
|
||||||
for(size_t i = 0; i < it.second->shape.size(); ++i)
|
for(size_t i = 0; i < it.second->shape.size(); ++i)
|
||||||
|
Loading…
Reference in New Issue
Block a user