From 2c55cdb3c0d37e6522f79dbff56e8ee24bb4c61e Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 29 Sep 2022 19:01:49 +0000 Subject: [PATCH] Merged PR 25889: Fixes bad memory access problem in hashing Fix bad memory access problem in hashing by using the graph allocator --- CHANGELOG.md | 1 + VERSION | 2 +- src/training/graph_group.cpp | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44145b89..b06abec7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `--output-sampling` now works with ensembles (requires proper normalization via e.g `--weights 0.5 0.5`) ### Fixed +- Use allocator in hashing - Read/restore checkpoints from main process only when training with MPI - Multi-loss casts type to first loss-type before accumulation (aborted before due to missing cast) - Throw `ShapeSizeException` if total expanded shape size exceeds numeric capacity of the maximum int value (2^31-1) diff --git a/VERSION b/VERSION index d15b7998..fb1b4bd9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.11.11 +v1.11.12 diff --git a/src/training/graph_group.cpp b/src/training/graph_group.cpp index 0ba1b279..cb95470f 100644 --- a/src/training/graph_group.cpp +++ b/src/training/graph_group.cpp @@ -99,7 +99,8 @@ void GraphGroup::syncParametersAndShards() { // compute hash value of parameters of 0-th graph (we only need to check one graph per node) for(int i = 0; i < hashes.size(); i++) { if(i == mpi_->myMPIRank()) { - hashes[i] = graphs_[0]->params()->vals()->hash(); // this is quite fast with on-GPU implementation + auto allocator = graphs_[0]->allocator(); + hashes[i] = graphs_[0]->params()->vals()->hash(1234, allocator); // this is quite fast with on-GPU implementation LOG(debug, "Parameter hash for graph 0 on node {}: {}", mpi_->myMPIRank(), hashes[i]); } }