marian-nmt/marian
1
1
Fork 0
mirror of https://github.com/marian-nmt/marian.git synced 2024-09-19 02:37:14 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix delay between saving and validation for async SGD

Browse Source
This commit is contained in:
Marcin Junczys-Dowmunt 2018-01-18 22:26:46 -08:00
parent 69b37bae7e
commit 958222ba9c
1 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/training/graph_group_async.cu
View File

@ -223,14 +223,8 @@ void AsyncGraphGroup::execute(Ptr<data::Batch> batch) {
scheduler_->update(cost, batch);
if(scheduler_->saving()) {
if(movingAvg_)
fetchParams(graph->params()->vals(), paramsAvg_, t_id);
this->save(graph);
}
if(scheduler_->validating()) {
// Wait with validation until all other threads are done with update.
if(scheduler_->saving() || scheduler_->validating()) {
// Wait with validation or saving until all other threads are done with update.
// We want to reuse the graphs for validation, so they need to be in
// a safe state.
pool_.wait_for_others(lock);
@ -238,9 +232,14 @@ void AsyncGraphGroup::execute(Ptr<data::Batch> batch) {
if(movingAvg_)
for(auto g : graphs_)
fetchParams(g->params()->vals(), paramsAvg_, t_id);
scheduler_->validate(graphs_);
// Validation is done, tell other threads to continue work.
if(scheduler_->saving())
this->save(graph);
if(scheduler_->validating())
scheduler_->validate(graphs_);
// Validation or saving is done, tell other threads to continue work.
pool_.notify_others();
}
}