marian-nmt/marian
1
1
Fork 0
mirror of https://github.com/marian-nmt/marian.git synced 2024-09-17 09:47:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merged PR 10304: Remove naked pointers, add binary read mode

Browse Source 
* Remove naked pointers in file_stream.{h,cpp}
* Add binary read mode
This commit is contained in:
Martin Junczys-Dowmunt 2019-11-05 23:18:45 +00:00
parent 7ba804b20b
commit 233281cc26
4 changed files with 30 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -22,7 +22,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Gradient-checkpointing
### Fixed
- Compilation for GCC >= 7 due to exception thrown in destrutor
- Removed naked pointer and potential memory-leak from file_stream.{cpp,h}
- Compilation for GCC >= 7 due to exception thrown in destructor
- Sort parameters by lexicographical order during allocation to ensure consistent
memory-layout during allocation, loading, saving.
- Output empty line when input is empty line. Previous behavior might result in

2
VERSION
View File

@ -1 +1 @@
v1.8.14
v1.8.15

51
src/common/file_stream.cpp
View File

@ -18,26 +18,23 @@ namespace io {
///////////////////////////////////////////////////////////////////////////////////////////////
InputFileStream::InputFileStream(const std::string &file)
: std::istream(NULL), file_(file), streamBuf1_(NULL), streamBuf2_(NULL) {
: std::istream(NULL), file_(file) {
ABORT_IF(!marian::filesystem::exists(file_), "File '{}' does not exist", file);
std::filebuf *fileBuf = new std::filebuf();
streamBuf1_ = fileBuf->open(file.c_str(), std::ios::in);
ABORT_IF(!streamBuf1_, "File can't be opened", file);
assert(fileBuf == streamBuf1_);
streamBuf1_.reset(new std::filebuf());
auto ret = static_cast<std::filebuf*>(streamBuf1_.get())->open(file.c_str(), std::ios::in | std::ios::binary);
ABORT_IF(!ret, "File cannot be opened", file);
ABORT_IF(ret != streamBuf1_.get(), "Return value is not equal to streambuf pointer, that is weird");
if(file_.extension() == marian::filesystem::Path(".gz")) {
streamBuf2_ = new zstr::istreambuf(streamBuf1_);
this->init(streamBuf2_);
streamBuf2_.reset(new zstr::istreambuf(streamBuf1_.get()));
this->init(streamBuf2_.get());
} else {
this->init(streamBuf1_);
this->init(streamBuf1_.get());
}
}
InputFileStream::~InputFileStream() {
delete streamBuf2_;
delete streamBuf1_;
}
InputFileStream::~InputFileStream() {}
bool InputFileStream::empty() {
return this->peek() == std::ifstream::traits_type::eof();
@ -66,27 +63,25 @@ std::istream &getline(std::istream &in, std::string &line) {
}
///////////////////////////////////////////////////////////////////////////////////////////////
OutputFileStream::OutputFileStream(const std::string &file)
: std::ostream(NULL), file_(file), streamBuf1_(NULL), streamBuf2_(NULL) {
std::filebuf *fileBuf = new std::filebuf();
streamBuf1_ = fileBuf->open(file.c_str(), std::ios::out | std::ios_base::binary);
ABORT_IF(!streamBuf1_, "File can't be opened", file);
assert(fileBuf == streamBuf1_);
: std::ostream(NULL), file_(file) {
streamBuf1_.reset(new std::filebuf());
auto ret = static_cast<std::filebuf*>(streamBuf1_.get())->open(file.c_str(), std::ios::out | std::ios_base::binary);
ABORT_IF(!ret, "File cannot be opened", file);
ABORT_IF(ret != streamBuf1_.get(), "Return value is not equal to streambuf pointer, that is weird");
if(file_.extension() == marian::filesystem::Path(".gz")) {
streamBuf2_ = new zstr::ostreambuf(streamBuf1_);
this->init(streamBuf2_);
streamBuf2_.reset(new zstr::ostreambuf(streamBuf1_.get()));
this->init(streamBuf2_.get());
} else {
this->init(streamBuf1_);
this->init(streamBuf1_.get());
}
}
OutputFileStream::OutputFileStream()
: std::ostream(NULL), streamBuf1_(NULL), streamBuf2_(NULL) {}
: std::ostream(NULL) {}
OutputFileStream::~OutputFileStream() {
this->flush();
delete streamBuf2_;
delete streamBuf1_;
}
///////////////////////////////////////////////////////////////////////////////////////////////
@ -150,12 +145,12 @@ void TemporaryFile::MakeTemp(const std::string &base) {
#endif
// open again with c++
std::filebuf *fileBuf = new std::filebuf();
streamBuf1_ = fileBuf->open(name, std::ios::out | std::ios_base::binary);
ABORT_IF(!streamBuf1_, "File can't be temp opened", name);
assert(fileBuf == streamBuf1_);
streamBuf1_.reset(new std::filebuf());
auto ret = static_cast<std::filebuf*>(streamBuf1_.get())->open(name, std::ios::out | std::ios_base::binary);
ABORT_IF(!streamBuf1_, "File cannot be temp opened", name);
ABORT_IF(ret != streamBuf1_.get(), "Return value is not equal to streambuf pointer, that is weird");
this->init(streamBuf1_);
this->init(streamBuf1_.get());
// close original file descriptor
ABORT_IF(close(fd), "Can't close file descriptor", name);

8
src/common/file_stream.h
View File

@ -38,8 +38,8 @@ public:
protected:
marian::filesystem::Path file_;
std::streambuf* streamBuf1_;
std::streambuf* streamBuf2_;
std::unique_ptr<std::streambuf> streamBuf1_;
std::unique_ptr<std::streambuf> streamBuf2_;
std::vector<char> readBuf_;
};
@ -63,8 +63,8 @@ protected:
explicit OutputFileStream(); // for temp file
marian::filesystem::Path file_;
std::streambuf* streamBuf1_;
std::streambuf* streamBuf2_;
std::unique_ptr<std::streambuf> streamBuf1_;
std::unique_ptr<std::streambuf> streamBuf2_;
};
///////////////////////////////////////////////////////////////////////////////////////////////