marian-nmt/marian
1
1
Fork 0
mirror of https://github.com/marian-nmt/marian.git synced 2024-09-17 09:47:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

figure out where temp file class is used

Browse Source
This commit is contained in:
Hieu Hoang 2019-10-21 10:48:14 -07:00
parent f8061fc298
commit fef5c301ca
6 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/common/file_stream.h
View File

@ -43,7 +43,7 @@ namespace std {
namespace marian {
namespace io {
class TemporaryFile {
class TemporaryFile2 {
private:
int fd_{-1};
bool unlink_;
@ -112,14 +112,14 @@ private:
}
public:
TemporaryFile(const std::string base = "/tmp/", bool earlyUnlink = true)
TemporaryFile2(const std::string base = "/tmp/", bool earlyUnlink = true)
: unlink_(earlyUnlink) {
std::string baseTemp(base);
NormalizeTempPrefix(baseTemp);
fd_ = MakeTemp(baseTemp);
}
~TemporaryFile() {
~TemporaryFile2() {
#ifdef _MSC_VER
if (fd_ == -1)
return;
@ -217,7 +217,7 @@ public:
ABORT_IF(fail(), "Error {} ({}) opening file '{}'", errno, strerror(errno), path());
}
explicit InputFileStream(TemporaryFile& tempfile) {
explicit InputFileStream(TemporaryFile2& tempfile) {
RewindFile(tempfile.getFileDescriptor());
temporary_reader_.reset(new ReadFDBuf(tempfile.getFileDescriptor()));
istream_.reset(new std::istream(temporary_reader_.get()));
@ -312,7 +312,7 @@ public:
ABORT_IF(!marian::filesystem::exists(file_), "File '{}' could not be opened", file);
}
OutputFileStream(TemporaryFile& tempfile) {
OutputFileStream(TemporaryFile2& tempfile) {
RewindFile(tempfile.getFileDescriptor());
temporary_writer_.reset(new WriteFDBuf(tempfile.getFileDescriptor()));
ostream_.reset(new std::ostream(temporary_writer_.get()));

2
src/data/corpus.cpp
View File

@ -192,7 +192,7 @@ void Corpus::shuffleData(const std::vector<std::string>& paths) {
// create temp files that contain the data in randomized order
tempFiles_.resize(numStreams);
for(size_t i = 0; i < numStreams; ++i) {
tempFiles_[i].reset(new io::TemporaryFile(options_->get<std::string>("tempdir")));
tempFiles_[i].reset(new io::TemporaryFile2(options_->get<std::string>("tempdir")));
io::OutputFileStream out(*tempFiles_[i]);
const auto& corpusStream = corpus[i];
for(auto id : ids_) {

2
src/data/corpus.h
View File

@ -18,7 +18,7 @@ namespace data {
class Corpus : public CorpusBase {
private:
std::vector<UPtr<io::TemporaryFile>> tempFiles_;
std::vector<UPtr<io::TemporaryFile2>> tempFiles_;
std::vector<size_t> ids_;
// for shuffle-in-ram

2
src/data/corpus_nbest.h
View File

@ -18,7 +18,7 @@ namespace data {
class CorpusNBest : public CorpusBase {
private:
std::vector<UPtr<io::TemporaryFile>> tempFiles_;
std::vector<UPtr<io::TemporaryFile2>> tempFiles_;
std::vector<size_t> ids_;
int lastNum_{-1};
std::vector<std::string> lastLines_;

4
src/training/validator.h
View File

@ -465,12 +465,12 @@ public:
// Set up output file
std::string fileName;
Ptr<io::TemporaryFile> tempFile;
Ptr<io::TemporaryFile2> tempFile;
if(options_->hasAndNotEmpty("valid-translation-output")) {
fileName = options_->get<std::string>("valid-translation-output");
} else {
tempFile.reset(new io::TemporaryFile(options_->get<std::string>("tempdir"), false));
tempFile.reset(new io::TemporaryFile2(options_->get<std::string>("tempdir"), false));
fileName = tempFile->getFileName();
}

2
vs/Marian.vcxproj
View File

@ -42,7 +42,7 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\Marian\</IntDir>
<IncludePath>..\src\3rd_party\fbgemm\third_party\cpuinfo\deps\clog\include;..\src\3rd_party\fbgemm\third_party\cpuinfo\src;..\src\3rd_party\fbgemm\third_party\cpuinfo\include;..\src\3rd_party\fbgemm\third_party\asmjit\src;%MKL_PATH%\include;..\src\3rd_party\fbgemm\include;%CUDA_PATH%\include;..\src;..\src\3rd_party;%BOOST_INCLUDE_PATH%;%ZLIB_PATH%\include;$(VC_IncludePath);$(WindowsSDK_IncludePath);</IncludePath>
<IncludePath>..\src\3rd_party\fbgemm\third_party\cpuinfo\deps\clog\include;..\src\3rd_party\fbgemm\third_party\cpuinfo\src;..\src\3rd_party\fbgemm\third_party\cpuinfo\include;..\src\3rd_party\fbgemm\third_party\asmjit\src;%MKL_PATH%\include;..\src\3rd_party\fbgemm\include;%CUDA_PATH%\include;..\src;..\src\3rd_party;%BOOST_INCLUDE_PATH%;%ZLIB_PATH%\include;$(VC_IncludePath);$(WindowsSDK_IncludePath)</IncludePath>
<LibraryPath>%CUDA_PATH%\lib\x64;%BOOST_LIB_PATH%;%ZLIB_PATH%\lib;%MKL_PATH%\lib\intel64;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);$(NETFXKitsDir)Lib\um\x64</LibraryPath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">