mirror of
https://github.com/google/sentencepiece.git
synced 2024-09-11 10:55:42 +03:00
Introduced new Filesystem API phase 2.
This commit is contained in:
parent
f617fcb697
commit
e9b3073703
1
.gitignore
vendored
1
.gitignore
vendored
@ -5,6 +5,7 @@ Makefile.in
|
||||
/py-compile
|
||||
/test-driver
|
||||
/ylwrap
|
||||
/build
|
||||
|
||||
/autom4te.cache
|
||||
/autoscan.log
|
||||
|
@ -64,12 +64,6 @@ static constexpr int64 kint64max = ((int64)(0x7FFFFFFFFFFFFFFF));
|
||||
|
||||
static constexpr uint32 kUnicodeError = 0xFFFD;
|
||||
|
||||
#ifdef OS_WIN
|
||||
#define OUTPUT_MODE std::ios::binary | std::ios::out
|
||||
#else
|
||||
#define OUTPUT_MODE std::ios::out
|
||||
#endif
|
||||
|
||||
#if defined(OS_WIN) && defined(UNICODE) && defined(_UNICODE)
|
||||
#define WPATH(path) (::sentencepiece::win32::Utf8ToWide(path).c_str())
|
||||
#else
|
||||
|
@ -13,8 +13,16 @@
|
||||
// limitations under the License.!
|
||||
|
||||
#include "filesystem.h"
|
||||
#include <iostream>
|
||||
|
||||
#include "util.h"
|
||||
|
||||
#ifdef OS_WIN
|
||||
#define OUTPUT_MODE std::ios::binary | std::ios::out
|
||||
#else
|
||||
#define OUTPUT_MODE std::ios::out
|
||||
#endif
|
||||
|
||||
namespace sentencepiece {
|
||||
namespace filesystem {
|
||||
|
||||
@ -38,7 +46,15 @@ class PosixReadableFile : public ReadableFile {
|
||||
return static_cast<bool>(std::getline(*is_, *line));
|
||||
}
|
||||
|
||||
bool ReadAll(std::string *line) { return true; }
|
||||
bool ReadAll(std::string *line) {
|
||||
if (is_ == &std::cin) {
|
||||
LOG(ERROR) << "ReadAll is not supported for stdin.";
|
||||
return false;
|
||||
}
|
||||
line->assign(std::istreambuf_iterator<char>(*is_),
|
||||
std::istreambuf_iterator<char>());
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
util::Status status_;
|
||||
@ -72,7 +88,7 @@ class PosixWritableFile : public WritableFile {
|
||||
private:
|
||||
util::Status status_;
|
||||
std::ostream *os_;
|
||||
}; // namespace filesystem
|
||||
};
|
||||
|
||||
std::unique_ptr<ReadableFile> NewReadableFile(absl::string_view filename) {
|
||||
return port::MakeUnique<PosixReadableFile>(filename);
|
||||
|
@ -43,13 +43,11 @@ SentencePieceProcessor::SentencePieceProcessor() {}
|
||||
SentencePieceProcessor::~SentencePieceProcessor() {}
|
||||
|
||||
util::Status SentencePieceProcessor::Load(util::min_string_view filename) {
|
||||
std::ifstream ifs(WPATH(filename.data()), std::ios::binary | std::ios::in);
|
||||
if (!ifs) {
|
||||
return util::StatusBuilder(util::error::NOT_FOUND)
|
||||
<< "\"" << filename.data() << "\": " << util::StrError(errno);
|
||||
}
|
||||
|
||||
return Load(&ifs);
|
||||
auto input = filesystem::NewReadableFile(string_util::ToSV(filename));
|
||||
RETURN_IF_ERROR(input->status());
|
||||
std::string proto;
|
||||
CHECK_OR_RETURN(input->ReadAll(&proto));
|
||||
return LoadFromSerializedProto(proto);
|
||||
}
|
||||
|
||||
void SentencePieceProcessor::LoadOrDie(util::min_string_view filename) {
|
||||
@ -57,10 +55,9 @@ void SentencePieceProcessor::LoadOrDie(util::min_string_view filename) {
|
||||
}
|
||||
|
||||
util::Status SentencePieceProcessor::Load(std::istream *is) {
|
||||
CHECK_OR_RETURN(is) << "input ifstream is null";
|
||||
auto model_proto = port::MakeUnique<ModelProto>();
|
||||
CHECK_OR_RETURN(model_proto->ParseFromIstream(is)) << "Model file is broken";
|
||||
return Load(std::move(model_proto));
|
||||
return util::StatusBuilder(util::error::UNIMPLEMENTED)
|
||||
<< "std::stream API is deprecated. Use LoadFromSerializedProto() "
|
||||
<< "to load model from any serialized blob object.";
|
||||
}
|
||||
|
||||
util::Status SentencePieceProcessor::Load(const ModelProto &model_proto) {
|
||||
|
@ -19,7 +19,6 @@
|
||||
#include <string.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <fstream>
|
||||
#include <memory>
|
||||
#include <random>
|
||||
#include <sstream>
|
||||
|
Loading…
Reference in New Issue
Block a user