google/sentencepiece
1
1
Fork 0
mirror of https://github.com/google/sentencepiece.git synced 2024-09-11 10:55:42 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fixed Unicode/Byte compatiblities on Python module

Browse Source
This commit is contained in:
Taku Kudo 2018-08-14 21:43:43 +09:00
parent e6efe0c874
commit 453fd9a5cf
3 changed files with 39 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -61,3 +61,6 @@ dist/
*.swp *.swp
*.swo *.swo
*.pyc *.pyc
m.model
m.vocab

27
python/sentencepiece.i
View File

@ -8,10 +8,12 @@
namespace { namespace {
PyObject* kUnicodeInput = reinterpret_cast<PyObject* >(0x1); PyObject* kUnicodeInput = reinterpret_cast<PyObject* >(0x1);
PyObject* kByteInput = reinterpret_cast<PyObject* >(0x2);
inline void ReleaseResultObject(PyObject *obj) { inline void ReleaseResultObject(PyObject *obj) {
if (obj != nullptr && obj != kUnicodeInput) if (obj != nullptr && obj != kUnicodeInput && obj != kByteInput) {
Py_XDECREF(obj); Py_XDECREF(obj);
}
} }
class PyInputString { class PyInputString {
@ -25,7 +27,7 @@ class PyInputString {
} else if (PyBytes_Check(obj)) { } else if (PyBytes_Check(obj)) {
// Python3, Bytes // Python3, Bytes
PyBytes_AsStringAndSize(obj, &str_, &size_); PyBytes_AsStringAndSize(obj, &str_, &size_);
input_type_ = nullptr; input_type_ = kByteInput;
} }
#else #else
if (PyUnicode_Check(obj)) { if (PyUnicode_Check(obj)) {
@ -36,7 +38,7 @@ class PyInputString {
} else if (PyString_Check(obj)) { } else if (PyString_Check(obj)) {
// Python2, Bytes, // Python2, Bytes,
PyString_AsStringAndSize(obj, &str_, &size_); PyString_AsStringAndSize(obj, &str_, &size_);
input_type_ = nullptr; input_type_ = kByteInput;
} }
#endif #endif
else { else {
@ -48,6 +50,14 @@ class PyInputString {
bool IsAvalable() const { return str_ != nullptr; } bool IsAvalable() const { return str_ != nullptr; }
PyObject *input_type() const { return input_type_; } PyObject *input_type() const { return input_type_; }
static bool IsUnicode(PyObject *resultobj) {
#if PY_VERSION_HEX >= 0x03000000
return (resultobj == nullptr || resultobj == kUnicodeInput);
#else
return (resultobj != nullptr && resultobj != kByteInput);
#endif
}
private: private:
PyObject* input_type_ = nullptr; PyObject* input_type_ = nullptr;
char* str_ = nullptr; char* str_ = nullptr;
@ -56,14 +66,13 @@ class PyInputString {
PyObject* MakePyOutputString(const std::string& output, PyObject* MakePyOutputString(const std::string& output,
PyObject *resultobj) { PyObject *resultobj) {
if (PyInputString::IsUnicode(resultobj)) {
return PyUnicode_FromStringAndSize(output.data(), output.size());
}
#if PY_VERSION_HEX >= 0x03000000 #if PY_VERSION_HEX >= 0x03000000
return resultobj != nullptr ? return PyBytes_FromStringAndSize(output.data(), output.size());
PyBytes_FromStringAndSize(output.data(), output.size()) :
PyUnicode_FromStringAndSize(output.data(), output.size());
#else #else
return resultobj == nullptr ? return PyString_FromStringAndSize(output.data(), output.size());
PyString_FromStringAndSize(output.data(), output.size()) :
PyUnicode_FromStringAndSize(output.data(), output.size());
#endif #endif
} }

27
python/sentencepiece_wrap.cxx
View File

@ -3124,10 +3124,12 @@ namespace swig {
namespace { namespace {
PyObject* kUnicodeInput = reinterpret_cast<PyObject* >(0x1); PyObject* kUnicodeInput = reinterpret_cast<PyObject* >(0x1);
PyObject* kByteInput = reinterpret_cast<PyObject* >(0x2);
inline void ReleaseResultObject(PyObject *obj) { inline void ReleaseResultObject(PyObject *obj) {
if (obj != nullptr && obj != kUnicodeInput) if (obj != nullptr && obj != kUnicodeInput && obj != kByteInput) {
Py_XDECREF(obj); Py_XDECREF(obj);
}
} }
class PyInputString { class PyInputString {
@ -3141,7 +3143,7 @@ class PyInputString {
} else if (PyBytes_Check(obj)) { } else if (PyBytes_Check(obj)) {
// Python3, Bytes // Python3, Bytes
PyBytes_AsStringAndSize(obj, &str_, &size_); PyBytes_AsStringAndSize(obj, &str_, &size_);
input_type_ = nullptr; input_type_ = kByteInput;
} }
#else #else
if (PyUnicode_Check(obj)) { if (PyUnicode_Check(obj)) {
@ -3152,7 +3154,7 @@ class PyInputString {
} else if (PyString_Check(obj)) { } else if (PyString_Check(obj)) {
// Python2, Bytes, // Python2, Bytes,
PyString_AsStringAndSize(obj, &str_, &size_); PyString_AsStringAndSize(obj, &str_, &size_);
input_type_ = nullptr; input_type_ = kByteInput;
} }
#endif #endif
else { else {
@ -3164,6 +3166,14 @@ class PyInputString {
bool IsAvalable() const { return str_ != nullptr; } bool IsAvalable() const { return str_ != nullptr; }
PyObject *input_type() const { return input_type_; } PyObject *input_type() const { return input_type_; }
static bool IsUnicode(PyObject *resultobj) {
#if PY_VERSION_HEX >= 0x03000000
return (resultobj == nullptr || resultobj == kUnicodeInput);
#else
return (resultobj != nullptr && resultobj != kByteInput);
#endif
}
private: private:
PyObject* input_type_ = nullptr; PyObject* input_type_ = nullptr;
char* str_ = nullptr; char* str_ = nullptr;
@ -3172,14 +3182,13 @@ class PyInputString {
PyObject* MakePyOutputString(const std::string& output, PyObject* MakePyOutputString(const std::string& output,
PyObject *resultobj) { PyObject *resultobj) {
if (PyInputString::IsUnicode(resultobj)) {
return PyUnicode_FromStringAndSize(output.data(), output.size());
}
#if PY_VERSION_HEX >= 0x03000000 #if PY_VERSION_HEX >= 0x03000000
return resultobj != nullptr ? return PyBytes_FromStringAndSize(output.data(), output.size());
PyBytes_FromStringAndSize(output.data(), output.size()) :
PyUnicode_FromStringAndSize(output.data(), output.size());
#else #else
return resultobj == nullptr ? return PyString_FromStringAndSize(output.data(), output.size());
PyString_FromStringAndSize(output.data(), output.size()) :
PyUnicode_FromStringAndSize(output.data(), output.size());
#endif #endif
} }