google/sentencepiece
1
1
Fork 0
mirror of https://github.com/google/sentencepiece.git synced 2024-08-16 06:10:47 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fixed Unicode/Byte compatiblities on Python module

Browse Source
This commit is contained in:
Taku Kudo 2018-08-14 21:43:43 +09:00
parent e6efe0c874
commit 453fd9a5cf
3 changed files with 39 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -61,3 +61,6 @@ dist/
*.swp
*.swo
*.pyc
m.model
m.vocab

27
python/sentencepiece.i
View File

@ -8,10 +8,12 @@
namespace {
PyObject* kUnicodeInput = reinterpret_cast<PyObject* >(0x1);
PyObject* kByteInput = reinterpret_cast<PyObject* >(0x2);
inline void ReleaseResultObject(PyObject *obj) {
if (obj != nullptr && obj != kUnicodeInput)
if (obj != nullptr && obj != kUnicodeInput && obj != kByteInput) {
Py_XDECREF(obj);
}
}
class PyInputString {
@ -25,7 +27,7 @@ class PyInputString {
} else if (PyBytes_Check(obj)) {
// Python3, Bytes
PyBytes_AsStringAndSize(obj, &str_, &size_);
input_type_ = nullptr;
input_type_ = kByteInput;
}
#else
if (PyUnicode_Check(obj)) {
@ -36,7 +38,7 @@ class PyInputString {
} else if (PyString_Check(obj)) {
// Python2, Bytes,
PyString_AsStringAndSize(obj, &str_, &size_);
input_type_ = nullptr;
input_type_ = kByteInput;
}
#endif
else {
@ -48,6 +50,14 @@ class PyInputString {
bool IsAvalable() const { return str_ != nullptr; }
PyObject *input_type() const { return input_type_; }
static bool IsUnicode(PyObject *resultobj) {
#if PY_VERSION_HEX >= 0x03000000
return (resultobj == nullptr || resultobj == kUnicodeInput);
#else
return (resultobj != nullptr && resultobj != kByteInput);
#endif
}
private:
PyObject* input_type_ = nullptr;
char* str_ = nullptr;
@ -56,14 +66,13 @@ class PyInputString {
PyObject* MakePyOutputString(const std::string& output,
PyObject *resultobj) {
if (PyInputString::IsUnicode(resultobj)) {
return PyUnicode_FromStringAndSize(output.data(), output.size());
}
#if PY_VERSION_HEX >= 0x03000000
return resultobj != nullptr ?
PyBytes_FromStringAndSize(output.data(), output.size()) :
PyUnicode_FromStringAndSize(output.data(), output.size());
return PyBytes_FromStringAndSize(output.data(), output.size());
#else
return resultobj == nullptr ?
PyString_FromStringAndSize(output.data(), output.size()) :
PyUnicode_FromStringAndSize(output.data(), output.size());
return PyString_FromStringAndSize(output.data(), output.size());
#endif
}

27
python/sentencepiece_wrap.cxx
View File

@ -3124,10 +3124,12 @@ namespace swig {
namespace {
PyObject* kUnicodeInput = reinterpret_cast<PyObject* >(0x1);
PyObject* kByteInput = reinterpret_cast<PyObject* >(0x2);
inline void ReleaseResultObject(PyObject *obj) {
if (obj != nullptr && obj != kUnicodeInput)
if (obj != nullptr && obj != kUnicodeInput && obj != kByteInput) {
Py_XDECREF(obj);
}
}
class PyInputString {
@ -3141,7 +3143,7 @@ class PyInputString {
} else if (PyBytes_Check(obj)) {
// Python3, Bytes
PyBytes_AsStringAndSize(obj, &str_, &size_);
input_type_ = nullptr;
input_type_ = kByteInput;
}
#else
if (PyUnicode_Check(obj)) {
@ -3152,7 +3154,7 @@ class PyInputString {
} else if (PyString_Check(obj)) {
// Python2, Bytes,
PyString_AsStringAndSize(obj, &str_, &size_);
input_type_ = nullptr;
input_type_ = kByteInput;
}
#endif
else {
@ -3164,6 +3166,14 @@ class PyInputString {
bool IsAvalable() const { return str_ != nullptr; }
PyObject *input_type() const { return input_type_; }
static bool IsUnicode(PyObject *resultobj) {
#if PY_VERSION_HEX >= 0x03000000
return (resultobj == nullptr || resultobj == kUnicodeInput);
#else
return (resultobj != nullptr && resultobj != kByteInput);
#endif
}
private:
PyObject* input_type_ = nullptr;
char* str_ = nullptr;
@ -3172,14 +3182,13 @@ class PyInputString {
PyObject* MakePyOutputString(const std::string& output,
PyObject *resultobj) {
if (PyInputString::IsUnicode(resultobj)) {
return PyUnicode_FromStringAndSize(output.data(), output.size());
}
#if PY_VERSION_HEX >= 0x03000000
return resultobj != nullptr ?
PyBytes_FromStringAndSize(output.data(), output.size()) :
PyUnicode_FromStringAndSize(output.data(), output.size());
return PyBytes_FromStringAndSize(output.data(), output.size());
#else
return resultobj == nullptr ?
PyString_FromStringAndSize(output.data(), output.size()) :
PyUnicode_FromStringAndSize(output.data(), output.size());
return PyString_FromStringAndSize(output.data(), output.size());
#endif
}