mirror of
https://github.com/google/sentencepiece.git
synced 2024-09-11 10:55:42 +03:00
Fixed Unicode/Byte compatiblities on Python module
This commit is contained in:
parent
e6efe0c874
commit
453fd9a5cf
3
.gitignore
vendored
3
.gitignore
vendored
@ -61,3 +61,6 @@ dist/
|
|||||||
*.swp
|
*.swp
|
||||||
*.swo
|
*.swo
|
||||||
*.pyc
|
*.pyc
|
||||||
|
|
||||||
|
m.model
|
||||||
|
m.vocab
|
||||||
|
@ -8,10 +8,12 @@
|
|||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
PyObject* kUnicodeInput = reinterpret_cast<PyObject* >(0x1);
|
PyObject* kUnicodeInput = reinterpret_cast<PyObject* >(0x1);
|
||||||
|
PyObject* kByteInput = reinterpret_cast<PyObject* >(0x2);
|
||||||
|
|
||||||
inline void ReleaseResultObject(PyObject *obj) {
|
inline void ReleaseResultObject(PyObject *obj) {
|
||||||
if (obj != nullptr && obj != kUnicodeInput)
|
if (obj != nullptr && obj != kUnicodeInput && obj != kByteInput) {
|
||||||
Py_XDECREF(obj);
|
Py_XDECREF(obj);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class PyInputString {
|
class PyInputString {
|
||||||
@ -25,7 +27,7 @@ class PyInputString {
|
|||||||
} else if (PyBytes_Check(obj)) {
|
} else if (PyBytes_Check(obj)) {
|
||||||
// Python3, Bytes
|
// Python3, Bytes
|
||||||
PyBytes_AsStringAndSize(obj, &str_, &size_);
|
PyBytes_AsStringAndSize(obj, &str_, &size_);
|
||||||
input_type_ = nullptr;
|
input_type_ = kByteInput;
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
if (PyUnicode_Check(obj)) {
|
if (PyUnicode_Check(obj)) {
|
||||||
@ -36,7 +38,7 @@ class PyInputString {
|
|||||||
} else if (PyString_Check(obj)) {
|
} else if (PyString_Check(obj)) {
|
||||||
// Python2, Bytes,
|
// Python2, Bytes,
|
||||||
PyString_AsStringAndSize(obj, &str_, &size_);
|
PyString_AsStringAndSize(obj, &str_, &size_);
|
||||||
input_type_ = nullptr;
|
input_type_ = kByteInput;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
else {
|
else {
|
||||||
@ -48,6 +50,14 @@ class PyInputString {
|
|||||||
bool IsAvalable() const { return str_ != nullptr; }
|
bool IsAvalable() const { return str_ != nullptr; }
|
||||||
PyObject *input_type() const { return input_type_; }
|
PyObject *input_type() const { return input_type_; }
|
||||||
|
|
||||||
|
static bool IsUnicode(PyObject *resultobj) {
|
||||||
|
#if PY_VERSION_HEX >= 0x03000000
|
||||||
|
return (resultobj == nullptr || resultobj == kUnicodeInput);
|
||||||
|
#else
|
||||||
|
return (resultobj != nullptr && resultobj != kByteInput);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
PyObject* input_type_ = nullptr;
|
PyObject* input_type_ = nullptr;
|
||||||
char* str_ = nullptr;
|
char* str_ = nullptr;
|
||||||
@ -56,14 +66,13 @@ class PyInputString {
|
|||||||
|
|
||||||
PyObject* MakePyOutputString(const std::string& output,
|
PyObject* MakePyOutputString(const std::string& output,
|
||||||
PyObject *resultobj) {
|
PyObject *resultobj) {
|
||||||
|
if (PyInputString::IsUnicode(resultobj)) {
|
||||||
|
return PyUnicode_FromStringAndSize(output.data(), output.size());
|
||||||
|
}
|
||||||
#if PY_VERSION_HEX >= 0x03000000
|
#if PY_VERSION_HEX >= 0x03000000
|
||||||
return resultobj != nullptr ?
|
return PyBytes_FromStringAndSize(output.data(), output.size());
|
||||||
PyBytes_FromStringAndSize(output.data(), output.size()) :
|
|
||||||
PyUnicode_FromStringAndSize(output.data(), output.size());
|
|
||||||
#else
|
#else
|
||||||
return resultobj == nullptr ?
|
return PyString_FromStringAndSize(output.data(), output.size());
|
||||||
PyString_FromStringAndSize(output.data(), output.size()) :
|
|
||||||
PyUnicode_FromStringAndSize(output.data(), output.size());
|
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3124,10 +3124,12 @@ namespace swig {
|
|||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
PyObject* kUnicodeInput = reinterpret_cast<PyObject* >(0x1);
|
PyObject* kUnicodeInput = reinterpret_cast<PyObject* >(0x1);
|
||||||
|
PyObject* kByteInput = reinterpret_cast<PyObject* >(0x2);
|
||||||
|
|
||||||
inline void ReleaseResultObject(PyObject *obj) {
|
inline void ReleaseResultObject(PyObject *obj) {
|
||||||
if (obj != nullptr && obj != kUnicodeInput)
|
if (obj != nullptr && obj != kUnicodeInput && obj != kByteInput) {
|
||||||
Py_XDECREF(obj);
|
Py_XDECREF(obj);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class PyInputString {
|
class PyInputString {
|
||||||
@ -3141,7 +3143,7 @@ class PyInputString {
|
|||||||
} else if (PyBytes_Check(obj)) {
|
} else if (PyBytes_Check(obj)) {
|
||||||
// Python3, Bytes
|
// Python3, Bytes
|
||||||
PyBytes_AsStringAndSize(obj, &str_, &size_);
|
PyBytes_AsStringAndSize(obj, &str_, &size_);
|
||||||
input_type_ = nullptr;
|
input_type_ = kByteInput;
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
if (PyUnicode_Check(obj)) {
|
if (PyUnicode_Check(obj)) {
|
||||||
@ -3152,7 +3154,7 @@ class PyInputString {
|
|||||||
} else if (PyString_Check(obj)) {
|
} else if (PyString_Check(obj)) {
|
||||||
// Python2, Bytes,
|
// Python2, Bytes,
|
||||||
PyString_AsStringAndSize(obj, &str_, &size_);
|
PyString_AsStringAndSize(obj, &str_, &size_);
|
||||||
input_type_ = nullptr;
|
input_type_ = kByteInput;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
else {
|
else {
|
||||||
@ -3164,6 +3166,14 @@ class PyInputString {
|
|||||||
bool IsAvalable() const { return str_ != nullptr; }
|
bool IsAvalable() const { return str_ != nullptr; }
|
||||||
PyObject *input_type() const { return input_type_; }
|
PyObject *input_type() const { return input_type_; }
|
||||||
|
|
||||||
|
static bool IsUnicode(PyObject *resultobj) {
|
||||||
|
#if PY_VERSION_HEX >= 0x03000000
|
||||||
|
return (resultobj == nullptr || resultobj == kUnicodeInput);
|
||||||
|
#else
|
||||||
|
return (resultobj != nullptr && resultobj != kByteInput);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
PyObject* input_type_ = nullptr;
|
PyObject* input_type_ = nullptr;
|
||||||
char* str_ = nullptr;
|
char* str_ = nullptr;
|
||||||
@ -3172,14 +3182,13 @@ class PyInputString {
|
|||||||
|
|
||||||
PyObject* MakePyOutputString(const std::string& output,
|
PyObject* MakePyOutputString(const std::string& output,
|
||||||
PyObject *resultobj) {
|
PyObject *resultobj) {
|
||||||
|
if (PyInputString::IsUnicode(resultobj)) {
|
||||||
|
return PyUnicode_FromStringAndSize(output.data(), output.size());
|
||||||
|
}
|
||||||
#if PY_VERSION_HEX >= 0x03000000
|
#if PY_VERSION_HEX >= 0x03000000
|
||||||
return resultobj != nullptr ?
|
return PyBytes_FromStringAndSize(output.data(), output.size());
|
||||||
PyBytes_FromStringAndSize(output.data(), output.size()) :
|
|
||||||
PyUnicode_FromStringAndSize(output.data(), output.size());
|
|
||||||
#else
|
#else
|
||||||
return resultobj == nullptr ?
|
return PyString_FromStringAndSize(output.data(), output.size());
|
||||||
PyString_FromStringAndSize(output.data(), output.size()) :
|
|
||||||
PyUnicode_FromStringAndSize(output.data(), output.size());
|
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user