From 6bbf6db180b49b00ee58c40d7e5003c2baca7e67 Mon Sep 17 00:00:00 2001 From: Wilker Aziz Date: Tue, 13 Nov 2012 17:27:33 +0100 Subject: [PATCH] Wrapping OnDiskPt, Added OnDiskPt/Util.h with the tokenization procedure, so that they can be called from the Python wrap. --- OnDiskPt/Util.h | 93 ++ OnDiskPt/queryOnDiskPt.cpp | 103 +-- contrib/python/moses/cdictree.pxd | 3 - contrib/python/moses/condiskpt.pxd | 45 + contrib/python/moses/dictree.cpp | 1274 +++++++++++++++++++--------- contrib/python/moses/dictree.pyx | 21 +- contrib/python/setup.py | 2 +- 7 files changed, 1056 insertions(+), 485 deletions(-) create mode 100644 OnDiskPt/Util.h create mode 100644 contrib/python/moses/condiskpt.pxd diff --git a/OnDiskPt/Util.h b/OnDiskPt/Util.h new file mode 100644 index 000000000..92dd173c0 --- /dev/null +++ b/OnDiskPt/Util.h @@ -0,0 +1,93 @@ +#pragma once +#include +#include +#include "OnDiskWrapper.h" +#include "Phrase.h" +#include "SourcePhrase.h" +#include "Word.h" +#include "PhraseNode.h" + + +namespace OnDiskPt +{ + +void Tokenize(Phrase &phrase + , const std::string &token, bool addSourceNonTerm, bool addTargetNonTerm + , OnDiskWrapper &onDiskWrapper) +{ + + bool nonTerm = false; + size_t tokSize = token.size(); + int comStr =token.compare(0, 1, "["); + + if (comStr == 0) { + comStr = token.compare(tokSize - 1, 1, "]"); + nonTerm = comStr == 0; + } + + if (nonTerm) { + // non-term + size_t splitPos = token.find_first_of("[", 2); + std::string wordStr = token.substr(0, splitPos); + + if (splitPos == std::string::npos) { + // lhs - only 1 word + WordPtr word (new Word()); + word->CreateFromString(wordStr, onDiskWrapper.GetVocab()); + phrase.AddWord(word); + } else { + // source & target non-terms + if (addSourceNonTerm) { + WordPtr word( new Word()); + word->CreateFromString(wordStr, onDiskWrapper.GetVocab()); + phrase.AddWord(word); + } + + wordStr = token.substr(splitPos, tokSize - splitPos); + if (addTargetNonTerm) { + WordPtr word(new Word()); + word->CreateFromString(wordStr, onDiskWrapper.GetVocab()); + phrase.AddWord(word); + } + + } + } else { + // term + WordPtr word(new Word()); + word->CreateFromString(token, onDiskWrapper.GetVocab()); + phrase.AddWord(word); + } +} + +SourcePhrase Tokenize(const std::vector& tokens, OnDiskWrapper &onDiskWrapper) +{ + SourcePhrase sourcePhrase; + if (tokens.size() > 0){ + std::vector::const_iterator token = tokens.begin(); + for (; token + 1 != tokens.end(); ++token){ + Tokenize(sourcePhrase, *token, true, true, onDiskWrapper); + } + // last position. LHS non-term + Tokenize(sourcePhrase, *token, false, true, onDiskWrapper); + } + return sourcePhrase; +} + +const PhraseNode* Query(const SourcePhrase& sourcePhrase, OnDiskWrapper& onDiskWrapper) +{ + const PhraseNode *node = &onDiskWrapper.GetRootSourceNode(); + assert(node); + + for (size_t pos = 0; pos < sourcePhrase.GetSize(); ++pos) + { + const Word &word = sourcePhrase.GetWord(pos); + node = node->GetChild(word, onDiskWrapper); + if (node == NULL) + { + break; + } + } + return node; +} + +} diff --git a/OnDiskPt/queryOnDiskPt.cpp b/OnDiskPt/queryOnDiskPt.cpp index 202ddf7fe..f61a1bfc9 100644 --- a/OnDiskPt/queryOnDiskPt.cpp +++ b/OnDiskPt/queryOnDiskPt.cpp @@ -9,6 +9,7 @@ #include "moses/Util.h" #include "OnDiskWrapper.h" #include "SourcePhrase.h" +#include "Util.h" using namespace std; using namespace OnDiskPt; @@ -17,55 +18,6 @@ void usage(); typedef unsigned int uint; -void Tokenize(OnDiskPt::Phrase &phrase - , const std::string &token, bool addSourceNonTerm, bool addTargetNonTerm - , OnDiskPt::OnDiskWrapper &onDiskWrapper) -{ - - bool nonTerm = false; - size_t tokSize = token.size(); - int comStr =token.compare(0, 1, "["); - - if (comStr == 0) { - comStr = token.compare(tokSize - 1, 1, "]"); - nonTerm = comStr == 0; - } - - if (nonTerm) { - // non-term - size_t splitPos = token.find_first_of("[", 2); - string wordStr = token.substr(0, splitPos); - - if (splitPos == string::npos) { - // lhs - only 1 word - WordPtr word (new Word()); - word->CreateFromString(wordStr, onDiskWrapper.GetVocab()); - phrase.AddWord(word); - } else { - // source & target non-terms - if (addSourceNonTerm) { - WordPtr word( new Word()); - word->CreateFromString(wordStr, onDiskWrapper.GetVocab()); - phrase.AddWord(word); - } - - wordStr = token.substr(splitPos, tokSize - splitPos); - if (addTargetNonTerm) { - WordPtr word(new Word()); - word->CreateFromString(wordStr, onDiskWrapper.GetVocab()); - phrase.AddWord(word); - } - - } - } else { - // term - WordPtr word(new Word()); - word->CreateFromString(token, onDiskWrapper.GetVocab()); - phrase.AddWord(word); - } -} - - int main(int argc, char **argv) { int tableLimit = 20; @@ -89,52 +41,20 @@ int main(int argc, char **argv) if(ttable == "") usage(); - OnDiskWrapper onDiskWrapper; + OnDiskWrapper onDiskWrapper; bool retDb = onDiskWrapper.BeginLoad(ttable); - CHECK(retDb); - - cerr << "Ready..." << endl; - + CHECK(retDb); + + cerr << "Ready..." << endl; + std::string line; while(getline(std::cin, line)) { std::vector tokens; tokens = Moses::Tokenize(line, " "); - cerr << "line: " << line << endl; - - // create source phrase - SourcePhrase sourcePhrase; - - for (size_t pos = 0; pos < tokens.size(); ++pos) - { - const string &tok = tokens[pos]; - - if (pos == tokens.size() - 1) - { // last position. LHS non-term - Tokenize(sourcePhrase, tok, false, true, onDiskWrapper); - } - else - { - Tokenize(sourcePhrase, tok, true, true, onDiskWrapper); - } - } - - const PhraseNode *node = &onDiskWrapper.GetRootSourceNode(); - cerr << "node=" << node << endl; - assert(node); - - for (size_t pos = 0; pos < sourcePhrase.GetSize(); ++pos) - { - const Word &word = sourcePhrase.GetWord(pos); - cerr << word << " "; - node = node->GetChild(word, onDiskWrapper); - cerr << "node=" << node << endl; - - if (node == NULL) - { - break; - } - } + cerr << "line: " << line << endl; + SourcePhrase sourcePhrase = Tokenize(tokens, onDiskWrapper); + const PhraseNode* node = Query(sourcePhrase, onDiskWrapper); if (node) { // source phrase points to a bunch of rules @@ -148,8 +68,6 @@ int main(int argc, char **argv) cerr << " "; targetPhrase.DebugPrint(cerr, onDiskWrapper.GetVocab()); cerr << endl; - - } } else @@ -162,12 +80,11 @@ int main(int argc, char **argv) } cerr << "Finished." << endl; - } void usage() { - std::cerr << "Usage: queryOnDiskPt [-n ] [-a] -t \n" + std::cerr << "Usage: queryOnDiskPt [-n ] [-a] -t \n" "-tlimit max number of rules per source phrase (default: 20)\n" "-t phrase table\n"; exit(1); diff --git a/contrib/python/moses/cdictree.pxd b/contrib/python/moses/cdictree.pxd index 4a9446a27..3a8af4d26 100644 --- a/contrib/python/moses/cdictree.pxd +++ b/contrib/python/moses/cdictree.pxd @@ -1,6 +1,5 @@ from libcpp.string cimport string from libcpp.vector cimport vector -from libcpp.pair cimport pair ctypedef string* str_pointer ctypedef string* const_str_pointer "const str_pointer" @@ -16,8 +15,6 @@ cdef extern from 'PhraseDictionaryTree.h' namespace 'Moses': Tokens fnames Scores fvalues - - cdef cppclass PhraseDictionaryTree: PhraseDictionaryTree(unsigned nscores) void UseWordAlignment(bint use) diff --git a/contrib/python/moses/condiskpt.pxd b/contrib/python/moses/condiskpt.pxd new file mode 100644 index 000000000..b13954c56 --- /dev/null +++ b/contrib/python/moses/condiskpt.pxd @@ -0,0 +1,45 @@ +from libcpp.string cimport string +from libcpp.vector cimport vector + +cdef extern from 'Word.h' namespace 'OnDiskPt': + cdef cppclass Word + +cdef extern from 'Phrase.h' namespace 'OnDiskPt': + cdef cppclass Phrase + +cdef extern from 'SourcePhrase.h' namespace 'OnDiskPt': + cdef cppclass SourcePhrase + +#cdef extern from 'TargetPhrase.h' namespace 'OnDiskPt': +# cdef cppclass TargetPhrase + +#cdef extern from 'OnDiskWrapper.h' namespace 'OnDiskPt': +# cdef cppclass OnDiskWrapper + +cdef extern from 'TargetPhraseCollection.h' namespace 'OnDiskPt': + cdef cppclass TargetPhraseCollection + +cdef extern from 'PhraseNode.h' namespace 'OnDiskPt': + cdef cppclass PhraseNode +# cdef cppclass PhraseNodePointer 'PhaseNode*' + +cdef extern from 'OnDiskWrapper.h' namespace 'OnDiskPt': + cdef cppclass OnDiskWrapper + +cdef extern from 'PhraseNode.h' namespace 'OnDiskPt': + cdef cppclass PhraseNode: + PhraseNode* GetChild(Word& word, OnDiskWrapper& wrapper) + TargetPhraseCollection* GetTargetPhraseCollection(unsigned tableLimit, OnDiskWrapper& wrapper) + +cdef extern from 'OnDiskWrapper.h' namespace 'OnDiskPt': + + cdef cppclass OnDiskWrapper: + OnDiskWrapper() + bint BeginLoad(string& path) + PhraseNode& GetRootSourceNode() + +cdef extern from 'Util.h' namespace 'OnDiskPt': +# cdef void Tokenize(Phrase& phrase, string& token, bint addSourceNonTerm, bint addTargetNonTerm, OnDiskWrapper& wrapper) + cdef SourcePhrase Tokenize(vector[string]& tokens, OnDiskWrapper& wrapper) + + diff --git a/contrib/python/moses/dictree.cpp b/contrib/python/moses/dictree.cpp index a4b874387..4d8dcd15f 100644 --- a/contrib/python/moses/dictree.cpp +++ b/contrib/python/moses/dictree.cpp @@ -1,4 +1,4 @@ -/* Generated by Cython 0.17.1 on Tue Nov 13 11:38:08 2012 */ +/* Generated by Cython 0.17.1 on Tue Nov 13 17:26:02 2012 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -255,9 +255,14 @@ #include "stdexcept" #include "typeinfo" #include -#include #include "PhraseDictionaryTree.h" #include "Util.h" +#include "Word.h" +#include "Phrase.h" +#include "SourcePhrase.h" +#include "TargetPhraseCollection.h" +#include "PhraseNode.h" +#include "OnDiskWrapper.h" #ifdef _OPENMP #include #endif /* _OPENMP */ @@ -345,9 +350,10 @@ static const char *__pyx_filename; static const char *__pyx_f[] = { "dictree.pyx", + "stringsource", }; -/* "cdictree.pxd":8 +/* "cdictree.pxd":7 * ctypedef string* const_str_pointer "const str_pointer" * ctypedef vector[const_str_pointer] Tokens * ctypedef float FValue # <<<<<<<<<<<<<< @@ -357,11 +363,12 @@ static const char *__pyx_f[] = { typedef float __pyx_t_5moses_8cdictree_FValue; /*--- Type declarations ---*/ +struct __pyx_obj_5moses_7dictree_OnDiskWrapper; struct __pyx_obj_5moses_7dictree_PhraseDictionaryTree; struct __pyx_obj_5moses_7dictree_QueryResult; -/* "cdictree.pxd":5 - * from libcpp.pair cimport pair +/* "cdictree.pxd":4 + * from libcpp.vector cimport vector * * ctypedef string* str_pointer # <<<<<<<<<<<<<< * ctypedef string* const_str_pointer "const str_pointer" @@ -369,7 +376,7 @@ struct __pyx_obj_5moses_7dictree_QueryResult; */ typedef std::string *__pyx_t_5moses_8cdictree_str_pointer; -/* "cdictree.pxd":6 +/* "cdictree.pxd":5 * * ctypedef string* str_pointer * ctypedef string* const_str_pointer "const str_pointer" # <<<<<<<<<<<<<< @@ -378,7 +385,7 @@ typedef std::string *__pyx_t_5moses_8cdictree_str_pointer; */ typedef std::string *const str_pointer; -/* "cdictree.pxd":7 +/* "cdictree.pxd":6 * ctypedef string* str_pointer * ctypedef string* const_str_pointer "const str_pointer" * ctypedef vector[const_str_pointer] Tokens # <<<<<<<<<<<<<< @@ -387,7 +394,7 @@ typedef std::string *const str_pointer; */ typedef std::vector __pyx_t_5moses_8cdictree_Tokens; -/* "cdictree.pxd":9 +/* "cdictree.pxd":8 * ctypedef vector[const_str_pointer] Tokens * ctypedef float FValue * ctypedef vector[FValue] Scores # <<<<<<<<<<<<<< @@ -397,7 +404,7 @@ typedef std::vector __pyx_t_5moses_8cdictree_Tokens; typedef std::vector<__pyx_t_5moses_8cdictree_FValue> __pyx_t_5moses_8cdictree_Scores; struct __pyx_opt_args_5moses_7dictree_get_query_result; -/* "moses/dictree.pyx":59 +/* "moses/dictree.pyx":60 * return repr((repr(self.words), repr(self.scores), repr(self.wa))) * * cdef QueryResult get_query_result(cdictree.StringTgtCand& cand, object wa = None): # <<<<<<<<<<<<<< @@ -409,7 +416,21 @@ struct __pyx_opt_args_5moses_7dictree_get_query_result { PyObject *wa; }; -/* "moses/dictree.pyx":65 +/* "moses/dictree.pyx":139 + * return phrases + * + * cdef class OnDiskWrapper(object): # <<<<<<<<<<<<<< + * + * cdef condiskpt.OnDiskWrapper* wrapper + */ +struct __pyx_obj_5moses_7dictree_OnDiskWrapper { + PyObject_HEAD + OnDiskPt::OnDiskWrapper *wrapper; + PyObject *delimiters; +}; + + +/* "moses/dictree.pyx":66 * return QueryResult(words, scores, wa) * * cdef class PhraseDictionaryTree(object): # <<<<<<<<<<<<<< @@ -426,7 +447,7 @@ struct __pyx_obj_5moses_7dictree_PhraseDictionaryTree { }; -/* "moses/dictree.pyx":20 +/* "moses/dictree.pyx":21 * raise TypeError('Cannot convert %s to string' % type(data)) * * cdef class QueryResult(object): # <<<<<<<<<<<<<< @@ -760,18 +781,18 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Module declarations from 'cython' */ -/* Module declarations from 'libcpp.utility' */ - -/* Module declarations from 'libcpp.pair' */ - /* Module declarations from 'moses.cdictree' */ +/* Module declarations from 'moses.condiskpt' */ + /* Module declarations from 'moses.dictree' */ static PyTypeObject *__pyx_ptype_5moses_7dictree_QueryResult = 0; static PyTypeObject *__pyx_ptype_5moses_7dictree_PhraseDictionaryTree = 0; +static PyTypeObject *__pyx_ptype_5moses_7dictree_OnDiskWrapper = 0; static int __pyx_f_5moses_7dictree_fsign(float, int __pyx_skip_dispatch); /*proto*/ static PyObject *__pyx_f_5moses_7dictree_as_str(PyObject *); /*proto*/ static struct __pyx_obj_5moses_7dictree_QueryResult *__pyx_f_5moses_7dictree_get_query_result(struct Moses::StringTgtCand &, struct __pyx_opt_args_5moses_7dictree_get_query_result *__pyx_optional_args); /*proto*/ +static std::string __pyx_convert_string_from_py_(PyObject *); /*proto*/ #define __Pyx_MODULE_NAME "moses.dictree" int __pyx_module_is_main_moses__dictree = 0; @@ -799,6 +820,9 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_4path___get__(s static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_7nscores___get__(struct __pyx_obj_5moses_7dictree_PhraseDictionaryTree *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_2wa___get__(struct __pyx_obj_5moses_7dictree_PhraseDictionaryTree *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_10delimiters___get__(struct __pyx_obj_5moses_7dictree_PhraseDictionaryTree *__pyx_v_self); /* proto */ +static int __pyx_pf_5moses_7dictree_13OnDiskWrapper___cinit__(struct __pyx_obj_5moses_7dictree_OnDiskWrapper *__pyx_v_self, PyObject *__pyx_v_path, PyObject *__pyx_v_delimiters); /* proto */ +static PyObject *__pyx_pf_5moses_7dictree_13OnDiskWrapper_2query(struct __pyx_obj_5moses_7dictree_OnDiskWrapper *__pyx_v_self, PyObject *__pyx_v_line); /* proto */ +static PyObject *__pyx_pf_5moses_7dictree_13OnDiskWrapper_10delimiters___get__(struct __pyx_obj_5moses_7dictree_OnDiskWrapper *__pyx_v_self); /* proto */ static char __pyx_k_1[] = "UTF-8"; static char __pyx_k_3[] = "Cannot convert %s to string"; static char __pyx_k_5[] = " ||| "; @@ -813,7 +837,7 @@ static char __pyx_k_13[] = ".binphr.tgtvoc"; static char __pyx_k_14[] = ".binphr.srctree"; static char __pyx_k_15[] = ".binphr.tgtdata"; static char __pyx_k_17[] = "moses.dictree"; -static char __pyx_k_20[] = "/media/Data/tools/moses/mosesdecoder/contrib/python/moses/dictree.pyx"; +static char __pyx_k_20[] = "/opt/MLDAT_fast/USERS/waziz/tools/moses/contrib/python/moses/dictree.pyx"; static char __pyx_k__x[] = "x"; static char __pyx_k__y[] = "y"; static char __pyx_k__os[] = "os"; @@ -892,8 +916,8 @@ static PyObject *__pyx_k_tuple_21; static PyObject *__pyx_k_codeobj_19; static PyObject *__pyx_k_codeobj_22; -/* "moses/dictree.pyx":9 - * cimport cdictree +/* "moses/dictree.pyx":10 + * cimport condiskpt * * cpdef int fsign(float x): # <<<<<<<<<<<<<< * """Simply returns the sign of float x (zero is assumed +), it's defined here just so one gains a little bit with static typing""" @@ -907,7 +931,7 @@ static int __pyx_f_5moses_7dictree_fsign(float __pyx_v_x, CYTHON_UNUSED int __py long __pyx_t_1; __Pyx_RefNannySetupContext("fsign", 0); - /* "moses/dictree.pyx":11 + /* "moses/dictree.pyx":12 * cpdef int fsign(float x): * """Simply returns the sign of float x (zero is assumed +), it's defined here just so one gains a little bit with static typing""" * return 1 if x >= 0 else -1 # <<<<<<<<<<<<<< @@ -937,7 +961,7 @@ static PyObject *__pyx_pw_5moses_7dictree_1fsign(PyObject *__pyx_self, PyObject __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("fsign (wrapper)", 0); assert(__pyx_arg_x); { - __pyx_v_x = __pyx_PyFloat_AsFloat(__pyx_arg_x); if (unlikely((__pyx_v_x == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_v_x = __pyx_PyFloat_AsFloat(__pyx_arg_x); if (unlikely((__pyx_v_x == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 10; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } goto __pyx_L4_argument_unpacking_done; __pyx_L3_error:; @@ -950,8 +974,8 @@ static PyObject *__pyx_pw_5moses_7dictree_1fsign(PyObject *__pyx_self, PyObject return __pyx_r; } -/* "moses/dictree.pyx":9 - * cimport cdictree +/* "moses/dictree.pyx":10 + * cimport condiskpt * * cpdef int fsign(float x): # <<<<<<<<<<<<<< * """Simply returns the sign of float x (zero is assumed +), it's defined here just so one gains a little bit with static typing""" @@ -967,7 +991,7 @@ static PyObject *__pyx_pf_5moses_7dictree_fsign(CYTHON_UNUSED PyObject *__pyx_se int __pyx_clineno = 0; __Pyx_RefNannySetupContext("fsign", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyInt_FromLong(__pyx_f_5moses_7dictree_fsign(__pyx_v_x, 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyInt_FromLong(__pyx_f_5moses_7dictree_fsign(__pyx_v_x, 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 10; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -985,7 +1009,7 @@ static PyObject *__pyx_pf_5moses_7dictree_fsign(CYTHON_UNUSED PyObject *__pyx_se return __pyx_r; } -/* "moses/dictree.pyx":13 +/* "moses/dictree.pyx":14 * return 1 if x >= 0 else -1 * * cdef bytes as_str(data): # <<<<<<<<<<<<<< @@ -1004,7 +1028,7 @@ static PyObject *__pyx_f_5moses_7dictree_as_str(PyObject *__pyx_v_data) { int __pyx_clineno = 0; __Pyx_RefNannySetupContext("as_str", 0); - /* "moses/dictree.pyx":14 + /* "moses/dictree.pyx":15 * * cdef bytes as_str(data): * if isinstance(data, bytes): # <<<<<<<<<<<<<< @@ -1017,7 +1041,7 @@ static PyObject *__pyx_f_5moses_7dictree_as_str(PyObject *__pyx_v_data) { __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_2) { - /* "moses/dictree.pyx":15 + /* "moses/dictree.pyx":16 * cdef bytes as_str(data): * if isinstance(data, bytes): * return data # <<<<<<<<<<<<<< @@ -1025,14 +1049,14 @@ static PyObject *__pyx_f_5moses_7dictree_as_str(PyObject *__pyx_v_data) { * return data.encode('UTF-8') */ __Pyx_XDECREF(((PyObject *)__pyx_r)); - if (!(likely(PyBytes_CheckExact(__pyx_v_data))||((__pyx_v_data) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_v_data)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(PyBytes_CheckExact(__pyx_v_data))||((__pyx_v_data) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_v_data)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 16; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_INCREF(__pyx_v_data); __pyx_r = ((PyObject*)__pyx_v_data); goto __pyx_L0; goto __pyx_L3; } - /* "moses/dictree.pyx":16 + /* "moses/dictree.pyx":17 * if isinstance(data, bytes): * return data * elif isinstance(data, unicode): # <<<<<<<<<<<<<< @@ -1045,7 +1069,7 @@ static PyObject *__pyx_f_5moses_7dictree_as_str(PyObject *__pyx_v_data) { __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_2) { - /* "moses/dictree.pyx":17 + /* "moses/dictree.pyx":18 * return data * elif isinstance(data, unicode): * return data.encode('UTF-8') # <<<<<<<<<<<<<< @@ -1053,12 +1077,12 @@ static PyObject *__pyx_f_5moses_7dictree_as_str(PyObject *__pyx_v_data) { * */ __Pyx_XDECREF(((PyObject *)__pyx_r)); - __pyx_t_1 = PyObject_GetAttr(__pyx_v_data, __pyx_n_s__encode); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 17; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_v_data, __pyx_n_s__encode); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_k_tuple_2), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 17; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_k_tuple_2), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (!(likely(PyBytes_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_t_3)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 17; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(PyBytes_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_t_3)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_r = ((PyObject*)__pyx_t_3); __pyx_t_3 = 0; goto __pyx_L0; @@ -1066,26 +1090,26 @@ static PyObject *__pyx_f_5moses_7dictree_as_str(PyObject *__pyx_v_data) { } __pyx_L3:; - /* "moses/dictree.pyx":18 + /* "moses/dictree.pyx":19 * elif isinstance(data, unicode): * return data.encode('UTF-8') * raise TypeError('Cannot convert %s to string' % type(data)) # <<<<<<<<<<<<<< * * cdef class QueryResult(object): */ - __pyx_t_3 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_3), ((PyObject *)Py_TYPE(__pyx_v_data))); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_3), ((PyObject *)Py_TYPE(__pyx_v_data))); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_3)); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_t_3)); __Pyx_GIVEREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_builtin_TypeError, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_builtin_TypeError, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_r = ((PyObject*)Py_None); __Pyx_INCREF(Py_None); goto __pyx_L0; @@ -1113,7 +1137,7 @@ static int __pyx_pw_5moses_7dictree_11QueryResult_1__cinit__(PyObject *__pyx_v_s static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__words,&__pyx_n_s__scores,&__pyx_n_s__wa,0}; PyObject* values[3] = {0,0,0}; - /* "moses/dictree.pyx":32 + /* "moses/dictree.pyx":33 * cdef readonly bytes wa * * def __cinit__(self, words, scores, wa = None): # <<<<<<<<<<<<<< @@ -1139,7 +1163,7 @@ static int __pyx_pw_5moses_7dictree_11QueryResult_1__cinit__(PyObject *__pyx_v_s case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__scores)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 33; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 2: if (kw_args > 0) { @@ -1148,7 +1172,7 @@ static int __pyx_pw_5moses_7dictree_11QueryResult_1__cinit__(PyObject *__pyx_v_s } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 33; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -1165,7 +1189,7 @@ static int __pyx_pw_5moses_7dictree_11QueryResult_1__cinit__(PyObject *__pyx_v_s } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 33; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("moses.dictree.QueryResult.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -1186,19 +1210,19 @@ static int __pyx_pf_5moses_7dictree_11QueryResult___cinit__(struct __pyx_obj_5mo int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__cinit__", 0); - /* "moses/dictree.pyx":35 + /* "moses/dictree.pyx":36 * """Requires a tuple of words (as strings) and a tuple of scores (as floats). * Word-alignment info (as string) may be provided""" * self.words = tuple(words) # <<<<<<<<<<<<<< * self.scores = tuple(scores) * self.wa = wa */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 35; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_v_words); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_words); __Pyx_GIVEREF(__pyx_v_words); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)(&PyTuple_Type))), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 35; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)(&PyTuple_Type))), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_GIVEREF(__pyx_t_2); @@ -1207,19 +1231,19 @@ static int __pyx_pf_5moses_7dictree_11QueryResult___cinit__(struct __pyx_obj_5mo __pyx_v_self->words = ((PyObject*)__pyx_t_2); __pyx_t_2 = 0; - /* "moses/dictree.pyx":36 + /* "moses/dictree.pyx":37 * Word-alignment info (as string) may be provided""" * self.words = tuple(words) * self.scores = tuple(scores) # <<<<<<<<<<<<<< * self.wa = wa * */ - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_v_scores); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_scores); __Pyx_GIVEREF(__pyx_v_scores); - __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)(&PyTuple_Type))), ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)(&PyTuple_Type))), ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_GIVEREF(__pyx_t_1); @@ -1228,14 +1252,14 @@ static int __pyx_pf_5moses_7dictree_11QueryResult___cinit__(struct __pyx_obj_5mo __pyx_v_self->scores = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "moses/dictree.pyx":37 + /* "moses/dictree.pyx":38 * self.words = tuple(words) * self.scores = tuple(scores) * self.wa = wa # <<<<<<<<<<<<<< * * @staticmethod */ - if (!(likely(PyBytes_CheckExact(__pyx_v_wa))||((__pyx_v_wa) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_v_wa)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(PyBytes_CheckExact(__pyx_v_wa))||((__pyx_v_wa) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_v_wa)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_INCREF(__pyx_v_wa); __Pyx_GIVEREF(__pyx_v_wa); __Pyx_GOTREF(__pyx_v_self->wa); @@ -1287,7 +1311,7 @@ static PyObject *__pyx_pw_5moses_7dictree_11QueryResult_3desc(CYTHON_UNUSED PyOb case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__y)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("desc", 0, 2, 3, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("desc", 0, 2, 3, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 2: if (kw_args > 0) { @@ -1296,7 +1320,7 @@ static PyObject *__pyx_pw_5moses_7dictree_11QueryResult_3desc(CYTHON_UNUSED PyOb } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "desc") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "desc") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -1313,7 +1337,7 @@ static PyObject *__pyx_pw_5moses_7dictree_11QueryResult_3desc(CYTHON_UNUSED PyOb } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("desc", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("desc", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("moses.dictree.QueryResult.desc", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -1336,7 +1360,7 @@ static PyObject *__pyx_pw_5moses_7dictree_11QueryResult_4desc_lambda1(PyObject * return __pyx_r; } -/* "moses/dictree.pyx":40 +/* "moses/dictree.pyx":41 * * @staticmethod * def desc(x, y, key = lambda r: r.scores[0]): # <<<<<<<<<<<<<< @@ -1354,9 +1378,9 @@ static PyObject *__pyx_lambda_funcdef_lambda1(CYTHON_UNUSED PyObject *__pyx_self int __pyx_clineno = 0; __Pyx_RefNannySetupContext("lambda1", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_GetAttr(__pyx_v_r, __pyx_n_s__scores); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_v_r, __pyx_n_s__scores); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_GetItemInt(__pyx_t_1, 0, sizeof(long), PyInt_FromLong); if (!__pyx_t_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_GetItemInt(__pyx_t_1, 0, sizeof(long), PyInt_FromLong); if (!__pyx_t_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_2; @@ -1388,7 +1412,7 @@ static PyObject *__pyx_pf_5moses_7dictree_11QueryResult_2desc(PyObject *__pyx_v_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("desc", 0); - /* "moses/dictree.pyx":44 + /* "moses/dictree.pyx":45 * Can only be used if scores is not an empty vector as * keys defaults to scores[0]""" * return fsign(key(y) - key(x)) # <<<<<<<<<<<<<< @@ -1396,29 +1420,29 @@ static PyObject *__pyx_pf_5moses_7dictree_11QueryResult_2desc(PyObject *__pyx_v_ * def __str__(self): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_v_y); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_y); __Pyx_GIVEREF(__pyx_v_y); - __pyx_t_2 = PyObject_Call(__pyx_v_key, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(__pyx_v_key, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_v_x); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_x); __Pyx_GIVEREF(__pyx_v_x); - __pyx_t_3 = PyObject_Call(__pyx_v_key, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_v_key, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __pyx_t_1 = PyNumber_Subtract(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyNumber_Subtract(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_4 = __pyx_PyFloat_AsFloat(__pyx_t_1); if (unlikely((__pyx_t_4 == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __pyx_PyFloat_AsFloat(__pyx_t_1); if (unlikely((__pyx_t_4 == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyInt_FromLong(__pyx_f_5moses_7dictree_fsign(__pyx_t_4, 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyInt_FromLong(__pyx_f_5moses_7dictree_fsign(__pyx_t_4, 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -1453,7 +1477,7 @@ static PyObject *__pyx_pw_5moses_7dictree_11QueryResult_5__str__(PyObject *__pyx return __pyx_r; } -/* "moses/dictree.pyx":46 +/* "moses/dictree.pyx":47 * return fsign(key(y) - key(x)) * * def __str__(self): # <<<<<<<<<<<<<< @@ -1479,7 +1503,7 @@ static PyObject *__pyx_pf_5moses_7dictree_11QueryResult_4__str__(struct __pyx_ob int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__str__", 0); - /* "moses/dictree.pyx":48 + /* "moses/dictree.pyx":49 * def __str__(self): * """Returns a string such as: ||| [||| word-alignment info]""" * if self.wa: # <<<<<<<<<<<<<< @@ -1489,7 +1513,7 @@ static PyObject *__pyx_pf_5moses_7dictree_11QueryResult_4__str__(struct __pyx_ob __pyx_t_1 = (((PyObject *)__pyx_v_self->wa) != Py_None) && (PyBytes_GET_SIZE(((PyObject *)__pyx_v_self->wa)) != 0); if (__pyx_t_1) { - /* "moses/dictree.pyx":49 + /* "moses/dictree.pyx":50 * """Returns a string such as: ||| [||| word-alignment info]""" * if self.wa: * return ' ||| '.join( (' '.join(self.words), # <<<<<<<<<<<<<< @@ -1497,77 +1521,77 @@ static PyObject *__pyx_pf_5moses_7dictree_11QueryResult_4__str__(struct __pyx_ob * self.wa) ) */ __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_5), __pyx_n_s__join); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_5), __pyx_n_s__join); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_6), __pyx_n_s__join); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_6), __pyx_n_s__join); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_INCREF(((PyObject *)__pyx_v_self->words)); PyTuple_SET_ITEM(__pyx_t_4, 0, ((PyObject *)__pyx_v_self->words)); __Pyx_GIVEREF(((PyObject *)__pyx_v_self->words)); - __pyx_t_5 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; - /* "moses/dictree.pyx":50 + /* "moses/dictree.pyx":51 * if self.wa: * return ' ||| '.join( (' '.join(self.words), * ' '.join([str(x) for x in self.scores]), # <<<<<<<<<<<<<< * self.wa) ) * else: */ - __pyx_t_4 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_6), __pyx_n_s__join); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_6), __pyx_n_s__join); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); - __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); if (unlikely(((PyObject *)__pyx_v_self->scores) == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __pyx_t_6 = ((PyObject *)__pyx_v_self->scores); __Pyx_INCREF(__pyx_t_6); __pyx_t_7 = 0; for (;;) { if (__pyx_t_7 >= PyTuple_GET_SIZE(__pyx_t_6)) break; #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_8 = PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_7); __Pyx_INCREF(__pyx_t_8); __pyx_t_7++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_7); __Pyx_INCREF(__pyx_t_8); __pyx_t_7++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #else - __pyx_t_8 = PySequence_ITEM(__pyx_t_6, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PySequence_ITEM(__pyx_t_6, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif __Pyx_XDECREF(__pyx_v_x); __pyx_v_x = __pyx_t_8; __pyx_t_8 = 0; - __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_v_x); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_v_x); __Pyx_GIVEREF(__pyx_v_x); - __pyx_t_9 = PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), ((PyObject *)__pyx_t_8), NULL); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_9 = PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), ((PyObject *)__pyx_t_8), NULL); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_9); __Pyx_DECREF(((PyObject *)__pyx_t_8)); __pyx_t_8 = 0; - if (unlikely(__Pyx_PyList_Append(__pyx_t_3, (PyObject*)__pyx_t_9))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(__Pyx_PyList_Append(__pyx_t_3, (PyObject*)__pyx_t_9))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_INCREF(((PyObject *)__pyx_t_3)); PyTuple_SET_ITEM(__pyx_t_6, 0, ((PyObject *)__pyx_t_3)); __Pyx_GIVEREF(((PyObject *)__pyx_t_3)); __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_4, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_4, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; - /* "moses/dictree.pyx":51 + /* "moses/dictree.pyx":52 * return ' ||| '.join( (' '.join(self.words), * ' '.join([str(x) for x in self.scores]), * self.wa) ) # <<<<<<<<<<<<<< * else: * return ' ||| '.join( (' '.join(self.words), */ - __pyx_t_6 = PyTuple_New(3); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyTuple_New(3); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_5); __Pyx_GIVEREF(__pyx_t_5); @@ -1578,12 +1602,12 @@ static PyObject *__pyx_pf_5moses_7dictree_11QueryResult_4__str__(struct __pyx_ob __Pyx_GIVEREF(((PyObject *)__pyx_v_self->wa)); __pyx_t_5 = 0; __pyx_t_3 = 0; - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_3, 0, ((PyObject *)__pyx_t_6)); __Pyx_GIVEREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; - __pyx_t_6 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; @@ -1594,7 +1618,7 @@ static PyObject *__pyx_pf_5moses_7dictree_11QueryResult_4__str__(struct __pyx_ob } /*else*/ { - /* "moses/dictree.pyx":53 + /* "moses/dictree.pyx":54 * self.wa) ) * else: * return ' ||| '.join( (' '.join(self.words), # <<<<<<<<<<<<<< @@ -1602,69 +1626,69 @@ static PyObject *__pyx_pf_5moses_7dictree_11QueryResult_4__str__(struct __pyx_ob * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_5), __pyx_n_s__join); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_5), __pyx_n_s__join); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_3 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_6), __pyx_n_s__join); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_6), __pyx_n_s__join); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(((PyObject *)__pyx_v_self->words)); PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)__pyx_v_self->words)); __Pyx_GIVEREF(((PyObject *)__pyx_v_self->words)); - __pyx_t_5 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - /* "moses/dictree.pyx":54 + /* "moses/dictree.pyx":55 * else: * return ' ||| '.join( (' '.join(self.words), * ' '.join([str(x) for x in self.scores]) ) ) # <<<<<<<<<<<<<< * * def __repr__(self): */ - __pyx_t_2 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_6), __pyx_n_s__join); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_6), __pyx_n_s__join); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); if (unlikely(((PyObject *)__pyx_v_self->scores) == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __pyx_t_4 = ((PyObject *)__pyx_v_self->scores); __Pyx_INCREF(__pyx_t_4); __pyx_t_7 = 0; for (;;) { if (__pyx_t_7 >= PyTuple_GET_SIZE(__pyx_t_4)) break; #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_9 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_7); __Pyx_INCREF(__pyx_t_9); __pyx_t_7++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_9 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_7); __Pyx_INCREF(__pyx_t_9); __pyx_t_7++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #else - __pyx_t_9 = PySequence_ITEM(__pyx_t_4, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_9 = PySequence_ITEM(__pyx_t_4, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif __Pyx_XDECREF(__pyx_v_x); __pyx_v_x = __pyx_t_9; __pyx_t_9 = 0; - __pyx_t_9 = PyTuple_New(1); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_9 = PyTuple_New(1); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_9); __Pyx_INCREF(__pyx_v_x); PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_v_x); __Pyx_GIVEREF(__pyx_v_x); - __pyx_t_8 = PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), ((PyObject *)__pyx_t_9), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), ((PyObject *)__pyx_t_9), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(((PyObject *)__pyx_t_9)); __pyx_t_9 = 0; - if (unlikely(__Pyx_PyList_Append(__pyx_t_3, (PyObject*)__pyx_t_8))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(__Pyx_PyList_Append(__pyx_t_3, (PyObject*)__pyx_t_8))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_INCREF(((PyObject *)__pyx_t_3)); PyTuple_SET_ITEM(__pyx_t_4, 0, ((PyObject *)__pyx_t_3)); __Pyx_GIVEREF(((PyObject *)__pyx_t_3)); __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; - __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_5); __Pyx_GIVEREF(__pyx_t_5); @@ -1672,12 +1696,12 @@ static PyObject *__pyx_pf_5moses_7dictree_11QueryResult_4__str__(struct __pyx_ob __Pyx_GIVEREF(__pyx_t_3); __pyx_t_5 = 0; __pyx_t_3 = 0; - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_3, 0, ((PyObject *)__pyx_t_4)); __Pyx_GIVEREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; - __pyx_t_4 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; @@ -1717,7 +1741,7 @@ static PyObject *__pyx_pw_5moses_7dictree_11QueryResult_7__repr__(PyObject *__py return __pyx_r; } -/* "moses/dictree.pyx":56 +/* "moses/dictree.pyx":57 * ' '.join([str(x) for x in self.scores]) ) ) * * def __repr__(self): # <<<<<<<<<<<<<< @@ -1737,7 +1761,7 @@ static PyObject *__pyx_pf_5moses_7dictree_11QueryResult_6__repr__(struct __pyx_o int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__repr__", 0); - /* "moses/dictree.pyx":57 + /* "moses/dictree.pyx":58 * * def __repr__(self): * return repr((repr(self.words), repr(self.scores), repr(self.wa))) # <<<<<<<<<<<<<< @@ -1747,20 +1771,20 @@ static PyObject *__pyx_pf_5moses_7dictree_11QueryResult_6__repr__(struct __pyx_o __Pyx_XDECREF(__pyx_r); __pyx_t_1 = ((PyObject *)__pyx_v_self->words); __Pyx_INCREF(__pyx_t_1); - __pyx_t_2 = PyObject_Repr(__pyx_t_1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Repr(__pyx_t_1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = ((PyObject *)__pyx_v_self->scores); __Pyx_INCREF(__pyx_t_1); - __pyx_t_3 = PyObject_Repr(__pyx_t_1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Repr(__pyx_t_1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = ((PyObject *)__pyx_v_self->wa); __Pyx_INCREF(__pyx_t_1); - __pyx_t_4 = PyObject_Repr(__pyx_t_1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_Repr(__pyx_t_1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_2); __Pyx_GIVEREF(__pyx_t_2); @@ -1771,7 +1795,7 @@ static PyObject *__pyx_pf_5moses_7dictree_11QueryResult_6__repr__(struct __pyx_o __pyx_t_2 = 0; __pyx_t_3 = 0; __pyx_t_4 = 0; - __pyx_t_4 = PyObject_Repr(((PyObject *)__pyx_t_1)); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_Repr(((PyObject *)__pyx_t_1)); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __pyx_r = __pyx_t_4; @@ -1804,7 +1828,7 @@ static PyObject *__pyx_pw_5moses_7dictree_11QueryResult_5words_1__get__(PyObject return __pyx_r; } -/* "moses/dictree.pyx":28 +/* "moses/dictree.pyx":29 * used as is, threfore saving some time.""" * * cdef readonly tuple words # <<<<<<<<<<<<<< @@ -1839,7 +1863,7 @@ static PyObject *__pyx_pw_5moses_7dictree_11QueryResult_6scores_1__get__(PyObjec return __pyx_r; } -/* "moses/dictree.pyx":29 +/* "moses/dictree.pyx":30 * * cdef readonly tuple words * cdef readonly tuple scores # <<<<<<<<<<<<<< @@ -1874,7 +1898,7 @@ static PyObject *__pyx_pw_5moses_7dictree_11QueryResult_2wa_1__get__(PyObject *_ return __pyx_r; } -/* "moses/dictree.pyx":30 +/* "moses/dictree.pyx":31 * cdef readonly tuple words * cdef readonly tuple scores * cdef readonly bytes wa # <<<<<<<<<<<<<< @@ -1898,7 +1922,7 @@ static PyObject *__pyx_pf_5moses_7dictree_11QueryResult_2wa___get__(struct __pyx return __pyx_r; } -/* "moses/dictree.pyx":59 +/* "moses/dictree.pyx":60 * return repr((repr(self.words), repr(self.scores), repr(self.wa))) * * cdef QueryResult get_query_result(cdictree.StringTgtCand& cand, object wa = None): # <<<<<<<<<<<<<< @@ -1931,35 +1955,35 @@ static struct __pyx_obj_5moses_7dictree_QueryResult *__pyx_f_5moses_7dictree_get } } - /* "moses/dictree.pyx":61 + /* "moses/dictree.pyx":62 * cdef QueryResult get_query_result(cdictree.StringTgtCand& cand, object wa = None): * """Converts a StringTgtCandidate (c++ object) and possibly a word-alignment info (string) to a QueryResult (python object).""" * cdef list words = [cand.tokens[i].c_str() for i in xrange(cand.tokens.size())] # <<<<<<<<<<<<<< * cdef list scores = [score for score in cand.scores] * return QueryResult(words, scores, wa) */ - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __pyx_v_cand.tokens.size(); for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - __pyx_t_4 = PyBytes_FromString((__pyx_v_cand.tokens[__pyx_v_i])->c_str()); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyBytes_FromString((__pyx_v_cand.tokens[__pyx_v_i])->c_str()); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_4)); - if (unlikely(__Pyx_PyList_Append(__pyx_t_1, (PyObject*)__pyx_t_4))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(__Pyx_PyList_Append(__pyx_t_1, (PyObject*)__pyx_t_4))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; } __Pyx_INCREF(((PyObject *)__pyx_t_1)); __pyx_v_words = __pyx_t_1; __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - /* "moses/dictree.pyx":62 + /* "moses/dictree.pyx":63 * """Converts a StringTgtCandidate (c++ object) and possibly a word-alignment info (string) to a QueryResult (python object).""" * cdef list words = [cand.tokens[i].c_str() for i in xrange(cand.tokens.size())] * cdef list scores = [score for score in cand.scores] # <<<<<<<<<<<<<< * return QueryResult(words, scores, wa) * */ - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_t_6 = &__pyx_v_cand.scores; __pyx_t_5 = __pyx_t_6->begin(); @@ -1967,18 +1991,18 @@ static struct __pyx_obj_5moses_7dictree_QueryResult *__pyx_f_5moses_7dictree_get if (!(__pyx_t_5 != __pyx_t_6->end())) break; __pyx_t_7 = *__pyx_t_5; ++__pyx_t_5; - __pyx_t_4 = PyFloat_FromDouble(__pyx_t_7); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyFloat_FromDouble(__pyx_t_7); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_XDECREF(__pyx_v_score); __pyx_v_score = __pyx_t_4; __pyx_t_4 = 0; - if (unlikely(__Pyx_PyList_Append(__pyx_t_1, (PyObject*)__pyx_v_score))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(__Pyx_PyList_Append(__pyx_t_1, (PyObject*)__pyx_v_score))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __Pyx_INCREF(((PyObject *)__pyx_t_1)); __pyx_v_scores = __pyx_t_1; __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - /* "moses/dictree.pyx":63 + /* "moses/dictree.pyx":64 * cdef list words = [cand.tokens[i].c_str() for i in xrange(cand.tokens.size())] * cdef list scores = [score for score in cand.scores] * return QueryResult(words, scores, wa) # <<<<<<<<<<<<<< @@ -1986,7 +2010,7 @@ static struct __pyx_obj_5moses_7dictree_QueryResult *__pyx_f_5moses_7dictree_get * cdef class PhraseDictionaryTree(object): */ __Pyx_XDECREF(((PyObject *)__pyx_r)); - __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(((PyObject *)__pyx_v_words)); PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_v_words)); @@ -1997,7 +2021,7 @@ static struct __pyx_obj_5moses_7dictree_QueryResult *__pyx_f_5moses_7dictree_get __Pyx_INCREF(__pyx_v_wa); PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_wa); __Pyx_GIVEREF(__pyx_v_wa); - __pyx_t_4 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5moses_7dictree_QueryResult)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5moses_7dictree_QueryResult)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __pyx_r = ((struct __pyx_obj_5moses_7dictree_QueryResult *)__pyx_t_4); @@ -2067,7 +2091,7 @@ static int __pyx_pw_5moses_7dictree_20PhraseDictionaryTree_1__cinit__(PyObject * } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -2081,15 +2105,15 @@ static int __pyx_pw_5moses_7dictree_20PhraseDictionaryTree_1__cinit__(PyObject * } __pyx_v_path = ((PyObject*)values[0]); if (values[1]) { - __pyx_v_nscores = __Pyx_PyInt_AsUnsignedInt(values[1]); if (unlikely((__pyx_v_nscores == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_v_nscores = __Pyx_PyInt_AsUnsignedInt(values[1]); if (unlikely((__pyx_v_nscores == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } else { __pyx_v_nscores = ((unsigned int)5); } if (values[2]) { - __pyx_v_wa = __Pyx_PyObject_IsTrue(values[2]); if (unlikely((__pyx_v_wa == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_v_wa = __Pyx_PyObject_IsTrue(values[2]); if (unlikely((__pyx_v_wa == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } else { - /* "moses/dictree.pyx":75 + /* "moses/dictree.pyx":76 * cdef readonly bytes delimiters * * def __cinit__(self, bytes path, unsigned nscores = 5, bint wa = False, delimiters = ' \t'): # <<<<<<<<<<<<<< @@ -2102,13 +2126,13 @@ static int __pyx_pw_5moses_7dictree_20PhraseDictionaryTree_1__cinit__(PyObject * } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 1, 4, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 1, 4, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("moses.dictree.PhraseDictionaryTree.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return -1; __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_path), (&PyBytes_Type), 1, "path", 1))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_path), (&PyBytes_Type), 1, "path", 1))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_r = __pyx_pf_5moses_7dictree_20PhraseDictionaryTree___cinit__(((struct __pyx_obj_5moses_7dictree_PhraseDictionaryTree *)__pyx_v_self), __pyx_v_path, __pyx_v_nscores, __pyx_v_wa, __pyx_v_delimiters); goto __pyx_L0; __pyx_L1_error:; @@ -2126,25 +2150,24 @@ static int __pyx_pf_5moses_7dictree_20PhraseDictionaryTree___cinit__(struct __py PyObject *__pyx_t_3 = NULL; int __pyx_t_4; int __pyx_t_5; - char *__pyx_t_6; - std::string __pyx_t_7; + std::string __pyx_t_6; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__cinit__", 0); - /* "moses/dictree.pyx":82 + /* "moses/dictree.pyx":83 * One can also specify the token delimiters, for Moses::Tokenize(text, delimiters), which is space or tab by default.""" * * if not PhraseDictionaryTree.isValidBinaryTable(path, wa): # <<<<<<<<<<<<<< * raise ValueError, "'%s' doesn't seem a valid binary table." % path * self.path = path */ - __pyx_t_1 = PyObject_GetAttr(((PyObject *)((PyObject*)__pyx_ptype_5moses_7dictree_PhraseDictionaryTree)), __pyx_n_s__isValidBinaryTable); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(((PyObject *)((PyObject*)__pyx_ptype_5moses_7dictree_PhraseDictionaryTree)), __pyx_n_s__isValidBinaryTable); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_v_wa); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_v_wa); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(((PyObject *)__pyx_v_path)); PyTuple_SET_ITEM(__pyx_t_3, 0, ((PyObject *)__pyx_v_path)); @@ -2152,32 +2175,32 @@ static int __pyx_pf_5moses_7dictree_20PhraseDictionaryTree___cinit__(struct __py PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2); __Pyx_GIVEREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_5 = (!__pyx_t_4); if (__pyx_t_5) { - /* "moses/dictree.pyx":83 + /* "moses/dictree.pyx":84 * * if not PhraseDictionaryTree.isValidBinaryTable(path, wa): * raise ValueError, "'%s' doesn't seem a valid binary table." % path # <<<<<<<<<<<<<< * self.path = path * self.nscores = nscores */ - __pyx_t_2 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_8), ((PyObject *)__pyx_v_path)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_8), ((PyObject *)__pyx_v_path)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_Raise(__pyx_builtin_ValueError, ((PyObject *)__pyx_t_2), 0, 0); __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L3; } __pyx_L3:; - /* "moses/dictree.pyx":84 + /* "moses/dictree.pyx":85 * if not PhraseDictionaryTree.isValidBinaryTable(path, wa): * raise ValueError, "'%s' doesn't seem a valid binary table." % path * self.path = path # <<<<<<<<<<<<<< @@ -2190,7 +2213,7 @@ static int __pyx_pf_5moses_7dictree_20PhraseDictionaryTree___cinit__(struct __py __Pyx_DECREF(((PyObject *)__pyx_v_self->path)); __pyx_v_self->path = __pyx_v_path; - /* "moses/dictree.pyx":85 + /* "moses/dictree.pyx":86 * raise ValueError, "'%s' doesn't seem a valid binary table." % path * self.path = path * self.nscores = nscores # <<<<<<<<<<<<<< @@ -2199,7 +2222,7 @@ static int __pyx_pf_5moses_7dictree_20PhraseDictionaryTree___cinit__(struct __py */ __pyx_v_self->nscores = __pyx_v_nscores; - /* "moses/dictree.pyx":86 + /* "moses/dictree.pyx":87 * self.path = path * self.nscores = nscores * self.wa = wa # <<<<<<<<<<<<<< @@ -2208,48 +2231,47 @@ static int __pyx_pf_5moses_7dictree_20PhraseDictionaryTree___cinit__(struct __py */ __pyx_v_self->wa = __pyx_v_wa; - /* "moses/dictree.pyx":87 + /* "moses/dictree.pyx":88 * self.nscores = nscores * self.wa = wa * self.delimiters = delimiters # <<<<<<<<<<<<<< * self.__tree = new cdictree.PhraseDictionaryTree(nscores) * self.__tree.UseWordAlignment(wa) */ - if (!(likely(PyBytes_CheckExact(__pyx_v_delimiters))||((__pyx_v_delimiters) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_v_delimiters)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 87; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(PyBytes_CheckExact(__pyx_v_delimiters))||((__pyx_v_delimiters) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_v_delimiters)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_INCREF(__pyx_v_delimiters); __Pyx_GIVEREF(__pyx_v_delimiters); __Pyx_GOTREF(__pyx_v_self->delimiters); __Pyx_DECREF(((PyObject *)__pyx_v_self->delimiters)); __pyx_v_self->delimiters = ((PyObject*)__pyx_v_delimiters); - /* "moses/dictree.pyx":88 + /* "moses/dictree.pyx":89 * self.wa = wa * self.delimiters = delimiters * self.__tree = new cdictree.PhraseDictionaryTree(nscores) # <<<<<<<<<<<<<< * self.__tree.UseWordAlignment(wa) - * self.__tree.Read(string(path)) + * self.__tree.Read(path) */ __pyx_v_self->__pyx___tree = new Moses::PhraseDictionaryTree(__pyx_v_nscores); - /* "moses/dictree.pyx":89 + /* "moses/dictree.pyx":90 * self.delimiters = delimiters * self.__tree = new cdictree.PhraseDictionaryTree(nscores) * self.__tree.UseWordAlignment(wa) # <<<<<<<<<<<<<< - * self.__tree.Read(string(path)) + * self.__tree.Read(path) * */ __pyx_v_self->__pyx___tree->UseWordAlignment(__pyx_v_wa); - /* "moses/dictree.pyx":90 + /* "moses/dictree.pyx":91 * self.__tree = new cdictree.PhraseDictionaryTree(nscores) * self.__tree.UseWordAlignment(wa) - * self.__tree.Read(string(path)) # <<<<<<<<<<<<<< + * self.__tree.Read(path) # <<<<<<<<<<<<<< * * def __dealloc__(self): */ - __pyx_t_6 = PyBytes_AsString(((PyObject *)__pyx_v_path)); if (unlikely((!__pyx_t_6) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - try {__pyx_t_7 = std::string(__pyx_t_6);} catch(...) {__Pyx_CppExn2PyErr(); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}} - __pyx_v_self->__pyx___tree->Read(__pyx_t_7); + __pyx_t_6 = __pyx_convert_string_from_py_(((PyObject *)__pyx_v_path)); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_self->__pyx___tree->Read(__pyx_t_6); __pyx_r = 0; goto __pyx_L0; @@ -2273,8 +2295,8 @@ static void __pyx_pw_5moses_7dictree_20PhraseDictionaryTree_3__dealloc__(PyObjec __Pyx_RefNannyFinishContext(); } -/* "moses/dictree.pyx":92 - * self.__tree.Read(string(path)) +/* "moses/dictree.pyx":93 + * self.__tree.Read(path) * * def __dealloc__(self): # <<<<<<<<<<<<<< * del self.__tree @@ -2285,7 +2307,7 @@ static void __pyx_pf_5moses_7dictree_20PhraseDictionaryTree_2__dealloc__(CYTHON_ __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "moses/dictree.pyx":93 + /* "moses/dictree.pyx":94 * * def __dealloc__(self): * del self.__tree # <<<<<<<<<<<<<< @@ -2331,7 +2353,7 @@ static PyObject *__pyx_pw_5moses_7dictree_20PhraseDictionaryTree_5isValidBinaryT } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "isValidBinaryTable") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "isValidBinaryTable") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -2343,10 +2365,10 @@ static PyObject *__pyx_pw_5moses_7dictree_20PhraseDictionaryTree_5isValidBinaryT } __pyx_v_stem = values[0]; if (values[1]) { - __pyx_v_wa = __Pyx_PyObject_IsTrue(values[1]); if (unlikely((__pyx_v_wa == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_v_wa = __Pyx_PyObject_IsTrue(values[1]); if (unlikely((__pyx_v_wa == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } else { - /* "moses/dictree.pyx":96 + /* "moses/dictree.pyx":97 * * @staticmethod * def isValidBinaryTable(stem, bint wa = False): # <<<<<<<<<<<<<< @@ -2358,7 +2380,7 @@ static PyObject *__pyx_pw_5moses_7dictree_20PhraseDictionaryTree_5isValidBinaryT } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("isValidBinaryTable", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("isValidBinaryTable", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("moses.dictree.PhraseDictionaryTree.isValidBinaryTable", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -2385,7 +2407,7 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_4isValidBinaryT int __pyx_clineno = 0; __Pyx_RefNannySetupContext("isValidBinaryTable", 0); - /* "moses/dictree.pyx":99 + /* "moses/dictree.pyx":100 * """This sanity check was added to the constructor, but you can access it from outside this class * to determine whether or not you are providing a valid stem to BinaryPhraseTable.""" * if wa: # <<<<<<<<<<<<<< @@ -2394,7 +2416,7 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_4isValidBinaryT */ if (__pyx_v_wa) { - /* "moses/dictree.pyx":100 + /* "moses/dictree.pyx":101 * to determine whether or not you are providing a valid stem to BinaryPhraseTable.""" * if wa: * return os.path.isfile(stem + ".binphr.idx") \ # <<<<<<<<<<<<<< @@ -2403,157 +2425,157 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_4isValidBinaryT */ __Pyx_XDECREF(__pyx_r); - /* "moses/dictree.pyx":101 + /* "moses/dictree.pyx":102 * if wa: * return os.path.isfile(stem + ".binphr.idx") \ * and os.path.isfile(stem + ".binphr.srctree.wa") \ # <<<<<<<<<<<<<< * and os.path.isfile(stem + ".binphr.srcvoc") \ * and os.path.isfile(stem + ".binphr.tgtdata.wa") \ */ - __pyx_t_1 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 100; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__path); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 100; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__path); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyObject_GetAttr(__pyx_t_2, __pyx_n_s__isfile); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 100; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_t_2, __pyx_n_s__isfile); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "moses/dictree.pyx":100 + /* "moses/dictree.pyx":101 * to determine whether or not you are providing a valid stem to BinaryPhraseTable.""" * if wa: * return os.path.isfile(stem + ".binphr.idx") \ # <<<<<<<<<<<<<< * and os.path.isfile(stem + ".binphr.srctree.wa") \ * and os.path.isfile(stem + ".binphr.srcvoc") \ */ - __pyx_t_2 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_9)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 100; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_9)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 100; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_2); __Pyx_GIVEREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 100; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_4) { __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "moses/dictree.pyx":101 + /* "moses/dictree.pyx":102 * if wa: * return os.path.isfile(stem + ".binphr.idx") \ * and os.path.isfile(stem + ".binphr.srctree.wa") \ # <<<<<<<<<<<<<< * and os.path.isfile(stem + ".binphr.srcvoc") \ * and os.path.isfile(stem + ".binphr.tgtdata.wa") \ */ - __pyx_t_3 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__path); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__path); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__isfile); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__isfile); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_10)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_10)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_1); __Pyx_GIVEREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_5), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_5), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_5)); __pyx_t_5 = 0; - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_4) { __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "moses/dictree.pyx":102 + /* "moses/dictree.pyx":103 * return os.path.isfile(stem + ".binphr.idx") \ * and os.path.isfile(stem + ".binphr.srctree.wa") \ * and os.path.isfile(stem + ".binphr.srcvoc") \ # <<<<<<<<<<<<<< * and os.path.isfile(stem + ".binphr.tgtdata.wa") \ * and os.path.isfile(stem + ".binphr.tgtvoc") */ - __pyx_t_5 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); - __pyx_t_3 = PyObject_GetAttr(__pyx_t_5, __pyx_n_s__path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_t_5, __pyx_n_s__path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__isfile); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__isfile); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_11)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_11)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_5, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_5, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_4) { __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "moses/dictree.pyx":103 + /* "moses/dictree.pyx":104 * and os.path.isfile(stem + ".binphr.srctree.wa") \ * and os.path.isfile(stem + ".binphr.srcvoc") \ * and os.path.isfile(stem + ".binphr.tgtdata.wa") \ # <<<<<<<<<<<<<< * and os.path.isfile(stem + ".binphr.tgtvoc") * else: */ - __pyx_t_6 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_5 = PyObject_GetAttr(__pyx_t_6, __pyx_n_s__path); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_GetAttr(__pyx_t_6, __pyx_n_s__path); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyObject_GetAttr(__pyx_t_5, __pyx_n_s__isfile); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_GetAttr(__pyx_t_5, __pyx_n_s__isfile); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_12)); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_12)); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); - __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5); __Pyx_GIVEREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_7), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_7), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_4) { __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - /* "moses/dictree.pyx":104 + /* "moses/dictree.pyx":105 * and os.path.isfile(stem + ".binphr.srcvoc") \ * and os.path.isfile(stem + ".binphr.tgtdata.wa") \ * and os.path.isfile(stem + ".binphr.tgtvoc") # <<<<<<<<<<<<<< * else: * return os.path.isfile(stem + ".binphr.idx") \ */ - __pyx_t_7 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = PyObject_GetAttr(__pyx_t_7, __pyx_n_s__path); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_GetAttr(__pyx_t_7, __pyx_n_s__path); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = PyObject_GetAttr(__pyx_t_6, __pyx_n_s__isfile); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyObject_GetAttr(__pyx_t_6, __pyx_n_s__isfile); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_13)); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_13)); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6); __Pyx_GIVEREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_t_8), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_t_8), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_8)); __pyx_t_8 = 0; @@ -2588,7 +2610,7 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_4isValidBinaryT } /*else*/ { - /* "moses/dictree.pyx":106 + /* "moses/dictree.pyx":107 * and os.path.isfile(stem + ".binphr.tgtvoc") * else: * return os.path.isfile(stem + ".binphr.idx") \ # <<<<<<<<<<<<<< @@ -2597,157 +2619,157 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_4isValidBinaryT */ __Pyx_XDECREF(__pyx_r); - /* "moses/dictree.pyx":107 + /* "moses/dictree.pyx":108 * else: * return os.path.isfile(stem + ".binphr.idx") \ * and os.path.isfile(stem + ".binphr.srctree") \ # <<<<<<<<<<<<<< * and os.path.isfile(stem + ".binphr.srcvoc") \ * and os.path.isfile(stem + ".binphr.tgtdata") \ */ - __pyx_t_1 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 106; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__path); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 106; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__path); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyObject_GetAttr(__pyx_t_2, __pyx_n_s__isfile); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 106; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_t_2, __pyx_n_s__isfile); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "moses/dictree.pyx":106 + /* "moses/dictree.pyx":107 * and os.path.isfile(stem + ".binphr.tgtvoc") * else: * return os.path.isfile(stem + ".binphr.idx") \ # <<<<<<<<<<<<<< * and os.path.isfile(stem + ".binphr.srctree") \ * and os.path.isfile(stem + ".binphr.srcvoc") \ */ - __pyx_t_2 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_9)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 106; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_9)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 106; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_2); __Pyx_GIVEREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 106; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_4) { __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "moses/dictree.pyx":107 + /* "moses/dictree.pyx":108 * else: * return os.path.isfile(stem + ".binphr.idx") \ * and os.path.isfile(stem + ".binphr.srctree") \ # <<<<<<<<<<<<<< * and os.path.isfile(stem + ".binphr.srcvoc") \ * and os.path.isfile(stem + ".binphr.tgtdata") \ */ - __pyx_t_3 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__path); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__path); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__isfile); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__isfile); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_14)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_14)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_1); __Pyx_GIVEREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_5), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_5), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_5)); __pyx_t_5 = 0; - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_4) { __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "moses/dictree.pyx":108 + /* "moses/dictree.pyx":109 * return os.path.isfile(stem + ".binphr.idx") \ * and os.path.isfile(stem + ".binphr.srctree") \ * and os.path.isfile(stem + ".binphr.srcvoc") \ # <<<<<<<<<<<<<< * and os.path.isfile(stem + ".binphr.tgtdata") \ * and os.path.isfile(stem + ".binphr.tgtvoc") */ - __pyx_t_5 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); - __pyx_t_3 = PyObject_GetAttr(__pyx_t_5, __pyx_n_s__path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_t_5, __pyx_n_s__path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__isfile); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__isfile); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_11)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_11)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_5, ((PyObject *)__pyx_t_8), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_5, ((PyObject *)__pyx_t_8), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_8)); __pyx_t_8 = 0; - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_4) { __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "moses/dictree.pyx":109 + /* "moses/dictree.pyx":110 * and os.path.isfile(stem + ".binphr.srctree") \ * and os.path.isfile(stem + ".binphr.srcvoc") \ * and os.path.isfile(stem + ".binphr.tgtdata") \ # <<<<<<<<<<<<<< * and os.path.isfile(stem + ".binphr.tgtvoc") * */ - __pyx_t_8 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); - __pyx_t_5 = PyObject_GetAttr(__pyx_t_8, __pyx_n_s__path); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_GetAttr(__pyx_t_8, __pyx_n_s__path); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_8 = PyObject_GetAttr(__pyx_t_5, __pyx_n_s__isfile); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_GetAttr(__pyx_t_5, __pyx_n_s__isfile); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_15)); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_15)); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_5); __Pyx_GIVEREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_4) { __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - /* "moses/dictree.pyx":110 + /* "moses/dictree.pyx":111 * and os.path.isfile(stem + ".binphr.srcvoc") \ * and os.path.isfile(stem + ".binphr.tgtdata") \ * and os.path.isfile(stem + ".binphr.tgtvoc") # <<<<<<<<<<<<<< * * def query(self, line, cmp = None, key = lambda x: x.scores[0], top = 0): */ - __pyx_t_6 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_8 = PyObject_GetAttr(__pyx_t_6, __pyx_n_s__path); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_GetAttr(__pyx_t_6, __pyx_n_s__path); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyObject_GetAttr(__pyx_t_8, __pyx_n_s__isfile); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_GetAttr(__pyx_t_8, __pyx_n_s__isfile); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_8 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_13)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_13)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); - __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __Pyx_GIVEREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_7), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_7), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; @@ -2814,7 +2836,7 @@ static PyObject *__pyx_pw_5moses_7dictree_20PhraseDictionaryTree_7query(PyObject static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__line,&__pyx_n_s__cmp,&__pyx_n_s__key,&__pyx_n_s__top,0}; PyObject* values[4] = {0,0,0,0}; - /* "moses/dictree.pyx":112 + /* "moses/dictree.pyx":113 * and os.path.isfile(stem + ".binphr.tgtvoc") * * def query(self, line, cmp = None, key = lambda x: x.scores[0], top = 0): # <<<<<<<<<<<<<< @@ -2857,7 +2879,7 @@ static PyObject *__pyx_pw_5moses_7dictree_20PhraseDictionaryTree_7query(PyObject } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "query") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "query") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -2876,7 +2898,7 @@ static PyObject *__pyx_pw_5moses_7dictree_20PhraseDictionaryTree_7query(PyObject } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("query", 0, 1, 4, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("query", 0, 1, 4, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("moses.dictree.PhraseDictionaryTree.query", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -2909,9 +2931,9 @@ static PyObject *__pyx_lambda_funcdef_lambda2(CYTHON_UNUSED PyObject *__pyx_self int __pyx_clineno = 0; __Pyx_RefNannySetupContext("lambda2", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_GetAttr(__pyx_v_x, __pyx_n_s__scores); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_v_x, __pyx_n_s__scores); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_GetItemInt(__pyx_t_1, 0, sizeof(long), PyInt_FromLong); if (!__pyx_t_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_GetItemInt(__pyx_t_1, 0, sizeof(long), PyInt_FromLong); if (!__pyx_t_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_2; @@ -2941,60 +2963,57 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_6query(struct _ PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; - char *__pyx_t_2; + std::string __pyx_t_2; std::string __pyx_t_3; - std::string __pyx_t_4; - std::vector *__pyx_t_5; - int __pyx_t_6; + std::vector *__pyx_t_4; + int __pyx_t_5; + size_t __pyx_t_6; size_t __pyx_t_7; - size_t __pyx_t_8; - PyObject *__pyx_t_9 = NULL; - std::vector *__pyx_t_10; - PyObject *__pyx_t_11 = NULL; - struct __pyx_opt_args_5moses_7dictree_get_query_result __pyx_t_12; - Py_ssize_t __pyx_t_13; + PyObject *__pyx_t_8 = NULL; + std::vector *__pyx_t_9; + PyObject *__pyx_t_10 = NULL; + struct __pyx_opt_args_5moses_7dictree_get_query_result __pyx_t_11; + Py_ssize_t __pyx_t_12; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("query", 0); - /* "moses/dictree.pyx":117 + /* "moses/dictree.pyx":118 * If 'cmp' is defined the return list is sorted. * If 'top' is defined, onlye the top elements will be returned.""" * cdef bytes text = as_str(line) # <<<<<<<<<<<<<< - * cdef vector[string] fphrase = cdictree.Tokenize(string(text), string(self.delimiters)) + * cdef vector[string] fphrase = cdictree.Tokenize(text, self.delimiters) * cdef vector[cdictree.StringTgtCand]* rv = new vector[cdictree.StringTgtCand]() */ - __pyx_t_1 = ((PyObject *)__pyx_f_5moses_7dictree_as_str(__pyx_v_line)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = ((PyObject *)__pyx_f_5moses_7dictree_as_str(__pyx_v_line)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_v_text = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "moses/dictree.pyx":118 + /* "moses/dictree.pyx":119 * If 'top' is defined, onlye the top elements will be returned.""" * cdef bytes text = as_str(line) - * cdef vector[string] fphrase = cdictree.Tokenize(string(text), string(self.delimiters)) # <<<<<<<<<<<<<< + * cdef vector[string] fphrase = cdictree.Tokenize(text, self.delimiters) # <<<<<<<<<<<<<< * cdef vector[cdictree.StringTgtCand]* rv = new vector[cdictree.StringTgtCand]() * cdef vector[string]* wa = NULL */ - __pyx_t_2 = PyBytes_AsString(((PyObject *)__pyx_v_text)); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - try {__pyx_t_3 = std::string(__pyx_t_2);} catch(...) {__Pyx_CppExn2PyErr(); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;}} - __pyx_t_2 = PyBytes_AsString(((PyObject *)__pyx_v_self->delimiters)); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - try {__pyx_t_4 = std::string(__pyx_t_2);} catch(...) {__Pyx_CppExn2PyErr(); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;}} - __pyx_v_fphrase = Moses::Tokenize(__pyx_t_3, __pyx_t_4); + __pyx_t_2 = __pyx_convert_string_from_py_(((PyObject *)__pyx_v_text)); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 119; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __pyx_convert_string_from_py_(((PyObject *)__pyx_v_self->delimiters)); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 119; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_fphrase = Moses::Tokenize(__pyx_t_2, __pyx_t_3); - /* "moses/dictree.pyx":119 + /* "moses/dictree.pyx":120 * cdef bytes text = as_str(line) - * cdef vector[string] fphrase = cdictree.Tokenize(string(text), string(self.delimiters)) + * cdef vector[string] fphrase = cdictree.Tokenize(text, self.delimiters) * cdef vector[cdictree.StringTgtCand]* rv = new vector[cdictree.StringTgtCand]() # <<<<<<<<<<<<<< * cdef vector[string]* wa = NULL * cdef list phrases */ - try {__pyx_t_5 = new std::vector();} catch(...) {__Pyx_CppExn2PyErr(); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 119; __pyx_clineno = __LINE__; goto __pyx_L1_error;}} - __pyx_v_rv = __pyx_t_5; + try {__pyx_t_4 = new std::vector();} catch(...) {__Pyx_CppExn2PyErr(); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 120; __pyx_clineno = __LINE__; goto __pyx_L1_error;}} + __pyx_v_rv = __pyx_t_4; - /* "moses/dictree.pyx":120 - * cdef vector[string] fphrase = cdictree.Tokenize(string(text), string(self.delimiters)) + /* "moses/dictree.pyx":121 + * cdef vector[string] fphrase = cdictree.Tokenize(text, self.delimiters) * cdef vector[cdictree.StringTgtCand]* rv = new vector[cdictree.StringTgtCand]() * cdef vector[string]* wa = NULL # <<<<<<<<<<<<<< * cdef list phrases @@ -3002,17 +3021,17 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_6query(struct _ */ __pyx_v_wa = NULL; - /* "moses/dictree.pyx":122 + /* "moses/dictree.pyx":123 * cdef vector[string]* wa = NULL * cdef list phrases * if not self.__tree.UseWordAlignment(): # <<<<<<<<<<<<<< * self.__tree.GetTargetCandidates(fphrase, rv[0]) * phrases = [get_query_result(rv[0][i]) for i in range(rv.size())] */ - __pyx_t_6 = (!__pyx_v_self->__pyx___tree->UseWordAlignment()); - if (__pyx_t_6) { + __pyx_t_5 = (!__pyx_v_self->__pyx___tree->UseWordAlignment()); + if (__pyx_t_5) { - /* "moses/dictree.pyx":123 + /* "moses/dictree.pyx":124 * cdef list phrases * if not self.__tree.UseWordAlignment(): * self.__tree.GetTargetCandidates(fphrase, rv[0]) # <<<<<<<<<<<<<< @@ -3021,22 +3040,22 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_6query(struct _ */ __pyx_v_self->__pyx___tree->GetTargetCandidates(__pyx_v_fphrase, (__pyx_v_rv[0])); - /* "moses/dictree.pyx":124 + /* "moses/dictree.pyx":125 * if not self.__tree.UseWordAlignment(): * self.__tree.GetTargetCandidates(fphrase, rv[0]) * phrases = [get_query_result(rv[0][i]) for i in range(rv.size())] # <<<<<<<<<<<<<< * else: * wa = new vector[string]() */ - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_7 = __pyx_v_rv->size(); - for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) { - __pyx_v_i = __pyx_t_8; - __pyx_t_9 = ((PyObject *)__pyx_f_5moses_7dictree_get_query_result(((__pyx_v_rv[0])[__pyx_v_i]), NULL)); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_9); - if (unlikely(__Pyx_PyList_Append(__pyx_t_1, (PyObject*)__pyx_t_9))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_6 = __pyx_v_rv->size(); + for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_6; __pyx_t_7+=1) { + __pyx_v_i = __pyx_t_7; + __pyx_t_8 = ((PyObject *)__pyx_f_5moses_7dictree_get_query_result(((__pyx_v_rv[0])[__pyx_v_i]), NULL)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + if (unlikely(__Pyx_PyList_Append(__pyx_t_1, (PyObject*)__pyx_t_8))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } __Pyx_INCREF(((PyObject *)__pyx_t_1)); __pyx_v_phrases = __pyx_t_1; @@ -3045,17 +3064,17 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_6query(struct _ } /*else*/ { - /* "moses/dictree.pyx":126 + /* "moses/dictree.pyx":127 * phrases = [get_query_result(rv[0][i]) for i in range(rv.size())] * else: * wa = new vector[string]() # <<<<<<<<<<<<<< * self.__tree.GetTargetCandidates(fphrase, rv[0], wa[0]) * phrases = [get_query_result(rv[0][i], wa[0][i].c_str()) for i in range(rv.size())] */ - try {__pyx_t_10 = new std::vector();} catch(...) {__Pyx_CppExn2PyErr(); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;}} - __pyx_v_wa = __pyx_t_10; + try {__pyx_t_9 = new std::vector();} catch(...) {__Pyx_CppExn2PyErr(); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; __pyx_clineno = __LINE__; goto __pyx_L1_error;}} + __pyx_v_wa = __pyx_t_9; - /* "moses/dictree.pyx":127 + /* "moses/dictree.pyx":128 * else: * wa = new vector[string]() * self.__tree.GetTargetCandidates(fphrase, rv[0], wa[0]) # <<<<<<<<<<<<<< @@ -3064,33 +3083,33 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_6query(struct _ */ __pyx_v_self->__pyx___tree->GetTargetCandidates(__pyx_v_fphrase, (__pyx_v_rv[0]), (__pyx_v_wa[0])); - /* "moses/dictree.pyx":128 + /* "moses/dictree.pyx":129 * wa = new vector[string]() * self.__tree.GetTargetCandidates(fphrase, rv[0], wa[0]) * phrases = [get_query_result(rv[0][i], wa[0][i].c_str()) for i in range(rv.size())] # <<<<<<<<<<<<<< * del wa * del rv */ - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_7 = __pyx_v_rv->size(); - for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) { - __pyx_v_i = __pyx_t_8; - __pyx_t_9 = PyBytes_FromString(((__pyx_v_wa[0])[__pyx_v_i]).c_str()); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_9)); - __pyx_t_12.__pyx_n = 1; - __pyx_t_12.wa = ((PyObject *)__pyx_t_9); - __pyx_t_11 = ((PyObject *)__pyx_f_5moses_7dictree_get_query_result(((__pyx_v_rv[0])[__pyx_v_i]), &__pyx_t_12)); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_11); - __Pyx_DECREF(((PyObject *)__pyx_t_9)); __pyx_t_9 = 0; - if (unlikely(__Pyx_PyList_Append(__pyx_t_1, (PyObject*)__pyx_t_11))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_t_6 = __pyx_v_rv->size(); + for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_6; __pyx_t_7+=1) { + __pyx_v_i = __pyx_t_7; + __pyx_t_8 = PyBytes_FromString(((__pyx_v_wa[0])[__pyx_v_i]).c_str()); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_8)); + __pyx_t_11.__pyx_n = 1; + __pyx_t_11.wa = ((PyObject *)__pyx_t_8); + __pyx_t_10 = ((PyObject *)__pyx_f_5moses_7dictree_get_query_result(((__pyx_v_rv[0])[__pyx_v_i]), &__pyx_t_11)); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(((PyObject *)__pyx_t_8)); __pyx_t_8 = 0; + if (unlikely(__Pyx_PyList_Append(__pyx_t_1, (PyObject*)__pyx_t_10))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; } __Pyx_INCREF(((PyObject *)__pyx_t_1)); __pyx_v_phrases = __pyx_t_1; __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - /* "moses/dictree.pyx":129 + /* "moses/dictree.pyx":130 * self.__tree.GetTargetCandidates(fphrase, rv[0], wa[0]) * phrases = [get_query_result(rv[0][i], wa[0][i].c_str()) for i in range(rv.size())] * del wa # <<<<<<<<<<<<<< @@ -3101,7 +3120,7 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_6query(struct _ } __pyx_L3:; - /* "moses/dictree.pyx":130 + /* "moses/dictree.pyx":131 * phrases = [get_query_result(rv[0][i], wa[0][i].c_str()) for i in range(rv.size())] * del wa * del rv # <<<<<<<<<<<<<< @@ -3110,50 +3129,50 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_6query(struct _ */ delete __pyx_v_rv; - /* "moses/dictree.pyx":131 + /* "moses/dictree.pyx":132 * del wa * del rv * if cmp: # <<<<<<<<<<<<<< * phrases.sort(cmp=cmp) * if top > 0: */ - __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_v_cmp); if (unlikely(__pyx_t_6 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__pyx_t_6) { + __pyx_t_5 = __Pyx_PyObject_IsTrue(__pyx_v_cmp); if (unlikely(__pyx_t_5 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_5) { - /* "moses/dictree.pyx":132 + /* "moses/dictree.pyx":133 * del rv * if cmp: * phrases.sort(cmp=cmp) # <<<<<<<<<<<<<< * if top > 0: * return phrases[0:top] */ - __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_phrases), __pyx_n_s__sort); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_phrases), __pyx_n_s__sort); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_11 = PyDict_New(); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_11)); - if (PyDict_SetItem(__pyx_t_11, ((PyObject *)__pyx_n_s__cmp), __pyx_v_cmp) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_9 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), ((PyObject *)__pyx_t_11)); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_9); + __pyx_t_10 = PyDict_New(); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_10)); + if (PyDict_SetItem(__pyx_t_10, ((PyObject *)__pyx_n_s__cmp), __pyx_v_cmp) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), ((PyObject *)__pyx_t_10)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_11)); __pyx_t_11 = 0; - __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_10)); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; goto __pyx_L8; } __pyx_L8:; - /* "moses/dictree.pyx":133 + /* "moses/dictree.pyx":134 * if cmp: * phrases.sort(cmp=cmp) * if top > 0: # <<<<<<<<<<<<<< * return phrases[0:top] * else: */ - __pyx_t_9 = PyObject_RichCompare(__pyx_v_top, __pyx_int_0, Py_GT); __Pyx_XGOTREF(__pyx_t_9); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_9); if (unlikely(__pyx_t_6 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (__pyx_t_6) { + __pyx_t_8 = PyObject_RichCompare(__pyx_v_top, __pyx_int_0, Py_GT); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_5 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (__pyx_t_5) { - /* "moses/dictree.pyx":134 + /* "moses/dictree.pyx":135 * phrases.sort(cmp=cmp) * if top > 0: * return phrases[0:top] # <<<<<<<<<<<<<< @@ -3161,21 +3180,22 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_6query(struct _ * return phrases */ __Pyx_XDECREF(__pyx_r); - __pyx_t_13 = __Pyx_PyIndex_AsSsize_t(__pyx_v_top); if (unlikely((__pyx_t_13 == (Py_ssize_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_9 = __Pyx_PySequence_GetSlice(((PyObject *)__pyx_v_phrases), 0, __pyx_t_13); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_9)); - __pyx_r = ((PyObject *)__pyx_t_9); - __pyx_t_9 = 0; + __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_v_top); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = __Pyx_PySequence_GetSlice(((PyObject *)__pyx_v_phrases), 0, __pyx_t_12); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_8)); + __pyx_r = ((PyObject *)__pyx_t_8); + __pyx_t_8 = 0; goto __pyx_L0; goto __pyx_L9; } /*else*/ { - /* "moses/dictree.pyx":136 + /* "moses/dictree.pyx":137 * return phrases[0:top] * else: * return phrases # <<<<<<<<<<<<<< * + * cdef class OnDiskWrapper(object): */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_phrases)); @@ -3188,8 +3208,8 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_6query(struct _ goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_9); - __Pyx_XDECREF(__pyx_t_11); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); __Pyx_AddTraceback("moses.dictree.PhraseDictionaryTree.query", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; @@ -3211,7 +3231,7 @@ static PyObject *__pyx_pw_5moses_7dictree_20PhraseDictionaryTree_4path_1__get__( return __pyx_r; } -/* "moses/dictree.pyx":70 +/* "moses/dictree.pyx":71 * * cdef cdictree.PhraseDictionaryTree* __tree * cdef readonly bytes path # <<<<<<<<<<<<<< @@ -3246,7 +3266,7 @@ static PyObject *__pyx_pw_5moses_7dictree_20PhraseDictionaryTree_7nscores_1__get return __pyx_r; } -/* "moses/dictree.pyx":71 +/* "moses/dictree.pyx":72 * cdef cdictree.PhraseDictionaryTree* __tree * cdef readonly bytes path * cdef readonly unsigned nscores # <<<<<<<<<<<<<< @@ -3263,7 +3283,7 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_7nscores___get_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyLong_FromUnsignedLong(__pyx_v_self->nscores); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyLong_FromUnsignedLong(__pyx_v_self->nscores); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 72; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -3292,7 +3312,7 @@ static PyObject *__pyx_pw_5moses_7dictree_20PhraseDictionaryTree_2wa_1__get__(Py return __pyx_r; } -/* "moses/dictree.pyx":72 +/* "moses/dictree.pyx":73 * cdef readonly bytes path * cdef readonly unsigned nscores * cdef readonly bint wa # <<<<<<<<<<<<<< @@ -3309,7 +3329,7 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_2wa___get__(str int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->wa); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 72; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->wa); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 73; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -3338,7 +3358,7 @@ static PyObject *__pyx_pw_5moses_7dictree_20PhraseDictionaryTree_10delimiters_1_ return __pyx_r; } -/* "moses/dictree.pyx":73 +/* "moses/dictree.pyx":74 * cdef readonly unsigned nscores * cdef readonly bint wa * cdef readonly bytes delimiters # <<<<<<<<<<<<<< @@ -3362,6 +3382,273 @@ static PyObject *__pyx_pf_5moses_7dictree_20PhraseDictionaryTree_10delimiters___ return __pyx_r; } +/* Python wrapper */ +static int __pyx_pw_5moses_7dictree_13OnDiskWrapper_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5moses_7dictree_13OnDiskWrapper_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_path = 0; + PyObject *__pyx_v_delimiters = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__path,&__pyx_n_s__delimiters,0}; + PyObject* values[2] = {0,0}; + values[1] = ((PyObject *)__pyx_kp_s_7); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__path)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__delimiters); + if (value) { values[1] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 144; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_path = ((PyObject*)values[0]); + __pyx_v_delimiters = values[1]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 144; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("moses.dictree.OnDiskWrapper.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_path), (&PyBytes_Type), 1, "path", 1))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 144; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_r = __pyx_pf_5moses_7dictree_13OnDiskWrapper___cinit__(((struct __pyx_obj_5moses_7dictree_OnDiskWrapper *)__pyx_v_self), __pyx_v_path, __pyx_v_delimiters); + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "moses/dictree.pyx":144 + * cdef readonly bytes delimiters + * + * def __cinit__(self, bytes path, delimiters = ' \t'): # <<<<<<<<<<<<<< + * self.delimiters = delimiters + * self.wrapper = new condiskpt.OnDiskWrapper() + */ + +static int __pyx_pf_5moses_7dictree_13OnDiskWrapper___cinit__(struct __pyx_obj_5moses_7dictree_OnDiskWrapper *__pyx_v_self, PyObject *__pyx_v_path, PyObject *__pyx_v_delimiters) { + int __pyx_r; + __Pyx_RefNannyDeclarations + char *__pyx_t_1; + std::string __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__cinit__", 0); + + /* "moses/dictree.pyx":145 + * + * def __cinit__(self, bytes path, delimiters = ' \t'): + * self.delimiters = delimiters # <<<<<<<<<<<<<< + * self.wrapper = new condiskpt.OnDiskWrapper() + * self.wrapper.BeginLoad(string(path)) + */ + if (!(likely(PyBytes_CheckExact(__pyx_v_delimiters))||((__pyx_v_delimiters) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_v_delimiters)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_INCREF(__pyx_v_delimiters); + __Pyx_GIVEREF(__pyx_v_delimiters); + __Pyx_GOTREF(__pyx_v_self->delimiters); + __Pyx_DECREF(((PyObject *)__pyx_v_self->delimiters)); + __pyx_v_self->delimiters = ((PyObject*)__pyx_v_delimiters); + + /* "moses/dictree.pyx":146 + * def __cinit__(self, bytes path, delimiters = ' \t'): + * self.delimiters = delimiters + * self.wrapper = new condiskpt.OnDiskWrapper() # <<<<<<<<<<<<<< + * self.wrapper.BeginLoad(string(path)) + * + */ + __pyx_v_self->wrapper = new OnDiskPt::OnDiskWrapper(); + + /* "moses/dictree.pyx":147 + * self.delimiters = delimiters + * self.wrapper = new condiskpt.OnDiskWrapper() + * self.wrapper.BeginLoad(string(path)) # <<<<<<<<<<<<<< + * + * def query(self, line): + */ + __pyx_t_1 = PyBytes_AsString(((PyObject *)__pyx_v_path)); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + try {__pyx_t_2 = std::string(__pyx_t_1);} catch(...) {__Pyx_CppExn2PyErr(); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;}} + __pyx_v_self->wrapper->BeginLoad(__pyx_t_2); + + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("moses.dictree.OnDiskWrapper.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5moses_7dictree_13OnDiskWrapper_3query(PyObject *__pyx_v_self, PyObject *__pyx_v_line); /*proto*/ +static PyObject *__pyx_pw_5moses_7dictree_13OnDiskWrapper_3query(PyObject *__pyx_v_self, PyObject *__pyx_v_line) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("query (wrapper)", 0); + __pyx_r = __pyx_pf_5moses_7dictree_13OnDiskWrapper_2query(((struct __pyx_obj_5moses_7dictree_OnDiskWrapper *)__pyx_v_self), ((PyObject *)__pyx_v_line)); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "moses/dictree.pyx":149 + * self.wrapper.BeginLoad(string(path)) + * + * def query(self, line): # <<<<<<<<<<<<<< + * cdef bytes text = as_str(line) + * cdef vector[string] ftokens = cdictree.Tokenize(text, self.delimiters) + */ + +static PyObject *__pyx_pf_5moses_7dictree_13OnDiskWrapper_2query(struct __pyx_obj_5moses_7dictree_OnDiskWrapper *__pyx_v_self, PyObject *__pyx_v_line) { + PyObject *__pyx_v_text = 0; + std::vector __pyx_v_ftokens; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + std::string __pyx_t_2; + std::string __pyx_t_3; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("query", 0); + + /* "moses/dictree.pyx":150 + * + * def query(self, line): + * cdef bytes text = as_str(line) # <<<<<<<<<<<<<< + * cdef vector[string] ftokens = cdictree.Tokenize(text, self.delimiters) + * #print 'query:', ftokens + */ + __pyx_t_1 = ((PyObject *)__pyx_f_5moses_7dictree_as_str(__pyx_v_line)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_text = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "moses/dictree.pyx":151 + * def query(self, line): + * cdef bytes text = as_str(line) + * cdef vector[string] ftokens = cdictree.Tokenize(text, self.delimiters) # <<<<<<<<<<<<<< + * #print 'query:', ftokens + * #cdef condiskpt.SourcePhrase fphrase = condiskpt.Tokenize(ftokens, self.wrapper[0]) + */ + __pyx_t_2 = __pyx_convert_string_from_py_(((PyObject *)__pyx_v_text)); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 151; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __pyx_convert_string_from_py_(((PyObject *)__pyx_v_self->delimiters)); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 151; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_ftokens = Moses::Tokenize(__pyx_t_2, __pyx_t_3); + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("moses.dictree.OnDiskWrapper.query", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_text); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5moses_7dictree_13OnDiskWrapper_10delimiters_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5moses_7dictree_13OnDiskWrapper_10delimiters_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5moses_7dictree_13OnDiskWrapper_10delimiters___get__(((struct __pyx_obj_5moses_7dictree_OnDiskWrapper *)__pyx_v_self)); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "moses/dictree.pyx":142 + * + * cdef condiskpt.OnDiskWrapper* wrapper + * cdef readonly bytes delimiters # <<<<<<<<<<<<<< + * + * def __cinit__(self, bytes path, delimiters = ' \t'): + */ + +static PyObject *__pyx_pf_5moses_7dictree_13OnDiskWrapper_10delimiters___get__(struct __pyx_obj_5moses_7dictree_OnDiskWrapper *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(((PyObject *)__pyx_v_self->delimiters)); + __pyx_r = ((PyObject *)__pyx_v_self->delimiters); + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.from_py":11 + * + * @cname("__pyx_convert_string_from_py_") + * cdef string __pyx_convert_string_from_py_(object o) except *: # <<<<<<<<<<<<<< + * return string(o, len(o)) + * + */ + +static std::string __pyx_convert_string_from_py_(PyObject *__pyx_v_o) { + std::string __pyx_r; + __Pyx_RefNannyDeclarations + char *__pyx_t_1; + Py_ssize_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_", 0); + + /* "string.from_py":12 + * @cname("__pyx_convert_string_from_py_") + * cdef string __pyx_convert_string_from_py_(object o) except *: + * return string(o, len(o)) # <<<<<<<<<<<<<< + * + * + */ + __pyx_t_1 = PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_r = std::string(((char *)__pyx_t_1), __pyx_t_2); + goto __pyx_L0; + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("string.from_py.__pyx_convert_string_from_py_", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + static PyObject *__pyx_tp_new_5moses_7dictree_QueryResult(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5moses_7dictree_QueryResult *p; PyObject *o = (*t->tp_alloc)(t, 0); @@ -3829,6 +4116,210 @@ static PyTypeObject __pyx_type_5moses_7dictree_PhraseDictionaryTree = { #endif }; +static PyObject *__pyx_tp_new_5moses_7dictree_OnDiskWrapper(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5moses_7dictree_OnDiskWrapper *p; + PyObject *o = (*t->tp_alloc)(t, 0); + if (!o) return 0; + p = ((struct __pyx_obj_5moses_7dictree_OnDiskWrapper *)o); + p->delimiters = ((PyObject*)Py_None); Py_INCREF(Py_None); + if (__pyx_pw_5moses_7dictree_13OnDiskWrapper_1__cinit__(o, a, k) < 0) { + Py_DECREF(o); o = 0; + } + return o; +} + +static void __pyx_tp_dealloc_5moses_7dictree_OnDiskWrapper(PyObject *o) { + struct __pyx_obj_5moses_7dictree_OnDiskWrapper *p = (struct __pyx_obj_5moses_7dictree_OnDiskWrapper *)o; + Py_CLEAR(p->delimiters); + (*Py_TYPE(o)->tp_free)(o); +} + +static int __pyx_tp_traverse_5moses_7dictree_OnDiskWrapper(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5moses_7dictree_OnDiskWrapper *p = (struct __pyx_obj_5moses_7dictree_OnDiskWrapper *)o; + if (p->delimiters) { + e = (*v)(p->delimiters, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5moses_7dictree_OnDiskWrapper(PyObject *o) { + struct __pyx_obj_5moses_7dictree_OnDiskWrapper *p = (struct __pyx_obj_5moses_7dictree_OnDiskWrapper *)o; + PyObject* tmp; + tmp = ((PyObject*)p->delimiters); + p->delimiters = ((PyObject*)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyObject *__pyx_getprop_5moses_7dictree_13OnDiskWrapper_delimiters(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5moses_7dictree_13OnDiskWrapper_10delimiters_1__get__(o); +} + +static PyMethodDef __pyx_methods_5moses_7dictree_OnDiskWrapper[] = { + {__Pyx_NAMESTR("query"), (PyCFunction)__pyx_pw_5moses_7dictree_13OnDiskWrapper_3query, METH_O, __Pyx_DOCSTR(0)}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_5moses_7dictree_OnDiskWrapper[] = { + {(char *)"delimiters", __pyx_getprop_5moses_7dictree_13OnDiskWrapper_delimiters, 0, 0, 0}, + {0, 0, 0, 0, 0} +}; + +static PyNumberMethods __pyx_tp_as_number_OnDiskWrapper = { + 0, /*nb_add*/ + 0, /*nb_subtract*/ + 0, /*nb_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_divide*/ + #endif + 0, /*nb_remainder*/ + 0, /*nb_divmod*/ + 0, /*nb_power*/ + 0, /*nb_negative*/ + 0, /*nb_positive*/ + 0, /*nb_absolute*/ + 0, /*nb_nonzero*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + 0, /*nb_and*/ + 0, /*nb_xor*/ + 0, /*nb_or*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_coerce*/ + #endif + 0, /*nb_int*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_long*/ + #else + 0, /*reserved*/ + #endif + 0, /*nb_float*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_oct*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*nb_hex*/ + #endif + 0, /*nb_inplace_add*/ + 0, /*nb_inplace_subtract*/ + 0, /*nb_inplace_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_inplace_divide*/ + #endif + 0, /*nb_inplace_remainder*/ + 0, /*nb_inplace_power*/ + 0, /*nb_inplace_lshift*/ + 0, /*nb_inplace_rshift*/ + 0, /*nb_inplace_and*/ + 0, /*nb_inplace_xor*/ + 0, /*nb_inplace_or*/ + 0, /*nb_floor_divide*/ + 0, /*nb_true_divide*/ + 0, /*nb_inplace_floor_divide*/ + 0, /*nb_inplace_true_divide*/ + #if PY_VERSION_HEX >= 0x02050000 + 0, /*nb_index*/ + #endif +}; + +static PySequenceMethods __pyx_tp_as_sequence_OnDiskWrapper = { + 0, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_OnDiskWrapper = { + 0, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_OnDiskWrapper = { + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getreadbuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getwritebuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getsegcount*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getcharbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_getbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_releasebuffer*/ + #endif +}; + +static PyTypeObject __pyx_type_5moses_7dictree_OnDiskWrapper = { + PyVarObject_HEAD_INIT(0, 0) + __Pyx_NAMESTR("moses.dictree.OnDiskWrapper"), /*tp_name*/ + sizeof(struct __pyx_obj_5moses_7dictree_OnDiskWrapper), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5moses_7dictree_OnDiskWrapper, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #else + 0, /*reserved*/ + #endif + 0, /*tp_repr*/ + &__pyx_tp_as_number_OnDiskWrapper, /*tp_as_number*/ + &__pyx_tp_as_sequence_OnDiskWrapper, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_OnDiskWrapper, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_OnDiskWrapper, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_5moses_7dictree_OnDiskWrapper, /*tp_traverse*/ + __pyx_tp_clear_5moses_7dictree_OnDiskWrapper, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5moses_7dictree_OnDiskWrapper, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_5moses_7dictree_OnDiskWrapper, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5moses_7dictree_OnDiskWrapper, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + #if PY_VERSION_HEX >= 0x02060000 + 0, /*tp_version_tag*/ + #endif +}; + static PyMethodDef __pyx_methods[] = { {__Pyx_NAMESTR("fsign"), (PyCFunction)__pyx_pw_5moses_7dictree_1fsign, METH_O, __Pyx_DOCSTR(__pyx_doc_5moses_7dictree_fsign)}, {0, 0, 0, 0} @@ -3894,15 +4385,15 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {0, 0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_n_s__staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_n_s__TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_n_s__staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_n_s__TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #if PY_MAJOR_VERSION >= 3 - __pyx_builtin_xrange = __Pyx_GetName(__pyx_b, __pyx_n_s__range); if (!__pyx_builtin_xrange) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_xrange = __Pyx_GetName(__pyx_b, __pyx_n_s__range); if (!__pyx_builtin_xrange) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #else - __pyx_builtin_xrange = __Pyx_GetName(__pyx_b, __pyx_n_s__xrange); if (!__pyx_builtin_xrange) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_xrange = __Pyx_GetName(__pyx_b, __pyx_n_s__xrange); if (!__pyx_builtin_xrange) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif - __pyx_builtin_ValueError = __Pyx_GetName(__pyx_b, __pyx_n_s__ValueError); if (!__pyx_builtin_ValueError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_range = __Pyx_GetName(__pyx_b, __pyx_n_s__range); if (!__pyx_builtin_range) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_ValueError = __Pyx_GetName(__pyx_b, __pyx_n_s__ValueError); if (!__pyx_builtin_ValueError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_range = __Pyx_GetName(__pyx_b, __pyx_n_s__range); if (!__pyx_builtin_range) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} return 0; __pyx_L1_error:; return -1; @@ -3912,28 +4403,28 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - /* "moses/dictree.pyx":17 + /* "moses/dictree.pyx":18 * return data * elif isinstance(data, unicode): * return data.encode('UTF-8') # <<<<<<<<<<<<<< * raise TypeError('Cannot convert %s to string' % type(data)) * */ - __pyx_k_tuple_2 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 17; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_k_tuple_2 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_2); __Pyx_INCREF(((PyObject *)__pyx_kp_s_1)); PyTuple_SET_ITEM(__pyx_k_tuple_2, 0, ((PyObject *)__pyx_kp_s_1)); __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_1)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_2)); - /* "moses/dictree.pyx":40 + /* "moses/dictree.pyx":41 * * @staticmethod * def desc(x, y, key = lambda r: r.scores[0]): # <<<<<<<<<<<<<< * """Returns the sign of key(y) - key(x). * Can only be used if scores is not an empty vector as */ - __pyx_k_tuple_18 = PyTuple_New(3); if (unlikely(!__pyx_k_tuple_18)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_k_tuple_18 = PyTuple_New(3); if (unlikely(!__pyx_k_tuple_18)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_18); __Pyx_INCREF(((PyObject *)__pyx_n_s__x)); PyTuple_SET_ITEM(__pyx_k_tuple_18, 0, ((PyObject *)__pyx_n_s__x)); @@ -3945,16 +4436,16 @@ static int __Pyx_InitCachedConstants(void) { PyTuple_SET_ITEM(__pyx_k_tuple_18, 2, ((PyObject *)__pyx_n_s__key)); __Pyx_GIVEREF(((PyObject *)__pyx_n_s__key)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_18)); - __pyx_k_codeobj_19 = (PyObject*)__Pyx_PyCode_New(3, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_18, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_20, __pyx_n_s__desc, 40, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_19)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_k_codeobj_19 = (PyObject*)__Pyx_PyCode_New(3, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_18, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_20, __pyx_n_s__desc, 41, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_19)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "moses/dictree.pyx":96 + /* "moses/dictree.pyx":97 * * @staticmethod * def isValidBinaryTable(stem, bint wa = False): # <<<<<<<<<<<<<< * """This sanity check was added to the constructor, but you can access it from outside this class * to determine whether or not you are providing a valid stem to BinaryPhraseTable.""" */ - __pyx_k_tuple_21 = PyTuple_New(2); if (unlikely(!__pyx_k_tuple_21)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_k_tuple_21 = PyTuple_New(2); if (unlikely(!__pyx_k_tuple_21)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_21); __Pyx_INCREF(((PyObject *)__pyx_n_s__stem)); PyTuple_SET_ITEM(__pyx_k_tuple_21, 0, ((PyObject *)__pyx_n_s__stem)); @@ -3963,7 +4454,7 @@ static int __Pyx_InitCachedConstants(void) { PyTuple_SET_ITEM(__pyx_k_tuple_21, 1, ((PyObject *)__pyx_n_s__wa)); __Pyx_GIVEREF(((PyObject *)__pyx_n_s__wa)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_21)); - __pyx_k_codeobj_22 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_21, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_20, __pyx_n_s__isValidBinaryTable, 96, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_22)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_k_codeobj_22 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_21, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_20, __pyx_n_s__isValidBinaryTable, 97, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_22)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -4044,10 +4535,10 @@ PyMODINIT_FUNC PyInit_dictree(void) /*--- Variable export code ---*/ /*--- Function export code ---*/ /*--- Type init code ---*/ - if (PyType_Ready(&__pyx_type_5moses_7dictree_QueryResult) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyType_Ready(&__pyx_type_5moses_7dictree_QueryResult) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #if CYTHON_COMPILING_IN_CPYTHON { - PyObject *wrapper = __Pyx_GetAttrString((PyObject *)&__pyx_type_5moses_7dictree_QueryResult, "__str__"); if (unlikely(!wrapper)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + PyObject *wrapper = __Pyx_GetAttrString((PyObject *)&__pyx_type_5moses_7dictree_QueryResult, "__str__"); if (unlikely(!wrapper)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) { __pyx_wrapperbase_5moses_7dictree_11QueryResult_4__str__ = *((PyWrapperDescrObject *)wrapper)->d_base; __pyx_wrapperbase_5moses_7dictree_11QueryResult_4__str__.doc = __pyx_doc_5moses_7dictree_11QueryResult_4__str__; @@ -4055,11 +4546,14 @@ PyMODINIT_FUNC PyInit_dictree(void) } } #endif - if (__Pyx_SetAttrString(__pyx_m, "QueryResult", (PyObject *)&__pyx_type_5moses_7dictree_QueryResult) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "QueryResult", (PyObject *)&__pyx_type_5moses_7dictree_QueryResult) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5moses_7dictree_QueryResult = &__pyx_type_5moses_7dictree_QueryResult; - if (PyType_Ready(&__pyx_type_5moses_7dictree_PhraseDictionaryTree) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "PhraseDictionaryTree", (PyObject *)&__pyx_type_5moses_7dictree_PhraseDictionaryTree) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyType_Ready(&__pyx_type_5moses_7dictree_PhraseDictionaryTree) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "PhraseDictionaryTree", (PyObject *)&__pyx_type_5moses_7dictree_PhraseDictionaryTree) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5moses_7dictree_PhraseDictionaryTree = &__pyx_type_5moses_7dictree_PhraseDictionaryTree; + if (PyType_Ready(&__pyx_type_5moses_7dictree_OnDiskWrapper) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "OnDiskWrapper", (PyObject *)&__pyx_type_5moses_7dictree_OnDiskWrapper) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_5moses_7dictree_OnDiskWrapper = &__pyx_type_5moses_7dictree_OnDiskWrapper; /*--- Type import code ---*/ /*--- Variable import code ---*/ /*--- Function import code ---*/ @@ -4077,111 +4571,111 @@ PyMODINIT_FUNC PyInit_dictree(void) if (PyObject_SetAttr(__pyx_m, __pyx_n_s__os, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 5; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "moses/dictree.pyx":40 + /* "moses/dictree.pyx":41 * * @staticmethod * def desc(x, y, key = lambda r: r.scores[0]): # <<<<<<<<<<<<<< * """Returns the sign of key(y) - key(x). * Can only be used if scores is not an empty vector as */ - __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5moses_7dictree_11QueryResult_4desc_lambda1, 0, NULL, __pyx_n_s_17, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5moses_7dictree_11QueryResult_4desc_lambda1, 0, NULL, __pyx_n_s_17, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_k_4 = __pyx_t_1; __Pyx_GIVEREF(__pyx_t_1); __pyx_t_1 = 0; - /* "moses/dictree.pyx":39 + /* "moses/dictree.pyx":40 * self.wa = wa * * @staticmethod # <<<<<<<<<<<<<< * def desc(x, y, key = lambda r: r.scores[0]): * """Returns the sign of key(y) - key(x). */ - __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5moses_7dictree_11QueryResult_3desc, NULL, __pyx_n_s_17); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5moses_7dictree_11QueryResult_3desc, NULL, __pyx_n_s_17); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); __Pyx_GIVEREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5moses_7dictree_QueryResult->tp_dict, __pyx_n_s__desc, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem((PyObject *)__pyx_ptype_5moses_7dictree_QueryResult->tp_dict, __pyx_n_s__desc, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5moses_7dictree_QueryResult); - /* "moses/dictree.pyx":40 + /* "moses/dictree.pyx":41 * * @staticmethod * def desc(x, y, key = lambda r: r.scores[0]): # <<<<<<<<<<<<<< * """Returns the sign of key(y) - key(x). * Can only be used if scores is not an empty vector as */ - __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5moses_7dictree_QueryResult, __pyx_n_s__desc); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5moses_7dictree_QueryResult, __pyx_n_s__desc); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); __Pyx_GIVEREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5moses_7dictree_QueryResult->tp_dict, __pyx_n_s__desc, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem((PyObject *)__pyx_ptype_5moses_7dictree_QueryResult->tp_dict, __pyx_n_s__desc, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5moses_7dictree_QueryResult); - /* "moses/dictree.pyx":96 + /* "moses/dictree.pyx":97 * * @staticmethod * def isValidBinaryTable(stem, bint wa = False): # <<<<<<<<<<<<<< * """This sanity check was added to the constructor, but you can access it from outside this class * to determine whether or not you are providing a valid stem to BinaryPhraseTable.""" */ - __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5moses_7dictree_20PhraseDictionaryTree_5isValidBinaryTable, NULL, __pyx_n_s_17); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5moses_7dictree_20PhraseDictionaryTree_5isValidBinaryTable, NULL, __pyx_n_s_17); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); __Pyx_GIVEREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5moses_7dictree_PhraseDictionaryTree->tp_dict, __pyx_n_s__isValidBinaryTable, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem((PyObject *)__pyx_ptype_5moses_7dictree_PhraseDictionaryTree->tp_dict, __pyx_n_s__isValidBinaryTable, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5moses_7dictree_PhraseDictionaryTree); - /* "moses/dictree.pyx":95 + /* "moses/dictree.pyx":96 * del self.__tree * * @staticmethod # <<<<<<<<<<<<<< * def isValidBinaryTable(stem, bint wa = False): * """This sanity check was added to the constructor, but you can access it from outside this class */ - __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5moses_7dictree_PhraseDictionaryTree, __pyx_n_s__isValidBinaryTable); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5moses_7dictree_PhraseDictionaryTree, __pyx_n_s__isValidBinaryTable); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); __Pyx_GIVEREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5moses_7dictree_PhraseDictionaryTree->tp_dict, __pyx_n_s__isValidBinaryTable, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem((PyObject *)__pyx_ptype_5moses_7dictree_PhraseDictionaryTree->tp_dict, __pyx_n_s__isValidBinaryTable, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5moses_7dictree_PhraseDictionaryTree); - /* "moses/dictree.pyx":112 + /* "moses/dictree.pyx":113 * and os.path.isfile(stem + ".binphr.tgtvoc") * * def query(self, line, cmp = None, key = lambda x: x.scores[0], top = 0): # <<<<<<<<<<<<<< * """Queries the phrase table and returns a list of matches. * Each match is a QueryResult. */ - __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5moses_7dictree_20PhraseDictionaryTree_5query_lambda2, 0, NULL, __pyx_n_s_17, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5moses_7dictree_20PhraseDictionaryTree_5query_lambda2, 0, NULL, __pyx_n_s_17, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_k_16 = __pyx_t_1; __Pyx_GIVEREF(__pyx_t_1); @@ -4196,6 +4690,14 @@ PyMODINIT_FUNC PyInit_dictree(void) __Pyx_GOTREF(((PyObject *)__pyx_t_1)); if (PyObject_SetAttr(__pyx_m, __pyx_n_s____test__, ((PyObject *)__pyx_t_1)) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + + /* "string.from_py":11 + * + * @cname("__pyx_convert_string_from_py_") + * cdef string __pyx_convert_string_from_py_(object o) except *: # <<<<<<<<<<<<<< + * return string(o, len(o)) + * + */ goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); diff --git a/contrib/python/moses/dictree.pyx b/contrib/python/moses/dictree.pyx index 376080c4f..95a1b0053 100644 --- a/contrib/python/moses/dictree.pyx +++ b/contrib/python/moses/dictree.pyx @@ -5,6 +5,7 @@ from libcpp.vector cimport vector import os import cython cimport cdictree +cimport condiskpt cpdef int fsign(float x): """Simply returns the sign of float x (zero is assumed +), it's defined here just so one gains a little bit with static typing""" @@ -87,7 +88,7 @@ cdef class PhraseDictionaryTree(object): self.delimiters = delimiters self.__tree = new cdictree.PhraseDictionaryTree(nscores) self.__tree.UseWordAlignment(wa) - self.__tree.Read(string(path)) + self.__tree.Read(path) def __dealloc__(self): del self.__tree @@ -115,7 +116,7 @@ cdef class PhraseDictionaryTree(object): If 'cmp' is defined the return list is sorted. If 'top' is defined, onlye the top elements will be returned.""" cdef bytes text = as_str(line) - cdef vector[string] fphrase = cdictree.Tokenize(string(text), string(self.delimiters)) + cdef vector[string] fphrase = cdictree.Tokenize(text, self.delimiters) cdef vector[cdictree.StringTgtCand]* rv = new vector[cdictree.StringTgtCand]() cdef vector[string]* wa = NULL cdef list phrases @@ -134,4 +135,20 @@ cdef class PhraseDictionaryTree(object): return phrases[0:top] else: return phrases + +cdef class OnDiskWrapper(object): + + cdef condiskpt.OnDiskWrapper* wrapper + cdef readonly bytes delimiters + + def __cinit__(self, bytes path, delimiters = ' \t'): + self.delimiters = delimiters + self.wrapper = new condiskpt.OnDiskWrapper() + self.wrapper.BeginLoad(string(path)) + + def query(self, line): + cdef bytes text = as_str(line) + cdef vector[string] ftokens = cdictree.Tokenize(text, self.delimiters) + #print 'query:', ftokens + #cdef condiskpt.SourcePhrase fphrase = condiskpt.Tokenize(ftokens, self.wrapper[0]) diff --git a/contrib/python/setup.py b/contrib/python/setup.py index 9c730c93e..fc88e08e1 100644 --- a/contrib/python/setup.py +++ b/contrib/python/setup.py @@ -4,7 +4,7 @@ import sys import os mosesdir = os.path.abspath('../../') -includes = [mosesdir, os.path.join(mosesdir, 'moses')] +includes = [mosesdir, os.path.join(mosesdir, 'moses'), os.path.join(mosesdir, 'OnDiskPt')] libdir = os.path.join(mosesdir, 'lib') # options