moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-27 14:05:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

uncomment out lines that clang choked on. Now works

Browse Source
This commit is contained in:
Hieu Hoang 2015-04-02 22:39:44 +04:00
parent 00065fee15
commit 0c58e19491
1 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
contrib/c++tokenizer/tokenizer.cpp
View File

@ -1589,19 +1589,19 @@ Tokenizer::tokenize(std::istream& is, std::ostream& os)
results[ithread].resize(line_pos);
break;
}
//lines[ithread][line_pos].clear(); TODO clang error
lines[ithread][line_pos].clear();
} else if (skip_xml_p &&
(RE2::FullMatch(istr,tag_line_x) || RE2::FullMatch(istr,white_line_x))) {
//lines[ithread][line_pos].clear(); TODO clang error
lines[ithread][line_pos].clear();
} else {
//lines[ithread][line_pos] =
// std::string(SPC_BYTE).append(istr).append(SPC_BYTE); TODO clang error
lines[ithread][line_pos] =
std::string(SPC_BYTE).append(istr).append(SPC_BYTE);
}
}
if (line_pos) {
//workers[ithread] =
// boost::thread(VectorTokenizerCallable(this,lines[ithread],results[ithread])); TODO clang error
workers[ithread] =
boost::thread(VectorTokenizerCallable(this,lines[ithread],results[ithread]));
}
} // end for loop starting threads
@ -1772,7 +1772,7 @@ Tokenizer::splitter(const std::string &istr, bool *continuation_ptr) {
std::size_t finilen = 0;
std::size_t dotslen = 0;
#define SEQ_LIM 6
const std::size_t SEQ_LIM = 6;
charclass_t prev_class = empty;
charclass_t curr_class = empty;