moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-11-13 10:40:52 +03:00
Code Issues Packages Projects Releases Wiki Activity
7125096c29
mosesdecoder/contrib/c++tokenizer/tokenizer.cpp
Jeroen Vermeulen eca5824100 Remove trailing whitespace in C++ files.
2015-04-30 12:05:11 +07:00

2247 lines
81 KiB
C++
Raw Blame History

#include "tokenizer.h"
#include <re2/stringpiece.h>
#include <sstream>
#include <iterator>
#include <memory>
#include <vector>
#include <algorithm>
#include <cstring>
#include <set>
#include <glib.h>
#include <stdexcept>
#include <boost/thread.hpp>
namespace { // anonymous namespace
// frequently used regexp's are pre-compiled thus:
RE2 genl_tags_x("<[/!\\p{L}]+[^>]*>");
RE2 mult_spc_x(" +"); // multiple spaces
RE2 tag_line_x("^<.+>$"); // lines beginning and ending with open/close angle-bracket pairs
RE2 white_line_x("^\\s*$"); // lines entirely composed of whitespace
RE2 slash_x("([\\p{L}\\p{N}])(/)([\\p{L}\\p{N}])"); // and slash-conjoined " "
RE2 final_x("([^.])([.])([\\]\\)}>\"']*) ?$"); // sentence-final punctuation sequence (non qm em)
RE2 qx_x("([?!])"); // one qm/em mark
RE2 braces_x("([\\]\\[\\(\\){}<>])"); // any open or close of a pair
RE2 endq_x("([^'])' "); // post-token single-quote or doubled single-quote
RE2 letter_x("\\p{L}"); // a letter
RE2 lower_x("^\\p{Ll}"); // a lower-case letter
RE2 sinteger_x("^\\p{N}"); // not a digit mark
RE2 numprefixed_x("[-+/.@\\\\#\\%&\\p{Sc}\\p{N}]*[\\p{N}]+-[-'`\"\\p{L}]*\\p{L}");
RE2 quasinumeric_x("[-.;:@\\\\#\%&\\p{Sc}\\p{So}\\p{N}]*[\\p{N}]+");
RE2 numscript_x("([\\p{N}\\p{L}])([\\p{No}]+)(\\p{Ll})");
RE2 x1_v_d("([ ([{<])\""); // a valid non-letter preceeding a double-quote
RE2 x1_v_gg("([ ([{<])``"); // a valid non-letter preceeding directional doubled open single-quote
RE2 x1_v_g("([ ([{<])`([^`])"); // a valid non-letter preceeding directional unitary single-quote
RE2 x1_v_q("([ ([{<])'"); // a valid non-letter preceeding undirected embedded quotes
RE2 ndndcomma_x("([^\\p{N}]),([^\\p{N}])"); // non-digit,non-digit
RE2 pdndcomma_x("([\\p{N}]),([^\\p{N}])"); // digit,non-digit
RE2 ndpdcomma_x("([^\\p{N}]),([\\p{N}])"); // non-digit,digit
RE2 symbol_x("([;:@\\#\\$%&\\p{Sc}\\p{So}])"); // usable punctuation mark not a quote or a brace
RE2 contract_x("'([sSmMdD]) "); // english single letter contraction forms, as embedded
RE2 right_x("[({¿¡]+"); // symbols which conjoin to the right
RE2 left_x("[,.?!:;\\%\\p{Sc}})]+"); // symbols conjoin to the left
RE2 curr_en_x("^[Nn]?[\'][\\p{L}]"); // english contraction suffixes conjoin to the left
RE2 pre_en_x(".*[\\p{L}\\p{N}]+$"); // valid english contraction prefixes
RE2 curr_fr_x(".*[\\p{L}\\p{N}]+[\']"); // french/italian contraction prefixes conjoin to the right
RE2 post_fr_x("^[\\p{L}\\p{N}]*"); // valid french/italian contraction suffixes
// anything rarely used will just be given as a string and compiled on demand by RE2
const char *
SPC_BYTE = " ";
//const char *
//URL_VALID_SYM_CHARS = "-._~:/?#[]@!$&'()*+,;=";
inline bool
class_follows_p(gunichar *s, gunichar *e, GUnicodeType gclass) {
while (s < e) {
GUnicodeType tclass = g_unichar_type(*s);
if (tclass == gclass)
return true;
switch (tclass) {
case G_UNICODE_SPACING_MARK:
case G_UNICODE_LINE_SEPARATOR:
case G_UNICODE_PARAGRAPH_SEPARATOR:
case G_UNICODE_SPACE_SEPARATOR:
++s;
continue;
break;
default:
return false;
}
}
return false;
}
const char *ESCAPE_MOSES[] = {
"&#124;", // | 0
"&#91;", // [ 1
"&#93;", // ] 2
"&amp;", // & 3 (26)
"&lt;", // < 4 (3c)
"&gt;", // > 5 (3e)
"&apos;", // ' 6 (27)
"&quot;", // " 7 (22)
};
const std::set<std::string>
ESCAPE_SET = {
std::string(ESCAPE_MOSES[0]),
std::string(ESCAPE_MOSES[1]),
std::string(ESCAPE_MOSES[2]),
std::string(ESCAPE_MOSES[3]),
std::string(ESCAPE_MOSES[4]),
std::string(ESCAPE_MOSES[5]),
std::string(ESCAPE_MOSES[6]),
std::string(ESCAPE_MOSES[7]),
};
const std::map<std::wstring,gunichar>
ENTITY_MAP = {
{ std::wstring(L"&quot;"), L'"' },
{ std::wstring(L"&amp;"), L'&' },
{ std::wstring(L"&apos;"), L'\'' },
{ std::wstring(L"&lt;"), L'<' },
{ std::wstring(L"&gt;"), L'>' },
{ std::wstring(L"&nbsp;"), L'\u00A0' },
{ std::wstring(L"&iexcl;"), L'\u00A1' },
{ std::wstring(L"&cent;"), L'\u00A2' },
{ std::wstring(L"&pound;"), L'\u00A3' },
{ std::wstring(L"&curren;"), L'\u00A4' },
{ std::wstring(L"&yen;"), L'\u00A5' },
{ std::wstring(L"&brvbar;"), L'\u00A6' },
{ std::wstring(L"&sect;"), L'\u00A7' },
{ std::wstring(L"&uml;"), L'\u00A8' },
{ std::wstring(L"&copy;"), L'\u00A9' },
{ std::wstring(L"&ordf;"), L'\u00AA' },
{ std::wstring(L"&laquo;"), L'\u00AB' },
{ std::wstring(L"&not;"), L'\u00AC' },
{ std::wstring(L"&shy;"), L'\u00AD' },
{ std::wstring(L"&reg;"), L'\u00AE' },
{ std::wstring(L"&macr;"), L'\u00AF' },
{ std::wstring(L"&deg;"), L'\u00B0' },
{ std::wstring(L"&plusmn;"), L'\u00B1' },
{ std::wstring(L"&sup2;"), L'\u00B2' },
{ std::wstring(L"&sup3;"), L'\u00B3' },
{ std::wstring(L"&acute;"), L'\u00B4' },
{ std::wstring(L"&micro;"), L'\u00B5' },
{ std::wstring(L"&para;"), L'\u00B6' },
{ std::wstring(L"&middot;"), L'\u00B7' },
{ std::wstring(L"&cedil;"), L'\u00B8' },
{ std::wstring(L"&sup1;"), L'\u00B9' },
{ std::wstring(L"&ordm;"), L'\u00BA' },
{ std::wstring(L"&raquo;"), L'\u00BB' },
{ std::wstring(L"&frac14;"), L'\u00BC' },
{ std::wstring(L"&frac12;"), L'\u00BD' },
{ std::wstring(L"&frac34;"), L'\u00BE' },
{ std::wstring(L"&iquest;"), L'\u00BF' },
{ std::wstring(L"&Agrave;"), L'\u00C0' },
{ std::wstring(L"&Aacute;"), L'\u00C1' },
{ std::wstring(L"&Acirc;"), L'\u00C2' },
{ std::wstring(L"&Atilde;"), L'\u00C3' },
{ std::wstring(L"&Auml;"), L'\u00C4' },
{ std::wstring(L"&Aring;"), L'\u00C5' },
{ std::wstring(L"&AElig;"), L'\u00C6' },
{ std::wstring(L"&Ccedil;"), L'\u00C7' },
{ std::wstring(L"&Egrave;"), L'\u00C8' },
{ std::wstring(L"&Eacute;"), L'\u00C9' },
{ std::wstring(L"&Ecirc;"), L'\u00CA' },
{ std::wstring(L"&Euml;"), L'\u00CB' },
{ std::wstring(L"&Igrave;"), L'\u00CC' },
{ std::wstring(L"&Iacute;"), L'\u00CD' },
{ std::wstring(L"&Icirc;"), L'\u00CE' },
{ std::wstring(L"&Iuml;"), L'\u00CF' },
{ std::wstring(L"&ETH;"), L'\u00D0' },
{ std::wstring(L"&Ntilde;"), L'\u00D1' },
{ std::wstring(L"&Ograve;"), L'\u00D2' },
{ std::wstring(L"&Oacute;"), L'\u00D3' },
{ std::wstring(L"&Ocirc;"), L'\u00D4' },
{ std::wstring(L"&Otilde;"), L'\u00D5' },
{ std::wstring(L"&Ouml;"), L'\u00D6' },
{ std::wstring(L"&times;"), L'\u00D7' },
{ std::wstring(L"&Oslash;"), L'\u00D8' },
{ std::wstring(L"&Ugrave;"), L'\u00D9' },
{ std::wstring(L"&Uacute;"), L'\u00DA' },
{ std::wstring(L"&Ucirc;"), L'\u00DB' },
{ std::wstring(L"&Uuml;"), L'\u00DC' },
{ std::wstring(L"&Yacute;"), L'\u00DD' },
{ std::wstring(L"&THORN;"), L'\u00DE' },
{ std::wstring(L"&szlig;"), L'\u00DF' },
{ std::wstring(L"&agrave;"), L'\u00E0' },
{ std::wstring(L"&aacute;"), L'\u00E1' },
{ std::wstring(L"&acirc;"), L'\u00E2' },
{ std::wstring(L"&atilde;"), L'\u00E3' },
{ std::wstring(L"&auml;"), L'\u00E4' },
{ std::wstring(L"&aring;"), L'\u00E5' },
{ std::wstring(L"&aelig;"), L'\u00E6' },
{ std::wstring(L"&ccedil;"), L'\u00E7' },
{ std::wstring(L"&egrave;"), L'\u00E8' },
{ std::wstring(L"&eacute;"), L'\u00E9' },
{ std::wstring(L"&ecirc;"), L'\u00EA' },
{ std::wstring(L"&euml;"), L'\u00EB' },
{ std::wstring(L"&igrave;"), L'\u00EC' },
{ std::wstring(L"&iacute;"), L'\u00ED' },
{ std::wstring(L"&icirc;"), L'\u00EE' },
{ std::wstring(L"&iuml;"), L'\u00EF' },
{ std::wstring(L"&eth;"), L'\u00F0' },
{ std::wstring(L"&ntilde;"), L'\u00F1' },
{ std::wstring(L"&ograve;"), L'\u00F2' },
{ std::wstring(L"&oacute;"), L'\u00F3' },
{ std::wstring(L"&ocirc;"), L'\u00F4' },
{ std::wstring(L"&otilde;"), L'\u00F5' },
{ std::wstring(L"&ouml;"), L'\u00F6' },
{ std::wstring(L"&divide;"), L'\u00F7' },
{ std::wstring(L"&oslash;"), L'\u00F8' },
{ std::wstring(L"&ugrave;"), L'\u00F9' },
{ std::wstring(L"&uacute;"), L'\u00FA' },
{ std::wstring(L"&ucirc;"), L'\u00FB' },
{ std::wstring(L"&uuml;"), L'\u00FC' },
{ std::wstring(L"&yacute;"), L'\u00FD' },
{ std::wstring(L"&thorn;"), L'\u00FE' },
{ std::wstring(L"&yuml;"), L'\u00FF' },
{ std::wstring(L"&OElig;"), L'\u0152' },
{ std::wstring(L"&oelig;"), L'\u0153' },
{ std::wstring(L"&Scaron;"), L'\u0160' },
{ std::wstring(L"&scaron;"), L'\u0161' },
{ std::wstring(L"&Yuml;"), L'\u0178' },
{ std::wstring(L"&fnof;"), L'\u0192' },
{ std::wstring(L"&circ;"), L'\u02C6' },
{ std::wstring(L"&tilde;"), L'\u02DC' },
{ std::wstring(L"&Alpha;"), L'\u0391' },
{ std::wstring(L"&Beta;"), L'\u0392' },
{ std::wstring(L"&Gamma;"), L'\u0393' },
{ std::wstring(L"&Delta;"), L'\u0394' },
{ std::wstring(L"&Epsilon;"), L'\u0395' },
{ std::wstring(L"&Zeta;"), L'\u0396' },
{ std::wstring(L"&Eta;"), L'\u0397' },
{ std::wstring(L"&Theta;"), L'\u0398' },
{ std::wstring(L"&Iota;"), L'\u0399' },
{ std::wstring(L"&Kappa;"), L'\u039A' },
{ std::wstring(L"&Lambda;"), L'\u039B' },
{ std::wstring(L"&Mu;"), L'\u039C' },
{ std::wstring(L"&Nu;"), L'\u039D' },
{ std::wstring(L"&Xi;"), L'\u039E' },
{ std::wstring(L"&Omicron;"), L'\u039F' },
{ std::wstring(L"&Pi;"), L'\u03A0' },
{ std::wstring(L"&Rho;"), L'\u03A1' },
{ std::wstring(L"&Sigma;"), L'\u03A3' },
{ std::wstring(L"&Tau;"), L'\u03A4' },
{ std::wstring(L"&Upsilon;"), L'\u03A5' },
{ std::wstring(L"&Phi;"), L'\u03A6' },
{ std::wstring(L"&Chi;"), L'\u03A7' },
{ std::wstring(L"&Psi;"), L'\u03A8' },
{ std::wstring(L"&Omega;"), L'\u03A9' },
{ std::wstring(L"&alpha;"), L'\u03B1' },
{ std::wstring(L"&beta;"), L'\u03B2' },
{ std::wstring(L"&gamma;"), L'\u03B3' },
{ std::wstring(L"&delta;"), L'\u03B4' },
{ std::wstring(L"&epsilon;"), L'\u03B5' },
{ std::wstring(L"&zeta;"), L'\u03B6' },
{ std::wstring(L"&eta;"), L'\u03B7' },
{ std::wstring(L"&theta;"), L'\u03B8' },
{ std::wstring(L"&iota;"), L'\u03B9' },
{ std::wstring(L"&kappa;"), L'\u03BA' },
{ std::wstring(L"&lambda;"), L'\u03BB' },
{ std::wstring(L"&mu;"), L'\u03BC' },
{ std::wstring(L"&nu;"), L'\u03BD' },
{ std::wstring(L"&xi;"), L'\u03BE' },
{ std::wstring(L"&omicron;"), L'\u03BF' },
{ std::wstring(L"&pi;"), L'\u03C0' },
{ std::wstring(L"&rho;"), L'\u03C1' },
{ std::wstring(L"&sigmaf;"), L'\u03C2' },
{ std::wstring(L"&sigma;"), L'\u03C3' },
{ std::wstring(L"&tau;"), L'\u03C4' },
{ std::wstring(L"&upsilon;"), L'\u03C5' },
{ std::wstring(L"&phi;"), L'\u03C6' },
{ std::wstring(L"&chi;"), L'\u03C7' },
{ std::wstring(L"&psi;"), L'\u03C8' },
{ std::wstring(L"&omega;"), L'\u03C9' },
{ std::wstring(L"&thetasym;"), L'\u03D1' },
{ std::wstring(L"&upsih;"), L'\u03D2' },
{ std::wstring(L"&piv;"), L'\u03D6' },
{ std::wstring(L"&ensp;"), L'\u2002' },
{ std::wstring(L"&emsp;"), L'\u2003' },
{ std::wstring(L"&thinsp;"), L'\u2009' },
{ std::wstring(L"&zwnj;"), L'\u200C' },
{ std::wstring(L"&zwj;"), L'\u200D' },
{ std::wstring(L"&lrm;"), L'\u200E' },
{ std::wstring(L"&rlm;"), L'\u200F' },
{ std::wstring(L"&ndash;"), L'\u2013' },
{ std::wstring(L"&mdash;"), L'\u2014' },
{ std::wstring(L"&lsquo;"), L'\u2018' },
{ std::wstring(L"&rsquo;"), L'\u2019' },
{ std::wstring(L"&sbquo;"), L'\u201A' },
{ std::wstring(L"&ldquo;"), L'\u201C' },
{ std::wstring(L"&rdquo;"), L'\u201D' },
{ std::wstring(L"&bdquo;"), L'\u201E' },
{ std::wstring(L"&dagger;"), L'\u2020' },
{ std::wstring(L"&Dagger;"), L'\u2021' },
{ std::wstring(L"&bull;"), L'\u2022' },
{ std::wstring(L"&hellip;"), L'\u2026' },
{ std::wstring(L"&permil;"), L'\u2030' },
{ std::wstring(L"&prime;"), L'\u2032' },
{ std::wstring(L"&Prime;"), L'\u2033' },
{ std::wstring(L"&lsaquo;"), L'\u2039' },
{ std::wstring(L"&rsaquo;"), L'\u203A' },
{ std::wstring(L"&oline;"), L'\u203E' },
{ std::wstring(L"&frasl;"), L'\u2044' },
{ std::wstring(L"&euro;"), L'\u20AC' },
{ std::wstring(L"&image;"), L'\u2111' },
{ std::wstring(L"&weierp;"), L'\u2118' },
{ std::wstring(L"&real;"), L'\u211C' },
{ std::wstring(L"&trade;"), L'\u2122' },
{ std::wstring(L"&alefsym;"), L'\u2135' },
{ std::wstring(L"&larr;"), L'\u2190' },
{ std::wstring(L"&uarr;"), L'\u2191' },
{ std::wstring(L"&rarr;"), L'\u2192' },
{ std::wstring(L"&darr;"), L'\u2193' },
{ std::wstring(L"&harr;"), L'\u2194' },
{ std::wstring(L"&crarr;"), L'\u21B5' },
{ std::wstring(L"&lArr;"), L'\u21D0' },
{ std::wstring(L"&uArr;"), L'\u21D1' },
{ std::wstring(L"&rArr;"), L'\u21D2' },
{ std::wstring(L"&dArr;"), L'\u21D3' },
{ std::wstring(L"&hArr;"), L'\u21D4' },
{ std::wstring(L"&forall;"), L'\u2200' },
{ std::wstring(L"&part;"), L'\u2202' },
{ std::wstring(L"&exist;"), L'\u2203' },
{ std::wstring(L"&empty;"), L'\u2205' },
{ std::wstring(L"&nabla;"), L'\u2207' },
{ std::wstring(L"&isin;"), L'\u2208' },
{ std::wstring(L"&notin;"), L'\u2209' },
{ std::wstring(L"&ni;"), L'\u220B' },
{ std::wstring(L"&prod;"), L'\u220F' },
{ std::wstring(L"&sum;"), L'\u2211' },
{ std::wstring(L"&minus;"), L'\u2212' },
{ std::wstring(L"&lowast;"), L'\u2217' },
{ std::wstring(L"&radic;"), L'\u221A' },
{ std::wstring(L"&prop;"), L'\u221D' },
{ std::wstring(L"&infin;"), L'\u221E' },
{ std::wstring(L"&ang;"), L'\u2220' },
{ std::wstring(L"&and;"), L'\u2227' },
{ std::wstring(L"&or;"), L'\u2228' },
{ std::wstring(L"&cap;"), L'\u2229' },
{ std::wstring(L"&cup;"), L'\u222A' },
{ std::wstring(L"&int;"), L'\u222B' },
{ std::wstring(L"&there4;"), L'\u2234' },
{ std::wstring(L"&sim;"), L'\u223C' },
{ std::wstring(L"&cong;"), L'\u2245' },
{ std::wstring(L"&asymp;"), L'\u2248' },
{ std::wstring(L"&ne;"), L'\u2260' },
{ std::wstring(L"&equiv;"), L'\u2261' },
{ std::wstring(L"&le;"), L'\u2264' },
{ std::wstring(L"&ge;"), L'\u2265' },
{ std::wstring(L"&sub;"), L'\u2282' },
{ std::wstring(L"&sup;"), L'\u2283' },
{ std::wstring(L"&nsub;"), L'\u2284' },
{ std::wstring(L"&sube;"), L'\u2286' },
{ std::wstring(L"&supe;"), L'\u2287' },
{ std::wstring(L"&oplus;"), L'\u2295' },
{ std::wstring(L"&otimes;"), L'\u2297' },
{ std::wstring(L"&perp;"), L'\u22A5' },
{ std::wstring(L"&sdot;"), L'\u22C5' },
{ std::wstring(L"&lceil;"), L'\u2308' },
{ std::wstring(L"&rceil;"), L'\u2309' },
{ std::wstring(L"&lfloor;"), L'\u230A' },
{ std::wstring(L"&rfloor;"), L'\u230B' },
{ std::wstring(L"&lang;"), L'\u2329' },
{ std::wstring(L"&rang;"), L'\u232A' },
{ std::wstring(L"&loz;"), L'\u25CA' },
{ std::wstring(L"&spades;"), L'\u2660' },
{ std::wstring(L"&clubs;"), L'\u2663' },
{ std::wstring(L"&hearts;"), L'\u2665' },
{ std::wstring(L"&diams;"), L'\u2666' }
};
inline gunichar
get_entity(gunichar *ptr, size_t len) {
// try hex, decimal entity first
gunichar ech(0);
if (ptr[1] == gunichar(L'#') && len > 3) {
std::wstringstream wss;
int wch = 0;
try {
wss << std::hex << std::wstring((wchar_t *)(ptr+2),len-3);
wss >> wch;
ech = gunichar(wch);
} catch (...) {
ech = 0;
}
} else if (g_unichar_type(ptr[1]) == G_UNICODE_DECIMAL_NUMBER) {
std::wstringstream wss;
int wch = 0;
try {
wss << std::dec << std::wstring((wchar_t *)(ptr+1),len-2);
wss >> wch;
ech = gunichar(wch);
} catch (...) {
ech = 0;
}
}
if (ech)
return ech;
std::map<std::wstring,gunichar>::const_iterator it =
ENTITY_MAP.find(std::wstring((wchar_t *)(ptr),len));
return it != ENTITY_MAP.end() ? it->second : gunichar(0);
}
inline gunichar
get_entity(char *ptr, size_t len) {
glong ulen = 0;
gunichar *gtmp = g_utf8_to_ucs4_fast((const gchar *)ptr, len, &ulen);
gunichar gch = get_entity(gtmp,ulen);
g_free(gtmp);
return gch;
}
inline std::string
trim(const std::string& in)
{
std::size_t start = 0;
std::size_t limit = in.size();
while (start < limit && in.at(start) < '!') ++start;
while (start < limit && in.at(limit-1) < '!') --limit;
if (start == limit) return std::string("");
if (start > 0 || limit < in.size())
return in.substr(start,limit-start);
return std::string(in);
}
inline std::vector<std::string>
split(const std::string& in)
{
std::vector<std::string> outv;
std::istringstream iss(in);
std::copy(std::istream_iterator<std::string>(iss),
std::istream_iterator<std::string>(),
std::back_inserter(outv));
return outv;
}
}; // end anonymous namespace
#ifdef TOKENIZER_NAMESPACE
namespace TOKENIZER_NAMESPACE {
#endif
void
Tokenizer::set_config_dir(const std::string& dir) {
if (dir.empty()) {
cfg_dir = ".";
} else {
cfg_dir.assign(dir);
}
}
Tokenizer::Tokenizer(const Parameters& _)
: nthreads(_.nthreads ? _.nthreads : 1)
, chunksize(_.chunksize)
, lang_iso(_.lang_iso)
, english_p(_.lang_iso.compare("en")==0)
, latin_p((!english_p) && (_.lang_iso.compare("fr")==0 || _.lang_iso.compare("it")==0))
, skip_xml_p(_.detag_p)
, skip_alltags_p(_.alltag_p)
, entities_p(_.entities_p)
, escape_p(_.escape_p)
, unescape_p(_.unescape_p)
, aggressive_hyphen_p(_.aggro_p)
, supersub_p(_.supersub_p)
, url_p(_.url_p)
, downcase_p(_.downcase_p)
, normalize_p(_.normalize_p)
, penn_p(_.penn_p)
, narrow_latin_p(_.narrow_latin_p)
, narrow_kana_p(_.narrow_kana_p)
, refined_p(_.refined_p)
, drop_bad_p(_.drop_bad_p)
, splits_p(_.split_p)
, verbose_p(_.verbose_p)
, para_marks_p(_.para_marks_p)
, split_breaks_p(_.split_breaks_p)
{
if (_.cfg_path)
set_config_dir(_.cfg_path);
}
//
// dtor deletes dynamically allocated per-language RE2 compiled expressions
//
Tokenizer::~Tokenizer()
{
for (auto& ptr : prot_pat_vec) {
if (ptr == &numprefixed_x || ptr == &quasinumeric_x)
continue;
delete ptr;
}
}
//
// stuffs numeric-only prefixes into nbpre_num_set,
// others into nbpre_gen_set
//
std::pair<int,int>
Tokenizer::load_prefixes(std::ifstream& ifs)
{
RE2 numonly("(.*)[\\s]+(\\#NUMERIC_ONLY\\#)");
std::string line;
int nnon = 0;
int nnum = 0;
while (std::getline(ifs,line)) {
if (!line.empty() && line[0] != '#') {
std::string prefix;
if (RE2::PartialMatch(line,numonly,&prefix)) {
nbpre_num_set.insert(prefix);
gunichar * x=g_utf8_to_ucs4_fast((const gchar *)prefix.c_str(),prefix.size(),0);
nbpre_num_ucs4.insert(std::wstring((wchar_t *)x));
g_free(x);
nnum++;
} else {
nbpre_gen_set.insert(line);
gunichar * x=g_utf8_to_ucs4_fast((const gchar *)line.c_str(),line.size(),0);
nbpre_gen_ucs4.insert(std::wstring((wchar_t *)x));
g_free(x);
nnon++;
}
}
}
return std::make_pair(nnon,nnum);
}
//
// load files (make sure to call set_config_dir before, if ever
// for nonbreaking prefixes and protected patterns
//
void
Tokenizer::init(const char *cfg_dir_optional) {
if (cfg_dir_optional)
set_config_dir(std::string(cfg_dir_optional));
std::string dir_path(cfg_dir);
dir_path.append("/nonbreaking_prefixes");
if (::access(dir_path.c_str(),X_OK)) {
dir_path = cfg_dir;
}
std::string nbpre_path(dir_path);
nbpre_path.append("/nonbreaking_prefix.").append(lang_iso);
// default to generic version
if (::access(nbpre_path.c_str(),R_OK))
nbpre_path = nbpre_path.substr(0,nbpre_path.size()-lang_iso.size()-1);
if (::access(nbpre_path.c_str(),R_OK) == 0) {
std::ifstream cfg(nbpre_path.c_str());
try {
std::pair<int,int> counts = load_prefixes(cfg);
if (verbose_p) {
std::cerr << "loaded " << counts.first << " non-numeric, "
<< counts.second << " numeric prefixes from "
<< nbpre_path << std::endl;
}
} catch (...) {
std::ostringstream ess;
ess << "I/O error reading " << nbpre_path << " in " << __FILE__ << " at " << __LINE__;
throw std::runtime_error(ess.str());
}
} else if (verbose_p) {
std::cerr << "no prefix file found: " << nbpre_path << std::endl;
}
if (nbpre_gen_set.empty() && nbpre_num_set.empty()) {
std::ostringstream ess;
ess << "Error at " << __FILE__ << ":" << __LINE__ << " : "
<< "No known abbreviations for language " << lang_iso;
throw std::runtime_error(ess.str());
}
std::string protpat_path(cfg_dir);
protpat_path.append("/protected_pattern.").append(lang_iso);
// default to generic version
if (::access(protpat_path.c_str(),R_OK))
protpat_path = protpat_path.substr(0,protpat_path.size()-lang_iso.size()-1);
prot_pat_vec.push_back(&numprefixed_x);
prot_pat_vec.push_back(&quasinumeric_x);
if (::access(protpat_path.c_str(),R_OK) == 0) {
std::ifstream cfg(protpat_path.c_str());
char linebuf[1028];
int npat = 0;
try {
linebuf[0]='(';
while (cfg.good()) {
cfg.getline(linebuf+1,1024);
if (linebuf[1] && linebuf[1] != '#') {
strcat(linebuf,")");
prot_pat_vec.push_back(new RE2(linebuf));
npat++;
}
}
} catch (...) {
std::ostringstream ess;
ess << "I/O error reading " << protpat_path << " in " << __FILE__ << " at " << __LINE__;
throw std::runtime_error(ess.str());
}
if (verbose_p) {
std::cerr << "loaded " << npat << " protected patterns from "
<< protpat_path << std::endl;
}
} else if (verbose_p) {
std::cerr << "no protected file found: " << protpat_path << std::endl;
}
}
void
Tokenizer::reset() {
}
//
// apply ctor-selected tokenization to a string, in-place, no newlines allowed,
// assumes protections are applied already, some invariants are in place,
// e.g. that successive chars <= ' ' have been normalized to a single ' '
//
void
Tokenizer::protected_tokenize(std::string& text) {
std::vector<re2::StringPiece> words;
re2::StringPiece textpc(text);
int pos = 0;
if (textpc[pos] == ' ')
++pos;
size_t next = text.find(' ',pos);
while (next != std::string::npos) {
if (next - pos)
words.push_back(textpc.substr(pos,next-pos));
pos = next + 1;
while (pos < textpc.size() && textpc[pos] == ' ')
++pos;
next = textpc.find(' ',pos);
}
if (pos < textpc.size() && textpc[pos] != ' ')
words.push_back(textpc.substr(pos,textpc.size()-pos));
// regurgitate words with look-ahead handling for tokens with final mumble
std::string outs;
std::size_t nwords(words.size());
for (size_t ii = 0; ii < nwords; ++ii) {
bool more_p = ii < nwords - 1;
size_t len = words[ii].size();
bool sentence_break_p = len > 1 && words[ii][len-1] == '.';
// suppress break if it is an non-breaking prefix
if (sentence_break_p) {
re2::StringPiece pfx(words[ii].substr(0,len-1));
std::string pfxs(pfx.as_string());
if (nbpre_gen_set.find(pfxs) != nbpre_gen_set.end()) {
// general non-breaking prefix
sentence_break_p = false;
} else if (more_p && nbpre_num_set.find(pfxs) != nbpre_num_set.end() && RE2::PartialMatch(words[ii+1],sinteger_x)) {
// non-breaking before numeric
sentence_break_p = false;
} else if (pfxs.find('.') != std::string::npos && RE2::PartialMatch(pfx,letter_x)) {
// terminal isolated letter does not break
sentence_break_p = false;
} else if (more_p && RE2::PartialMatch(words[ii+1],lower_x)) {
// lower-case look-ahead does not break
sentence_break_p = false;
}
}
outs.append(words[ii].data(),len);
if (sentence_break_p)
outs.append(" .");
if (more_p)
outs.append(SPC_BYTE,1);
}
text.assign(outs.begin(),outs.end());
}
bool
Tokenizer::unescape(std::string& word) {
std::ostringstream oss;
std::size_t was = 0; // last processed
std::size_t pos = 0; // last unprocessed
std::size_t len = 0; // processed length
bool hit = false;
for (std::size_t endp=0;
(pos = word.find('&',was)) != std::string::npos && (endp = word.find(';',pos)) != std::string::npos;
was = endp == std::string::npos ? pos : 1+endp) {
len = endp - pos + 1;
glong ulen(0);
gunichar *gtmp = g_utf8_to_ucs4_fast((const gchar *)word.c_str()+pos, len, &ulen);
gunichar gbuf[2] = { 0 };
if ((gbuf[0] = get_entity(gtmp,ulen)) != gunichar(0)) {
gchar *gstr = g_ucs4_to_utf8(gbuf,ulen,0,0,0);
if (escape_p && ESCAPE_SET.find(std::string(gstr)) != ESCAPE_SET.end()) {
// do not unescape moses escapes when escape flag is turned on
oss << word.substr(was,1+endp-was);
} else {
if (was < pos)
oss << word.substr(was,pos-was);
oss << gstr;
was += ulen;
hit = true;
}
g_free(gstr);
} else {
oss << word.substr(was,1+endp-was);
}
g_free(gtmp);
}
if (was < word.size())
oss << word.substr(was);
if (hit)
word = oss.str();
return hit;
}
bool
Tokenizer::escape(std::string& text) {
bool mod_p = false;
std::string outs;
const char *pp = text.c_str(); // from pp to pt is uncopied
const char *ep = pp + text.size();
const char *pt = pp;
while (pt < ep) {
if (*pt & 0x80) {
const char *mk = (const char *)g_utf8_find_next_char((const gchar *)pt,(const gchar *)ep);
if (!mk) {
if (mod_p)
outs.append(pp,pt-pp+1);
} else {
if (mod_p)
outs.append(pp,mk-pp);
pt = --mk;
}
pp = ++pt;
continue;
}
const char *sequence_p = 0;
if (*pt < '?') {
if (*pt == '&') {
// check for a pre-existing escape
const char *sc = strchr(pt,';');
if (!sc || sc-pt < 2 || sc-pt > 9) {
sequence_p = ESCAPE_MOSES[3];
}
} else if (*pt == '\'') {
sequence_p = ESCAPE_MOSES[6];
} else if (*pt == '"') {
sequence_p = ESCAPE_MOSES[7];
}
} else if (*pt > ']') {
if (*pt =='|') { // 7c
sequence_p = ESCAPE_MOSES[0];
}
} else if (*pt > 'Z') {
if (*pt == '<') { // 3e
sequence_p = ESCAPE_MOSES[4];
} else if (*pt == '>') { // 3c
sequence_p = ESCAPE_MOSES[5];
} else if (*pt == '[') { // 5b
sequence_p = ESCAPE_MOSES[1];
} else if (*pt == ']') { // 5d
sequence_p = ESCAPE_MOSES[2];
}
}
if (sequence_p) {
if (pt > pp)
outs.append(pp,pt-pp);
outs.append(sequence_p);
mod_p = true;
pp = ++pt;
} else {
++pt;
}
}
if (mod_p) {
if (pp < pt) {
outs.append(pp,pt-pp);
}
text.assign(outs.begin(),outs.end());
}
return mod_p;
}
std::string
Tokenizer::penn_tokenize(const std::string& buf)
{
static const char *comma_refs = "\\1 , \\2";
static const char *isolate_ref = " \\1 ";
static const char *special_refs = "\\1 @\\2@ \\3";
std::string text(buf);
std::string outs;
if (skip_alltags_p)
RE2::GlobalReplace(&text,genl_tags_x,SPC_BYTE);
// directed quote patches
size_t len = text.size();
if (len > 2 && text.substr(0,2) == "``")
text.replace(0,2,"`` ",3);
else if (text[0] == '"')
text.replace(0,1,"`` ",3);
else if (text[0] == '`' || text[0] == '\'')
text.replace(0,1,"` ",2);
static char one_gg[] = "\\1 ``";
RE2::GlobalReplace(&text,x1_v_d,one_gg);
RE2::GlobalReplace(&text,x1_v_gg,one_gg);
RE2::GlobalReplace(&text,x1_v_g,"\\1 ` \\2");
RE2::GlobalReplace(&text,x1_v_q,"\\1 ` ");
// protect ellipsis
for (size_t pos = text.find("..."); pos != std::string::npos; pos = text.find("...",pos+11))
text.replace(pos,3,"MANYELIPSIS",11);
// numeric commas
RE2::GlobalReplace(&text,ndndcomma_x,comma_refs);
RE2::GlobalReplace(&text,pdndcomma_x,comma_refs);
RE2::GlobalReplace(&text,ndpdcomma_x,comma_refs);
// isolable symbols
RE2::GlobalReplace(&text,symbol_x,isolate_ref);
// isolable slash
RE2::GlobalReplace(&text,slash_x,special_refs);
// isolate final period
RE2::GlobalReplace(&text,final_x,"\\1 \\2\\3");
// isolate q.m., e.m.
RE2::GlobalReplace(&text,qx_x,isolate_ref);
// isolate braces
RE2::GlobalReplace(&text,braces_x,isolate_ref);
// convert open/close punctuation
RE2::GlobalReplace(&text,"\\(","-LRB-");
RE2::GlobalReplace(&text,"\\[","-LSB-");
RE2::GlobalReplace(&text,"\\{","-LCB-");
RE2::GlobalReplace(&text,"\\)","-RRB-");
RE2::GlobalReplace(&text,"\\]","-RSB-");
RE2::GlobalReplace(&text,"\\}","-RCB-");
// isolate double-dash hyphen
RE2::GlobalReplace(&text,"--"," -- ");
// insure leading and trailing space on line, to simplify exprs
// also make sure final . has one space on each side
len = text.size();
while (len > 1 && text[len-1] == ' ') --len;
if (len < text.size())
text.assign(text.substr(0,len));
if (len > 2 && text[len-1] == '.') {
if (text[len-2] != ' ') {
text.assign(text.substr(0,len-1));
text.append(" . ");
} else {
text.assign(text.substr(0,len-1));
text.append(". ");
}
} else {
text.append(SPC_BYTE,1);
}
std::string ntext(SPC_BYTE);
ntext.append(text);
// convert double quote to paired single-quotes
RE2::GlobalReplace(&ntext,"\""," '' ");
// deal with contractions in penn style
RE2::GlobalReplace(&ntext,endq_x,"\\1 ' ");
RE2::GlobalReplace(&ntext,contract_x," '\\1 ");
RE2::GlobalReplace(&ntext,"'ll "," 'll ");
RE2::GlobalReplace(&ntext,"'re "," 're ");
RE2::GlobalReplace(&ntext,"'ve "," 've ");
RE2::GlobalReplace(&ntext,"n't "," n't ");
RE2::GlobalReplace(&ntext,"'LL "," 'LL ");
RE2::GlobalReplace(&ntext,"'RE "," 'RE ");
RE2::GlobalReplace(&ntext,"'VE "," 'VE ");
RE2::GlobalReplace(&ntext,"N'T "," N'T ");
RE2::GlobalReplace(&ntext," ([Cc])annot "," \\1an not ");
RE2::GlobalReplace(&ntext," ([Dd])'ye "," \\1' ye ");
RE2::GlobalReplace(&ntext," ([Gg])imme "," \\1im me ");
RE2::GlobalReplace(&ntext," ([Gg])onna "," \\1on na ");
RE2::GlobalReplace(&ntext," ([Gg])otta "," \\1ot ta ");
RE2::GlobalReplace(&ntext," ([Ll])emme "," \\1em me ");
RE2::GlobalReplace(&ntext," ([Mm])ore'n "," \\1ore 'n ");
RE2::GlobalReplace(&ntext," '([Tt])is "," '\\1 is 'n ");
RE2::GlobalReplace(&ntext," '([Tt])was "," '\\1 was 'n ");
RE2::GlobalReplace(&ntext," '([Tt])were "," '\\1 were 'n ");
RE2::GlobalReplace(&ntext," ([Ww])anna "," \\1an na ");
protected_tokenize(ntext);
// restore ellipsis
RE2::GlobalReplace(&ntext,"MANYELIPSIS","...");
// collapse spaces
RE2::GlobalReplace(&ntext,mult_spc_x,SPC_BYTE);
// escape moses meta-characters
if (escape_p)
escape(ntext);
// strip out wrapping spaces from line in result string
outs.assign(ntext.substr(1,ntext.size()-2));
return outs;
}
std::string
Tokenizer::quik_tokenize(const std::string& buf)
{
std::string text(buf);
size_t pos;
int num = 0;
// this is the main moses-compatible tokenizer
// push all the prefixes matching protected patterns
std::vector<std::string> prot_stack;
std::string match;
for (auto& pat : prot_pat_vec) {
pos = 0;
while (RE2::PartialMatch(text.substr(pos),*pat,&match)) {
pos = text.find(match,pos);
if (pos == std::string::npos)
break;
size_t len = match.size();
if (text[pos-1] == ' ' || text[pos-1] == '\'' || text[pos-1] == '`'|| text[pos-1] == '"') {
char subst[32];
int nsubst = snprintf(subst,sizeof(subst)," THISISPROTECTED%.3d ",num++);
text.replace(pos,len,subst,nsubst);
prot_stack.push_back(match);
pos += nsubst;
} else {
pos += len;
}
}
}
const char *pt(text.c_str());
const char *ep(pt + text.size());
while (pt < ep && *pt >= 0 && *pt <= ' ')
++pt;
glong ulen(0);
gunichar *usrc(g_utf8_to_ucs4_fast((const gchar *)pt,ep - pt, &ulen)); // g_free
gunichar *ucs4(usrc);
gunichar *lim4(ucs4 + ulen);
gunichar *nxt4 = ucs4;
gunichar *ubuf(g_new0(gunichar,ulen*6+1)); // g_free
gunichar *uptr(ubuf);
gunichar prev_uch(0);
gunichar next_uch(*ucs4);
gunichar curr_uch(0);
GUnicodeType curr_type(G_UNICODE_UNASSIGNED);
GUnicodeType next_type((ucs4 && *ucs4) ? g_unichar_type(*ucs4) : G_UNICODE_UNASSIGNED);
GUnicodeType prev_type(G_UNICODE_UNASSIGNED);
bool post_break_p = false;
bool in_num_p = next_uch <= gunichar(L'9') && next_uch >= gunichar(L'0');
bool in_url_p = false;
int since_start = 0;
int alpha_prefix = 0;
int bad_length = 0;
while (ucs4 < lim4) {
prev_uch = curr_uch;
prev_type = curr_type;
curr_uch = next_uch;
curr_type = next_type;
if (++nxt4 >= lim4) {
next_uch = 0;
next_type = G_UNICODE_UNASSIGNED;
} else {
next_uch = *nxt4;
next_type = g_unichar_type(next_uch);
}
if (url_p) {
if (!in_url_p && *ucs4 < 0x80L) { // url chars must be in the basic plane
if (!since_start) {
if (std::isalpha(char(*ucs4)))
alpha_prefix++;
} else if (alpha_prefix == since_start
&& char(*ucs4) == ':'
&& next_type != G_UNICODE_SPACE_SEPARATOR) {
in_url_p = true;
}
}
}
bool pre_break_p = false;
const wchar_t *substitute_p = 0;
if (post_break_p) {
*uptr++ = gunichar(L' ');
since_start = bad_length = 0;
in_url_p = in_num_p = post_break_p = false;
}
retry:
switch (curr_type) {
case G_UNICODE_MODIFIER_LETTER:
case G_UNICODE_OTHER_LETTER:
case G_UNICODE_TITLECASE_LETTER:
if (in_url_p || in_num_p)
pre_break_p = true;
// fallthough
case G_UNICODE_UPPERCASE_LETTER:
case G_UNICODE_LOWERCASE_LETTER:
if (downcase_p && curr_type == G_UNICODE_UPPERCASE_LETTER)
curr_uch = g_unichar_tolower(*ucs4);
break;
case G_UNICODE_SPACING_MARK:
pre_break_p = true;
in_num_p = false;
curr_uch = 0;
break;
case G_UNICODE_DECIMAL_NUMBER:
case G_UNICODE_LETTER_NUMBER:
case G_UNICODE_OTHER_NUMBER:
if (!in_num_p && !in_url_p) {
switch (prev_type) {
case G_UNICODE_DASH_PUNCTUATION:
case G_UNICODE_FORMAT:
case G_UNICODE_OTHER_PUNCTUATION:
case G_UNICODE_UPPERCASE_LETTER:
case G_UNICODE_LOWERCASE_LETTER:
case G_UNICODE_DECIMAL_NUMBER:
break;
default:
pre_break_p = true;
}
}
in_num_p = true;
break;
case G_UNICODE_CONNECT_PUNCTUATION:
if (curr_uch != gunichar(L'_')) {
if (in_url_p) {
in_url_p = false;
post_break_p = pre_break_p = true;
}
}
if (in_num_p) {
post_break_p = pre_break_p = true;
} else {
switch (next_type) {
case G_UNICODE_LOWERCASE_LETTER:
case G_UNICODE_MODIFIER_LETTER:
case G_UNICODE_OTHER_LETTER:
case G_UNICODE_TITLECASE_LETTER:
break;
default:
post_break_p = pre_break_p = true;
}
switch (prev_type) {
case G_UNICODE_LOWERCASE_LETTER:
case G_UNICODE_MODIFIER_LETTER:
case G_UNICODE_OTHER_LETTER:
case G_UNICODE_TITLECASE_LETTER:
break;
default:
post_break_p = pre_break_p = true;
}
}
break;
case G_UNICODE_FORMAT:
in_url_p = in_num_p = false;
break;
case G_UNICODE_DASH_PUNCTUATION:
if (aggressive_hyphen_p && !in_url_p && curr_uch != next_uch && prev_uch != curr_uch && (!(prev_uch == L' ' || !prev_uch) && !(next_uch == L' ' || !next_uch))) {
substitute_p = L"@-@";
post_break_p = pre_break_p = true;
} else if ( ( curr_uch > gunichar(L'\u002D') && curr_uch < gunichar(L'\u2010') ) ||
( curr_uch > gunichar(L'\u2011')
&& curr_uch != gunichar(L'\u30A0')
&& curr_uch < gunichar(L'\uFE63') ) ) {
// dash, not a hyphen
post_break_p = pre_break_p = true;
} else if (next_type == G_UNICODE_SPACE_SEPARATOR) {
} else {
if (prev_type == curr_type) {
if (next_type != curr_type) {
post_break_p = !in_url_p;
}
} else if (next_type == curr_type) {
pre_break_p = !in_url_p;
} else if ((prev_type == G_UNICODE_UPPERCASE_LETTER ||
prev_type == G_UNICODE_LOWERCASE_LETTER) &&
next_type == G_UNICODE_DECIMAL_NUMBER) {
in_num_p = false;
} else if (in_num_p || since_start == 0) {
switch (next_type) {
case G_UNICODE_UPPERCASE_LETTER:
case G_UNICODE_LOWERCASE_LETTER:
case G_UNICODE_MODIFIER_LETTER:
case G_UNICODE_OTHER_LETTER:
case G_UNICODE_TITLECASE_LETTER:
case G_UNICODE_SPACE_SEPARATOR:
in_num_p = false;
break;
case G_UNICODE_DECIMAL_NUMBER:
case G_UNICODE_LETTER_NUMBER:
case G_UNICODE_OTHER_NUMBER:
case G_UNICODE_OTHER_PUNCTUATION:
break;
default:
post_break_p = true;
pre_break_p = prev_uch != curr_uch;
}
} else if (in_url_p) {
pre_break_p = curr_uch != gunichar(L'-');
} else {
switch (prev_type) {
case G_UNICODE_UPPERCASE_LETTER:
case G_UNICODE_LOWERCASE_LETTER:
case G_UNICODE_MODIFIER_LETTER:
case G_UNICODE_OTHER_LETTER:
case G_UNICODE_TITLECASE_LETTER:
case G_UNICODE_DECIMAL_NUMBER:
case G_UNICODE_LETTER_NUMBER:
case G_UNICODE_OTHER_NUMBER:
case G_UNICODE_OTHER_PUNCTUATION:
switch (next_type) {
case G_UNICODE_UPPERCASE_LETTER:
case G_UNICODE_LOWERCASE_LETTER:
case G_UNICODE_MODIFIER_LETTER:
case G_UNICODE_OTHER_LETTER:
case G_UNICODE_TITLECASE_LETTER:
case G_UNICODE_DECIMAL_NUMBER:
case G_UNICODE_LETTER_NUMBER:
case G_UNICODE_OTHER_NUMBER:
break;
case G_UNICODE_OTHER_PUNCTUATION:
if (prev_type != next_type)
break;
default:
post_break_p = pre_break_p = prev_uch != curr_uch;
}
break;
default:
post_break_p = pre_break_p = prev_uch != curr_uch;
break;
}
}
}
break;
case G_UNICODE_OTHER_PUNCTUATION:
switch (curr_uch) {
case gunichar(L':'):
case gunichar(L'/'):
if (refined_p && !in_url_p
&& prev_type == G_UNICODE_DECIMAL_NUMBER
&& next_type == G_UNICODE_DECIMAL_NUMBER) {
break;
}
// fall-through
case gunichar(L'!'):
case gunichar(L'#'):
case gunichar(L';'):
case gunichar(L'?'):
case gunichar(L'@'):
post_break_p = pre_break_p = !in_url_p || next_type != G_UNICODE_SPACE_SEPARATOR;
break;
case gunichar(L'+'):
post_break_p = pre_break_p = !in_num_p && since_start > 0;
in_num_p = in_num_p || since_start == 0;
break;
case gunichar(L'&'):
if (unescape_p) {
if (next_type == G_UNICODE_LOWERCASE_LETTER || next_type == G_UNICODE_UPPERCASE_LETTER
|| next_type == G_UNICODE_DECIMAL_NUMBER || next_uch == gunichar(L'#')) {
gunichar *eptr = nxt4;
GUnicodeType eptr_type(G_UNICODE_UNASSIGNED);
for (++eptr; eptr < lim4 && *eptr != gunichar(L';'); ++eptr) {
eptr_type = g_unichar_type(*eptr);
if (eptr_type != G_UNICODE_LOWERCASE_LETTER
&& eptr_type != G_UNICODE_UPPERCASE_LETTER
&& eptr_type != G_UNICODE_DECIMAL_NUMBER)
break;
}
gunichar ech(0);
if (*eptr == gunichar(L';') && (ech = get_entity(ucs4,eptr-ucs4+1))) {
curr_uch = ech;
curr_type = g_unichar_type(ech);
ucs4 = eptr;
nxt4 = ++eptr;
next_uch = *nxt4;
next_type = nxt4 < lim4 ? g_unichar_type(next_uch) : G_UNICODE_UNASSIGNED;
goto retry;
}
}
}
if (entities_p && !in_url_p) {
gunichar *cur4 = nxt4;
if (*cur4 == gunichar('#')) ++cur4;
while (g_unichar_isalnum(*cur4)) ++cur4;
if (cur4 > nxt4 && *cur4 == gunichar(';')) {
if (since_start) {
*uptr++ = gunichar(L' ');
since_start = 0;
}
++cur4;
memcpy(uptr,ucs4,cur4-ucs4);
uptr += cur4-ucs4;
ucs4 = cur4;
*uptr++ = gunichar(L' ');
pre_break_p = post_break_p = false;
curr_uch = *ucs4;
curr_type = ucs4 < lim4 ? g_unichar_type(curr_uch) : G_UNICODE_UNASSIGNED;
nxt4 = ++cur4;
next_uch = *nxt4;
next_type = nxt4 < lim4 ? g_unichar_type(next_uch) : G_UNICODE_UNASSIGNED;
goto retry;
}
}
post_break_p = pre_break_p = !in_url_p || next_type != G_UNICODE_SPACE_SEPARATOR;
if (escape_p)
substitute_p = L"&amp;";
break;
case gunichar(L'\''):
if (english_p) {
if (!in_url_p) {
bool next_letter_p = next_type == G_UNICODE_LOWERCASE_LETTER
|| next_type == G_UNICODE_UPPERCASE_LETTER;
pre_break_p = true;
if (next_letter_p && refined_p) {
// break sha n't instead of shan 't:
if (prev_uch == gunichar(L'n') || prev_uch == gunichar(L'N')) {
*(uptr - 1) = gunichar(L' ');
*(uptr++) = prev_uch;
pre_break_p = false;
}
}
post_break_p = since_start == 0
|| (!next_letter_p && next_type != G_UNICODE_DECIMAL_NUMBER);
}
} else if (latin_p) {
post_break_p = !in_url_p;
pre_break_p = !in_url_p && prev_type != G_UNICODE_LOWERCASE_LETTER && prev_type != G_UNICODE_UPPERCASE_LETTER;
} else {
post_break_p = pre_break_p = !in_url_p;
}
if (escape_p)
substitute_p = L"&apos;";
break;
case gunichar(L'"'):
post_break_p = pre_break_p = true;
if (escape_p)
substitute_p = L"&quot;";
break;
case gunichar(L','):
pre_break_p = !in_num_p || next_type != G_UNICODE_DECIMAL_NUMBER;
post_break_p = !in_num_p && next_type != G_UNICODE_DECIMAL_NUMBER;
break;
case gunichar(L'%'):
if (refined_p) {
pre_break_p = !in_num_p;
post_break_p = !in_num_p && next_type != G_UNICODE_DECIMAL_NUMBER;
} else {
post_break_p = pre_break_p = true;
}
break;
case gunichar(L'.'):
if (prev_uch != '.') {
if (!in_num_p) {
switch (next_type) {
case G_UNICODE_DECIMAL_NUMBER:
case G_UNICODE_LOWERCASE_LETTER:
case G_UNICODE_UPPERCASE_LETTER:
break;
default:
if (since_start > 0) {
switch (prev_type) {
case G_UNICODE_LOWERCASE_LETTER:
case G_UNICODE_UPPERCASE_LETTER: {
std::wstring k((wchar_t *)(uptr-since_start),since_start);
if (nbpre_gen_ucs4.find(k) != nbpre_gen_ucs4.end()) {
// general non-breaking prefix
} else if (nbpre_num_ucs4.find(k) != nbpre_num_ucs4.end() && class_follows_p(nxt4,lim4,G_UNICODE_DECIMAL_NUMBER)) {
// non-breaking before numeric
} else if (k.find(curr_uch) != std::wstring::npos) {
if (since_start > 1) {
GUnicodeType tclass = g_unichar_type(*(uptr-2));
switch (tclass) {
case G_UNICODE_UPPERCASE_LETTER:
case G_UNICODE_LOWERCASE_LETTER:
pre_break_p = true;
break;
default:
break;
}
}
// terminal isolated letter does not break
} else if (class_follows_p(nxt4,lim4,G_UNICODE_LOWERCASE_LETTER) ||
g_unichar_type(*nxt4) == G_UNICODE_DASH_PUNCTUATION) {
// lower-case look-ahead does not break
} else {
pre_break_p = true;
}
break;
}
default:
pre_break_p = true;
break;
}
}
break;
}
} else {
switch (next_type) {
case G_UNICODE_DECIMAL_NUMBER:
case G_UNICODE_LOWERCASE_LETTER:
case G_UNICODE_UPPERCASE_LETTER:
break;
default:
pre_break_p = true;
}
}
} else if (next_uch != '.') {
post_break_p = true;
}
break;
default:
post_break_p = pre_break_p = true;
break;
}
break;
case G_UNICODE_CLOSE_PUNCTUATION:
case G_UNICODE_FINAL_PUNCTUATION:
case G_UNICODE_INITIAL_PUNCTUATION:
case G_UNICODE_OPEN_PUNCTUATION:
switch (curr_uch) {
case gunichar(L'('):
case gunichar(L')'):
break;
case gunichar(L'['):
if (escape_p)
substitute_p = L"&#91;";
break;
case gunichar(L']'):
if (escape_p)
substitute_p = L"&#93;";
break;
default:
in_url_p = false;
}
post_break_p = pre_break_p = !in_url_p;
break;
case G_UNICODE_CURRENCY_SYMBOL:
if (refined_p) {
post_break_p = in_num_p; // was in number, so break it
pre_break_p = !in_num_p;
in_num_p = in_num_p || next_type == G_UNICODE_DECIMAL_NUMBER || next_uch == gunichar(L'.') || next_uch == gunichar(L',');
} else {
post_break_p = pre_break_p = true;
in_num_p = false;
}
if (curr_uch != gunichar(L'$'))
in_url_p = false;
break;
case G_UNICODE_MODIFIER_SYMBOL:
case G_UNICODE_MATH_SYMBOL:
switch (curr_uch) {
case gunichar(L'`'):
if (english_p) {
if (!in_url_p) {
pre_break_p = true;
post_break_p = since_start == 0 ||
(next_type != G_UNICODE_LOWERCASE_LETTER && next_type != G_UNICODE_UPPERCASE_LETTER && next_type != G_UNICODE_DECIMAL_NUMBER);
}
} else if (latin_p) {
post_break_p = !in_url_p;
pre_break_p = !in_url_p && prev_type != G_UNICODE_LOWERCASE_LETTER && prev_type != G_UNICODE_UPPERCASE_LETTER;
} else {
post_break_p = pre_break_p = !in_url_p;
}
if (escape_p)
substitute_p = L"&apos;";
else
curr_uch = gunichar(L'\'');
break;
case gunichar(L'|'):
if (escape_p)
substitute_p = L"&#124;";
post_break_p = pre_break_p = true;
break;
case gunichar(L'<'):
if (escape_p)
substitute_p = L"&lt;";
post_break_p = pre_break_p = true;
break;
case gunichar(L'>'):
if (escape_p)
substitute_p = L"&gt;";
post_break_p = pre_break_p = true;
break;
case gunichar(L'%'):
post_break_p = in_num_p;
pre_break_p = !in_num_p && !in_url_p;
in_num_p = false;
break;
case gunichar(L'='):
case gunichar(L'~'):
in_num_p = false;
post_break_p = pre_break_p = !in_url_p;
break;
case gunichar(L'+'):
post_break_p = pre_break_p = !in_url_p;
if (in_url_p) {
in_num_p = false;
} else if (refined_p) {
// handle floating point as e.g. 1.2e+3.4
bool next_digit_p = next_type == G_UNICODE_DECIMAL_NUMBER ||
next_uch == gunichar(L'.');
pre_break_p = !in_num_p;
in_num_p = next_digit_p && prev_type != G_UNICODE_DECIMAL_NUMBER;
post_break_p = !in_num_p;
} else {
in_num_p = in_num_p || since_start == 0;
}
break;
default:
post_break_p = pre_break_p = true;
break;
}
break;
case G_UNICODE_OTHER_SYMBOL:
post_break_p = pre_break_p = true;
break;
case G_UNICODE_CONTROL:
if (drop_bad_p) {
curr_uch = gunichar(L' ');
} else if (curr_uch < gunichar(L' ')) {
curr_uch = gunichar(L' ');
} else if (curr_uch == gunichar(L'\u0092') &&
(next_type == G_UNICODE_LOWERCASE_LETTER || next_type == G_UNICODE_UPPERCASE_LETTER)) {
// observed corpus corruption case
if (english_p) {
pre_break_p = true;
post_break_p = since_start == 0 ||
(next_type != G_UNICODE_LOWERCASE_LETTER && next_type != G_UNICODE_UPPERCASE_LETTER && next_type != G_UNICODE_DECIMAL_NUMBER);
} else if (latin_p) {
post_break_p = true;
pre_break_p = prev_type != G_UNICODE_LOWERCASE_LETTER && prev_type != G_UNICODE_UPPERCASE_LETTER;
} else {
post_break_p = pre_break_p = true;
}
if (escape_p)
substitute_p = L"&apos;";
else
curr_uch = gunichar(L'\'');
} else {
post_break_p = pre_break_p = true;
}
in_url_p = in_num_p = false;
break;
case G_UNICODE_LINE_SEPARATOR:
case G_UNICODE_SPACE_SEPARATOR:
curr_uch = gunichar(L' ');
in_url_p = in_num_p = false;
break;
case G_UNICODE_ENCLOSING_MARK:
in_url_p = false;
break;
case G_UNICODE_NON_SPACING_MARK:
case G_UNICODE_PRIVATE_USE:
case G_UNICODE_SURROGATE:
in_url_p = in_num_p = false;
break;
case G_UNICODE_UNASSIGNED:
default:
// malformed bytes are dropped (invalid utf8 unicode)
if (drop_bad_p) {
curr_uch = 0;
} else {
pre_break_p = since_start > 0 && bad_length == 0;
curr_type = G_UNICODE_UNASSIGNED;
}
in_url_p = in_num_p = false;
break;
}
if (pre_break_p || curr_uch == gunichar(L' ') || (bad_length && curr_type != G_UNICODE_UNASSIGNED)) {
if (since_start) {
// non-empty token emitted previously, so pre-break must emit token separator
*uptr++ = gunichar(L' ');
since_start = bad_length = 0;
}
if (curr_uch == gunichar(L' '))
// suppress emission below, fall-through to substitute logic
curr_uch = 0;
}
if (substitute_p) {
for (gunichar *sptr = (gunichar *)substitute_p; *sptr; ++sptr) {
*uptr++ = *sptr;
since_start++;
}
in_url_p = in_num_p = false;
} else if (curr_uch) {
*uptr++ = curr_uch;
since_start++;
if (curr_type == G_UNICODE_UNASSIGNED)
bad_length++;
}
ucs4 = nxt4;
}
glong nbytes = 0;
gchar *utf8 = g_ucs4_to_utf8(ubuf,uptr-ubuf,0,&nbytes,0); // g_free
if (utf8[nbytes-1] == ' ')
--nbytes;
text.assign((const char *)utf8,(const char *)(utf8 + nbytes));
g_free(utf8);
g_free(usrc);
g_free(ubuf);
// terminate token at superscript or subscript sequence when followed by lower-case
if (supersub_p)
RE2::GlobalReplace(&text,numscript_x,"\\1\\2 \\3");
// restore prefix-protected strings
num = 0;
for (auto& prot : prot_stack) {
char subst[32];
snprintf(subst,sizeof(subst),"THISISPROTECTED%.3d",num++);
size_t loc = text.find(subst);
while (loc != std::string::npos) {
text.replace(loc,18,prot.data(),prot.size());
loc = text.find(subst,loc+18);
}
}
// escape moses meta-characters
if (escape_p)
escape(text);
return text;
}
std::size_t
Tokenizer::tokenize(std::istream& is, std::ostream& os)
{
std::size_t line_no = 0;
std::size_t perchunk = chunksize ? chunksize : 2000;
std::vector< std::vector< std::string > > lines(nthreads);
std::vector< std::vector< std::string > > results(nthreads);
std::vector< boost::thread > workers(nthreads);
bool done_p = !(is.good() && os.good());
for (std::size_t tranche = 0; !done_p; ++tranche) {
// for loop starting threads for chunks of input
for (std::size_t ithread = 0; ithread < nthreads; ++ithread) {
lines[ithread].resize(perchunk);
std::size_t line_pos = 0;
for ( ; line_pos < perchunk; ++line_pos) {
std::string istr;
std::getline(is,istr);
if (skip_alltags_p) {
RE2::GlobalReplace(&istr,genl_tags_x,SPC_BYTE);
istr = trim(istr);
}
line_no++;
if (istr.empty()) {
if (is.eof()) {
done_p = true;
lines[ithread].resize(line_pos);
results[ithread].resize(line_pos);
break;
}
lines[ithread][line_pos].clear();
} else if (skip_xml_p &&
(RE2::FullMatch(istr,tag_line_x) || RE2::FullMatch(istr,white_line_x))) {
lines[ithread][line_pos].clear();
} else {
lines[ithread][line_pos] =
std::string(SPC_BYTE).append(istr).append(SPC_BYTE);
}
}
if (line_pos) {
workers[ithread] =
boost::thread(VectorTokenizerCallable(this,lines[ithread],results[ithread]));
}
} // end for loop starting threads
for (std::size_t ithread = 0; ithread < nthreads; ++ithread) {
if (!workers[ithread].joinable())
continue;
workers[ithread].join();
std::size_t nres = results[ithread].size();
std::size_t nlin = lines[ithread].size();
if (nlin != nres) {
std::ostringstream emsg;
emsg << "Tranche " << tranche
<< " worker " << ithread << "/" << nthreads
<< " |lines|==" << nlin << " != |results|==" << nres;
throw std::runtime_error(emsg.str());
}
for (std::size_t ires = 0; ires < nres; ++ires)
os << results[ithread][ires] << std::endl;
} // end loop over joined results
if (verbose_p) {
std::cerr << line_no << ' ';
std::cerr.flush();
}
} // end loop over chunks
return line_no;
}
std::string
Tokenizer::detokenize(const std::string& buf)
{
std::vector<std::string> words = split(trim(buf));
std::size_t squotes = 0;
std::size_t dquotes = 0;
std::string prepends("");
std::ostringstream oss;
std::size_t nwords = words.size();
std::size_t iword = 0;
if (unescape_p)
for (auto &word: words)
unescape(word);
for (auto &word: words) {
if (RE2::FullMatch(word,right_x)) {
if (iword)
oss << SPC_BYTE;
oss << word;
prepends.clear();
} else if (RE2::FullMatch(word,left_x)) {
oss << word;
prepends = SPC_BYTE;
} else if (english_p && iword
&& RE2::FullMatch(word,curr_en_x)
&& RE2::FullMatch(words[iword-1],pre_en_x)) {
oss << word;
prepends = SPC_BYTE;
} else if (latin_p && iword < nwords - 2
&& RE2::FullMatch(word,curr_fr_x)
&& RE2::FullMatch(words[iword+1],post_fr_x)) {
oss << prepends << word;
prepends.clear();
} else if (word.size() == 1) {
if ((word.at(0) == '\'' && ((squotes % 2) == 0 )) ||
(word.at(0) == '"' && ((dquotes % 2) == 0))) {
if (english_p && iword
&& word.at(0) == '\''
&& std::tolower(words[iword-1].at(words[iword-1].size()-1)) == 's') {
oss << word;
prepends = SPC_BYTE;
} else {
oss << prepends << word;
prepends.clear();
if (word.at(0) == '\'')
squotes++;
else
dquotes++;
}
} else {
if (std::isalnum(word.at(0)))
oss << prepends;
oss << word;
prepends = SPC_BYTE;
if (word.at(0) == '\'')
squotes++;
else if (word.at(0) == '"')
dquotes++;
}
} else {
oss << prepends << word;
prepends = SPC_BYTE;
}
iword++;
}
std::string text(oss.str());
RE2::GlobalReplace(&text," +",SPC_BYTE);
RE2::GlobalReplace(&text,"\n ","\n");
RE2::GlobalReplace(&text," \n","\n");
return trim(text);
}
std::size_t
Tokenizer::detokenize(std::istream& is, std::ostream& os)
{
size_t line_no = 0;
while (is.good() && os.good()) {
std::string istr;
std::getline(is,istr);
line_no ++;
if (istr.empty())
continue;
if (skip_xml_p && (RE2::FullMatch(istr,tag_line_x) || RE2::FullMatch(istr,white_line_x))) {
os << istr << std::endl;
} else {
os << detokenize(istr) << std::endl;
}
}
return line_no;
}
std::vector<std::string>
Tokenizer::splitter(const std::string &istr, bool *continuation_ptr) {
std::vector<std::string> parts;
glong ncp = 0;
glong ocp = 0;
glong icp = 0;
gunichar *ucs4 = g_utf8_to_ucs4_fast((gchar *)istr.c_str(),istr.size(),&ncp);
if (ncp == 0) {
g_free(ucs4);
return parts;
}
gunichar *uout = (gunichar *)g_malloc0(2*ncp*sizeof(gunichar));
const wchar_t GENL_HYPH = L'\u2010';
const wchar_t IDEO_STOP = L'\u3002';
const wchar_t KANA_MDOT = L'\u30FB';
const wchar_t WAVE_DASH = L'\u301C';
//const wchar_t WAVY_DASH = L'\u3030';
const wchar_t KANA_DHYP = L'\u30A0';
const wchar_t SMAL_HYPH = L'\uFE63';
const wchar_t WIDE_EXCL = L'\uFF01';
const wchar_t WIDE_PCTS = L'\uFF05';
//const wchar_t WIDE_HYPH = L'\uFF0D';
const wchar_t WIDE_STOP = L'\uFF0E';
const wchar_t WIDE_QUES = L'\uFF1F';
const wchar_t INVERT_QM = L'\u00BF';
const wchar_t INVERT_EX = L'\u00A1';
wchar_t currwc = 0;
std::size_t init_word = 0;
std::size_t fini_word = 0;
std::size_t finilen = 0;
std::size_t dotslen = 0;
const std::size_t SEQ_LIM = 6;
charclass_t prev_class = empty;
charclass_t curr_class = empty;
std::vector<charclass_t> seq(SEQ_LIM, empty);
std::vector<std::size_t> pos(SEQ_LIM, 0);
std::size_t seqpos = 0;
GUnicodeType curr_type = G_UNICODE_UNASSIGNED;
//bool prev_word_p = false;
bool curr_word_p = false;
std::vector<std::size_t> breaks;
std::set<std::size_t> suppress;
for (; icp <= ncp; ++icp) {
currwc = wchar_t(ucs4[icp]);
curr_type = g_unichar_type(currwc);
prev_class = curr_class;
//prev_word_p = curr_word_p;
switch (curr_type) {
case G_UNICODE_DECIMAL_NUMBER:
case G_UNICODE_OTHER_NUMBER:
curr_class = numba;
curr_word_p = true;
break;
case G_UNICODE_LOWERCASE_LETTER:
case G_UNICODE_MODIFIER_LETTER:
case G_UNICODE_OTHER_LETTER:
curr_class = letta;
curr_word_p = true;
break;
case G_UNICODE_UPPERCASE_LETTER:
case G_UNICODE_TITLECASE_LETTER:
curr_class = upper;
curr_word_p = true;
break;
case G_UNICODE_OPEN_PUNCTUATION:
case G_UNICODE_INITIAL_PUNCTUATION:
curr_class = pinit;
curr_word_p = false;
break;
case G_UNICODE_DASH_PUNCTUATION:
curr_class = hyphn;
if (currwc <= GENL_HYPH) {
curr_word_p = true;
} else if (currwc >= SMAL_HYPH) {
curr_word_p = true;
} else {
curr_word_p = (currwc >= WAVE_DASH) && (currwc <= KANA_DHYP);
}
break;
case G_UNICODE_CLOSE_PUNCTUATION:
case G_UNICODE_FINAL_PUNCTUATION:
curr_class = pfini;
curr_word_p = false;
break;
case G_UNICODE_OTHER_PUNCTUATION:
if (currwc == L'\'' || currwc == L'"') {
curr_class = quote;
curr_word_p = false;
} else if (currwc == L'.' || currwc == IDEO_STOP || currwc == WIDE_STOP || currwc == KANA_MDOT) {
curr_class = stops;
curr_word_p = true;
} else if (currwc == L'?' || currwc == '!' || currwc == WIDE_EXCL || currwc == WIDE_QUES) {
curr_class = marks;
curr_word_p = false;
} else if (currwc == INVERT_QM || currwc == INVERT_EX) {
curr_class = pinit;
curr_word_p = false;
} else if ( currwc == L'%' || currwc == WIDE_PCTS) {
curr_class = pfpct;
curr_word_p = true;
} else {
curr_class = empty;
curr_word_p = false;
}
break;
default:
if (!g_unichar_isgraph(currwc)) {
curr_class = blank;
} else {
curr_class = empty;
}
curr_word_p = false;
break;
}
// # condition for prefix test
// $words[$i] =~ /([\p{IsAlnum}\.\-]*)([\'\"\)\]\%\p{IsPf}]*)(\.+)$/
// $words[$i+1] =~ /^([ ]*[\'\"\(\[\¿\¡\p{IsPi}]*[ ]*[\p{IsUpper}0-9])/
bool check_abbr_p = false;
if (curr_class == stops) {
if (prev_class != stops) {
dotslen = 1;
} else {
dotslen++;
}
} else if (curr_word_p) {
if (!fini_word) {
init_word = ocp;
}
fini_word = ocp+1;
dotslen = finilen = 0;
} else if (curr_class >= quote && curr_class <= pfpct && curr_class != pinit) {
finilen++;
dotslen = 0;
init_word = fini_word = 0;
} else if (dotslen) {
if (fini_word > init_word) {
if (prev_class!=stops || seqpos<1 || (ocp-pos[seqpos-1])<dotslen)
check_abbr_p = false;
else
check_abbr_p = dotslen < 2;
}
dotslen = 0;
} else {
init_word = fini_word = 0;
}
if (check_abbr_p) {
// not a valid word character or post-word punctuation character: check word
std::wstring k((wchar_t *)uout+init_word,fini_word-init_word);
if (finilen == 0 && nbpre_gen_ucs4.find(k) != nbpre_gen_ucs4.end()) {
suppress.insert(std::size_t(ocp));
seqpos = 0;
} else {
bool acro_p = false;
bool found_upper_p = false;
for (glong ii = init_word; ii < ocp; ++ii) {
if (uout[ii] == L'.') {
acro_p = true;
} else if (acro_p) {
if (uout[ii] != L'.' && uout[ii] != L'-') {
GUnicodeType i_type = g_unichar_type(uout[ii]);
if (i_type != G_UNICODE_UPPERCASE_LETTER) {
acro_p = false;
} else {
found_upper_p = true;
}
}
}
}
if (acro_p && found_upper_p) {
suppress.insert(std::size_t(ocp));
seqpos = 0;
} else {
// check forward:
// $words[$i+1] =~ /^([ ]*[\'\"\(\[\¿\¡\p{IsPi}]*[ ]*[\p{IsUpper}0-9])/
int fcp = icp;
int state = (curr_class == pinit || curr_class == quote) ? 1 : 0;
bool num_p = true;
while (fcp < ncp) {
GUnicodeType f_type = g_unichar_type(ucs4[fcp]);
bool f_white = g_unichar_isgraph(ucs4[fcp]);
switch (state) {
case 0:
if (!f_white) {
++fcp;
continue;
} else if (f_type == G_UNICODE_INITIAL_PUNCTUATION || f_type == G_UNICODE_OPEN_PUNCTUATION ||
ucs4[fcp] == L'"'|| ucs4[fcp] == '\'' || ucs4[fcp] == INVERT_QM || ucs4[fcp] == INVERT_EX) {
num_p = false;
state = 1;
++fcp;
continue;
} else if (f_type == G_UNICODE_UPPERCASE_LETTER || f_type == G_UNICODE_DECIMAL_NUMBER) {
if (num_p)
num_p = f_type == G_UNICODE_DECIMAL_NUMBER;
state = 3;
++fcp;
}
break;
case 1:
if (!f_white) {
++fcp;
state = 2;
continue;
} else if (f_type == G_UNICODE_INITIAL_PUNCTUATION || f_type == G_UNICODE_OPEN_PUNCTUATION ||
ucs4[fcp] == L'"'|| ucs4[fcp] == '\'' || ucs4[fcp] == INVERT_QM || ucs4[fcp] == INVERT_EX) {
++fcp;
continue;
} else if (f_type == G_UNICODE_UPPERCASE_LETTER || f_type == G_UNICODE_DECIMAL_NUMBER) {
if (num_p)
num_p = f_type == G_UNICODE_DECIMAL_NUMBER;
state = 3;
++fcp;
}
break;
case 2:
if (!f_white) {
++fcp;
continue;
} else if (f_type == G_UNICODE_UPPERCASE_LETTER || f_type == G_UNICODE_DECIMAL_NUMBER) {
if (num_p)
num_p = f_type == G_UNICODE_DECIMAL_NUMBER;
state = 3;
++fcp;
break;
}
break;
}
break;
}
if (num_p && state == 3 && nbpre_num_ucs4.find(k) != nbpre_num_ucs4.end()) {
suppress.insert(std::size_t(ocp));
seqpos = 0;
}
}
}
init_word = fini_word = 0;
}
if (seqpos >= SEQ_LIM) {
seqpos = 0;
}
if (curr_class == stops || curr_class == marks) {
if (!seqpos) {
seq[seqpos] = curr_class;
pos[seqpos] = ocp;
seqpos++;
uout[ocp++] = gunichar(currwc);
continue;
} else if (seqpos>1 && (seq[seqpos-1]==blank || seq[seqpos-1]==quote || seq[seqpos-1]==pfini)) {
// handle "[?!.] ..." which is common in some corpora
if (seq[seqpos-2] == curr_class || seq[seqpos-2] == marks) {
seqpos--;
uout[ocp++] = gunichar(currwc);
continue;
}
seqpos = 0;
} else if (seq[seqpos-1] != curr_class) {
seqpos = 0;
} else if (curr_class == marks) {
seqpos = 0;
} else {
uout[ocp++] = gunichar(currwc);
continue;
}
}
if (!seqpos) {
if (curr_class != blank) {
uout[ocp++] = gunichar(currwc);
} else if (curr_class != prev_class) {
uout[ocp++] = L' ';
}
continue;
}
if (curr_class == blank) {
if (prev_class != blank) {
seq[seqpos] = blank;
pos[seqpos] = ocp;
seqpos++;
uout[ocp++] = L' ';
}
if (icp < ncp)
continue;
}
if (curr_class >= quote && curr_class <= pfini) {
if (prev_class < quote || prev_class > pfini) {
seq[seqpos] = curr_class;
pos[seqpos] = ocp;
seqpos++;
} else if (curr_class == quote && prev_class != curr_class) {
curr_class = prev_class;
} else if (prev_class == quote) {
seq[seqpos] = prev_class = curr_class;
}
uout[ocp++] = gunichar(currwc);
continue;
}
// $text =~ s/([?!]) +([\'\"\(\[\¿\¡\p{IsPi}]*[\p{IsUpper}])/$1\n$2/g;
// #multi-dots followed by sentence starters 2
// $text =~ s/(\.[\.]+) +([\'\"\(\[\¿\¡\p{IsPi}]*[\p{IsUpper}])/$1\n$2/g;
// # add breaks for sentences that end with some sort of punctuation inside a quote or parenthetical and are followed by a possible sentence starter punctuation and upper case 4
// $text =~ s/([?!\.][\ ]*[\'\"\)\]\p{IsPf}]+) +([\'\"\(\[\¿\¡\p{IsPi}]*[\ ]*[\p{IsUpper}])/$1\n$2/g;
// # add breaks for sentences that end with some sort of punctuation are followed by a sentence starter punctuation and upper case 8
// $text =~ s/([?!\.]) +([\'\"\(\[\¿\¡\p{IsPi}]+[\ ]*[\p{IsUpper}])/$1\n$2/g;
std::size_t iblank = 0;
if (curr_class == upper || icp == ncp) {
if (seqpos && (seq[0] == stops || seq[0] == marks)) {
switch (seqpos) {
case 2:
if (seq[1] == blank)
iblank = 1;
break;
case 3:
switch (seq[1]) {
case blank:
if (seq[2] == quote || seq[2] == pinit)
iblank = 1;
break;
case quote:
case pfini:
if (seq[2] == blank)
iblank = 2;
break;
default:
break;
}
break;
case 4:
switch (seq[1]) {
case blank:
iblank = 1;
switch (seq[2]) {
case quote:
switch (seq[3]) {
case quote:
case pinit:
break;
case blank:
iblank = 3;
break;
default:
iblank = 0; // invalid
break;
}
break;
case pinit:
if (seq[3] != blank)
iblank = 0; // invalid
break;
case pfini:
if (seq[3] == blank)
iblank = 3;
break;
default:
iblank = 0; // invalid
break;
}
break;
case quote:
case pfini:
iblank = (seq[2] == blank && (seq[3] == quote || seq[3] == pinit)) ? 2 : 0;
break;
default:
iblank = 0; // invalid
break;
}
break;
case 5:
iblank = (seq[1] == blank) ? 2 : 1;
if (seq[iblank] == quote || seq[iblank] == pfini)
iblank++;
if (seq[iblank] != blank) {
iblank = 0; // invalid
} else {
if (seq[iblank+1] != quote && seq[iblank+1] != pinit) {
iblank = 0; // invalid
} else if (iblank+2 < seqpos) {
if (seq[iblank+2] != blank)
iblank = 0; // invalid
}
}
break;
}
}
if (iblank && suppress.find(pos[iblank]) == suppress.end()) {
breaks.push_back(pos[iblank]);
suppress.insert(pos[iblank]);
}
}
uout[ocp++] = gunichar(currwc);
seqpos = 0;
}
std::vector<std::size_t>::iterator it = breaks.begin();
glong iop = 0;
while (iop < ocp) {
glong endpos = it == breaks.end() ? ocp : *it++;
glong nextpos = endpos + 1;
while (endpos > iop) {
std::size_t chkpos = endpos-1;
if (uout[chkpos] == L'\n' || uout[chkpos] == L' ') {
endpos = chkpos;
continue;
}
if (g_unichar_isgraph(uout[chkpos]))
break;
endpos = chkpos;
}
if (endpos > iop) {
gchar *pre = g_ucs4_to_utf8(uout+iop,endpos-iop,0,0,0);
parts.push_back(std::string(pre));
g_free(pre);
}
if (continuation_ptr)
*continuation_ptr = endpos > iop;
iop = nextpos;
}
g_free(uout);
g_free(ucs4);
return parts;
}
std::pair<std::size_t,std::size_t>
Tokenizer::splitter(std::istream& is, std::ostream& os)
{
std::pair<std::size_t,std::size_t> counts = { 0, 0 };
bool continuation_p = false;
bool pending_gap = false;
bool paragraph_p = false;
while (is.good() && os.good()) {
std::string istr;
std::getline(is,istr);
counts.first++;
if (istr.empty() && (is.eof() ||!para_marks_p))
continue;
if (skip_xml_p && (RE2::FullMatch(istr,tag_line_x) || RE2::FullMatch(istr,white_line_x)))
continue;
std::vector<std::string> sentences(splitter(istr,&continuation_p));
if (sentences.empty()) {
if (!paragraph_p) {
if (pending_gap)
os << std::endl;
pending_gap = false;
if (para_marks_p)
os << "<P>" << std::endl;
paragraph_p = true;
}
continue;
}
paragraph_p = false;
std::size_t nsents = sentences.size();
counts.second += nsents;
if (pending_gap) {
os << " ";
pending_gap = false;
}
for (std::size_t ii = 0; ii < nsents-1; ++ii)
os << sentences[ii] << std::endl;
os << sentences[nsents-1];
if (continuation_p)
pending_gap = !split_breaks_p;
if (!pending_gap)
os << std::endl;
}
if (pending_gap)
os << std::endl;
return counts;
}
#ifdef TOKENIZER_NAMESPACE
}; // namespace
#endif
Reference in New Issue View Git Blame Copy Permalink