gigablast/open-source-search-engine
1
0
Fork 0
mirror of https://github.com/gigablast/open-source-search-engine.git synced 2024-10-04 12:17:35 +03:00
Code Issues Projects Releases Wiki Activity
d19ee6ceea
open-source-search-engine/Xml.cpp
Matt 4c19453ea9 working with -m32 for basic testing. 
compiles for 64-bit.
2014-11-12 11:38:37 -08:00

1072 lines
31 KiB
C++
Raw Blame History

#include "gb-include.h"
#include "Xml.h"
#include "Mem.h" // mfree(), mmalloc()
#include "Unicode.h" // for html entities that return unicode
#include "Titledb.h"
#include "Words.h"
//#include "DateParse2.h"
Xml::Xml () {
m_xml = NULL;
m_xmlLen = 0;
m_nodes = NULL;
m_numNodes=0;
m_ownData = false;
m_version = TITLEREC_CURRENT_VERSION;
}
// . should free m_xml if m_copy is true
Xml::~Xml () {
reset();
}
// . for parsing xml conf files
bool Xml::getBool ( int32_t n0 , int32_t n1 , char *tagName , bool defaultBool ) {
int32_t len;
char *s = getTextForXmlTag ( n0 , n1 , tagName , &len , true );
if ( s ) return atob ( s , len );
// return the default if no non-white-space text
return defaultBool;
}
// . for parsing xml conf files
int32_t Xml::getLong ( int32_t n0 , int32_t n1 , char *tagName , int32_t defaultLong ) {
int32_t len;
char *s = getTextForXmlTag ( n0 , n1 , tagName , &len , false );
if ( s ) return atol2 ( s , len );
// return the default if no non-white-space text
return defaultLong;
}
// . for parsing xml conf files
int64_t Xml::getLongLong ( int32_t n0 , int32_t n1 , char *tagName ,
int64_t defaultLongLong ) {
int32_t len;
char *s = getTextForXmlTag ( n0 , n1 , tagName , &len , false );
if ( s ) return atoll2 ( s , len );
// return the default if no non-white-space text
return defaultLongLong;
}
// . for parsing xml conf files
float Xml::getFloat (int32_t n0 , int32_t n1 , char *tagName,float defaultFloat){
int32_t len;
char *s = getTextForXmlTag ( n0 , n1 , tagName , &len , false );
if ( s ) return atof2 ( s , len );
// return the default if no non-white-space text
return defaultFloat;
}
char *Xml::getString ( int32_t n0 , int32_t n1 , char *tagName, int32_t *len ,
bool skipLeadingSpaces ) const {
char *s = getTextForXmlTag ( n0, n1, tagName, len, skipLeadingSpaces );
if ( s ) return s;
// return the default if s is null
return NULL;
}
// . used by getValueAsBool/Long/String()
// . tagName is compound for xml tags, simple for html tags
// . NOTE: we skip over leading spaces
char *Xml::getTextForXmlTag ( int32_t n0 , int32_t n1 , char *tagName , int32_t *len ,
bool skipLeadingSpaces ) const {
// assume len is 0
*len = 0;
// get a matching xml TAG
int32_t num = getNodeNum ( n0 , n1 , tagName , gbstrlen(tagName) );
if ( num < 0 ) return NULL;
return getString ( num , skipLeadingSpaces , len );
}
char *Xml::getString ( int32_t num , bool skipLeadingSpaces , int32_t *len ) const {
// get the text of this tag (if any)
if ( ++num >= m_numNodes ) { *len = 0; return NULL; }
if ( ! m_nodes[num].isText() ) { *len = 0; return NULL; }
// if we don't skip leading spaces return it as is
if ( ! skipLeadingSpaces ) {
*len = m_nodes[num].m_nodeLen;
return m_nodes[num].m_node;
}
// get the string
char *s = m_nodes[num].m_node;
// set the length and return the string
int32_t slen = m_nodes[num].m_nodeLen;
// skip leading spaces
while ( is_wspace_utf8 ( s ) && slen > 0 ) { s++; slen--; }
// set len
*len = slen;
// return NULL if slen is 0
if ( slen == 0 ) return NULL;
// otherwise return s
return s;
}
char *Xml::getNode ( char *tagName , int32_t *len ) {
// assume len is 0
*len = 0;
// get a matching xml TAG
int32_t num = getNodeNum ( 0 , m_numNodes, tagName , gbstrlen(tagName) );
if ( num < 0 ) return NULL;
// no back tag if its like <languages/> it won't have one
XmlNode *node = &m_nodes[num];
if ( ! node->m_hasBackTag ) return NULL;
// scan for ending back tag
int32_t i ; for ( i = num + 1 ; i < m_numNodes ; i++ ) {
if ( m_nodes[i].m_hash != node->m_hash ) continue;
break;
}
if ( i >= m_numNodes ) return NULL;
// got the back tag
char *end = m_nodes[i].m_node;
char *s = m_nodes[num+1].m_node;
// trim spaces
while ( s < end && is_wspace_a ( *s ) ) s++;
while ( end-1 > s && is_wspace_a ( end[-1] ) ) end--;
*len = end - s;
return s;
}
int64_t Xml::getCompoundHash ( char *s , int32_t len ) const {
// setup
char *p = s;
char *start = s;
int32_t i = 0;
int64_t h = 0;
loop:
// find fisrt .
while ( i < len && p[i] != '.' ) i++;
// . hash from p to p[i]
// . tag names are always ascii, so use the ascii hasher, not utf8
h = hash64Upper_a ( start , &p[i] - start , h );
// bail if done
if ( i >= len ) return h;
// then period
h = hash64 ( "." , 1 , h );
// skip period
i++;
// start now points to next word
start = &p[i];
// continue
goto loop;
}
// . return -1 if not found
// . "tagName" is compound (i.e. "myhouse.myroom" )
int32_t Xml::getNodeNum ( int32_t n0 , int32_t n1 , char *tagName , int32_t tagNameLen ) const {
// . since i changed the hash to a zobrist hash, hashing
// "dns.ip" is not the same as hashing "dns" then "." then "ip"
// by passing the hash of the last to the next as the startHash
// . therefore, i now parse it up
int64_t h = getCompoundHash ( tagName , tagNameLen );
int32_t i;
if ( n1 > m_numNodes ) n1 = m_numNodes;
if ( n0 > m_numNodes ) n0 = m_numNodes;
if ( n1 < 0 ) n1 = 0;
if ( n0 < 0 ) n0 = 0;
for ( i = n0 ; i < n1; i++ ) {
// if node is text (non-tag) then skip
if ( ! m_nodes[i].isTag() ) continue;
//if ( m_nodes[i].m_compoundHash == h ) break;
if ( m_nodes[i].m_hash == h ) break;
}
// return -1 if not found at all
if ( i >= n1 ) return -1;
return i;
}
void Xml::reset ( ) {
// free old nodes array if any
if ( m_nodes ) mfree ( m_nodes, m_maxNumNodes*sizeof(XmlNode),"Xml1");
if ( m_ownData && m_xml ) mfree ( m_xml, m_allocSize, "Xml1");
m_xml = NULL;
m_nodes = NULL;
m_numNodes = 0;
m_maxNumNodes = 0;
m_allocSize = 0;
}
bool Xml::getCompoundName ( int32_t node , SafeBuf *sb ) {
XmlNode *buf[256];
XmlNode *xn = &m_nodes[node];
int32_t np = 0;
for ( ; xn ; xn = xn->m_parent ) {
if ( ! xn->m_nodeId ) continue;
if ( np >= 256 ) {g_errno = EBUFTOOSMALL;return false;}
buf[np++] = xn;
}
// ignore that initial <?xml ..> tag they all have
if ( np > 0 &&
buf[np-1]->m_tagNameLen == 3 &&
strncasecmp(buf[np-1]->m_tagName,"xml",3) == 0 )
np--;
for ( int32_t i = np - 1 ; i >= 0 ; i-- ) {
XmlNode *xn = buf[i];
sb->safeMemcpy ( xn->m_tagName , xn->m_tagNameLen );
sb->pushChar('.');
}
// remove last '.'
if ( sb->length() ) sb->m_length--;
sb->nullTerm();
return true;
}
#include "HttpMime.h" // CT_JSON
// "s" must be in utf8
bool Xml::set ( char *s ,
int32_t slen ,
bool ownData ,
int32_t allocSize ,
bool pureXml ,
int32_t version ,
bool setParentsArg ,
int32_t niceness ,
char contentType ) {
// just in case
reset();
m_niceness = niceness;
// clear it
g_errno = 0;
// if we own the data we free on reset/destruction
m_ownData = ownData;
m_version = version;
// use explicit allocSize if we passed one
m_allocSize = allocSize?allocSize:slen+1;
// make pointers to data
m_xml = s;
m_xmlLen = slen; //i;
// debug msg time
if ( g_conf.m_logTimingBuild )
logf(LOG_TIMING,
"build: xml: set: 4a. %"UINT64"",gettimeofdayInMilliseconds());
// sanity check
if ( !s || slen <= 0) return true;
if ( s[slen] != '\0' ) {
log(LOG_LOGIC,"build: Xml: Content is not null terminated.");
char *xx = NULL; *xx = 0;
//sleep(100);
g_errno = EBADENGINEER;
return false;
}
// if json go no further. TODO: also do this for CT_TEXT etc.
if ( contentType == CT_JSON ) {
m_numNodes = 0;
// make the array
m_maxNumNodes = 1;
m_nodes =(XmlNode *)mmalloc(sizeof(XmlNode)*m_maxNumNodes,"x");
if ( ! m_nodes ) return false;
XmlNode *xd = &m_nodes[m_numNodes];
// hack the node
xd->m_node = s;
xd->m_nodeLen = slen;
xd->m_isSelfLink = 0;
// . nodeId for text nodes is 0
xd->m_nodeId = 0;
xd->m_hasBackTag = false;
xd->m_hash = 0;
xd->m_pairTagNum = -1;
m_numNodes++;
return true;
}
// override
if ( contentType == CT_XML )
pureXml = true;
QUICKPOLL((niceness));
int32_t i;
// . replacing NULL bytes with spaces in the buffer
// . utf8 should never have any 0 bytes in it either!
for ( i = 0 ; i < slen ; i++ ) if ( !s[i] ) s[i] = ' ';
// counting the max num nodes
for ( i = 0 ; s[i] ; i++ ) if ( s[i] == '<' ) m_maxNumNodes++;
// account for the text (non-tag) nodes (padding nodes between tags)
m_maxNumNodes *= 2 ;
// if we only have one tag we can still have 3 nodes!
m_maxNumNodes++;
// debug msg time
if ( g_conf.m_logTimingBuild )
logf(LOG_TIMING,
"build: xml: set: 4b. %"UINT64"",gettimeofdayInMilliseconds());
// . truncate it to avoid spammers
// . now i limit to 30k nodes because of those damned xls docs!
// . they have 300,000+ nodes some of 'em
// now allow 35k nodes for every 100k doclen
int32_t num100k = slen/(100*1024);
if (num100k <= 0) num100k = 1;
int32_t bigMax = 35*1024 * num100k;
if (m_maxNumNodes > bigMax){
log(LOG_WARN, "build: xml: doclen %"INT32", "
"too many nodes: counted %"INT32", max %"INT32" "
"...truncating", slen, m_maxNumNodes, bigMax);
m_maxNumNodes = bigMax;
}
// breathe
QUICKPOLL ( niceness );
m_nodes = (XmlNode *) mmalloc (sizeof(XmlNode) * m_maxNumNodes,"Xml1");
if ( ! m_nodes ) {
reset();
return log("build: Could not allocate %"INT32" "
"bytes need to parse document.",
(int32_t)sizeof(XmlNode)*m_maxNumNodes);
}
// debug msg time
if ( g_conf.m_logTimingBuild )
logf(LOG_TIMING,
"build: xml: set: 4c. %"UINT64"",gettimeofdayInMilliseconds());
XmlNode *parent = NULL;
XmlNode *parentStackStart[256];
XmlNode **parentStackPtr = &parentStackStart[0];
XmlNode **parentStackEnd = &parentStackStart[256];
// . TODO: do this on demand
// . now fill our nodes array
// . loop over the xml
// . i is byte-index in buffer
int32_t oldi;
for ( i = 0 ; i < m_xmlLen && m_numNodes < m_maxNumNodes ; ) {
// breathe
QUICKPOLL(niceness);
// remember oldi
oldi = i;
// convenience ptr
XmlNode *xi = &m_nodes[m_numNodes];
// set that node
i += xi->set (&m_xml[i],pureXml,version);
// set his parent xml node if is xml
xi->m_parent = parent;
// if not text node then he's the new parent
if ( pureXml &&
xi->m_nodeId &&
xi->m_nodeId != TAG_COMMENT ) {
// if we are a back tag pop the stack
if ( ! xi->isFrontTag() ) {
// pop old parent
if ( parentStackPtr > parentStackStart )
parent = *(--parentStackPtr);
}
// we are a front tag...
else {
// did we overflow?
if ( parentStackPtr >= parentStackEnd ) {
log("xml: xml parent overflow");
g_errno = EBUFTOOSMALL;
return false;
}
// push the old parent ptr
if ( parent ) *parentStackPtr++ = parent;
// set the new parent to us
parent = xi;
}
}
// in script?
if ( xi->m_nodeId != TAG_SCRIPT ) {
m_numNodes++;
continue;
}
if ( ! xi->isFrontTag() ) {
m_numNodes++;
continue;
}
// ok, we got a <script> tag now
m_numNodes++;
// scan for </script>
char *pstart = &m_xml[i];
char *p = pstart;
char *pend = &m_xml[0] + m_xmlLen;
// scan -- 5 continues -- node 1570 is text of script
for ( ; p < pend ; p++ ) {
// breathe
QUICKPOLL(m_niceness);
// keep going if not a tag
if ( p[0] != '<' ) continue;
// </script> or </gbframe> stops it
if ( p[1] == '/' ) {
if ( to_lower_a(p[2]) == 's' &&
to_lower_a(p[3]) == 'c' &&
to_lower_a(p[4]) == 'r' &&
to_lower_a(p[5]) == 'i' &&
to_lower_a(p[6]) == 'p' &&
to_lower_a(p[7]) == 't' )
break;
if ( to_lower_a(p[2]) == 'g' &&
to_lower_a(p[3]) == 'b' &&
to_lower_a(p[4]) == 'f' &&
to_lower_a(p[5]) == 'r' &&
to_lower_a(p[6]) == 'a' &&
to_lower_a(p[7]) == 'm' )
break;
}
// another <script> stops it
if ( to_lower_a(p[1]) == 's' &&
to_lower_a(p[2]) == 'c' &&
to_lower_a(p[3]) == 'r' &&
to_lower_a(p[4]) == 'i' &&
to_lower_a(p[5]) == 'p' &&
to_lower_a(p[6]) == 't' )
break;
}
// make sure we do not breach! i saw this happen once!
if ( m_numNodes >= m_maxNumNodes ) break;
// was it like <script></script> then no scripttext tag?
if ( p - pstart == 0 )
continue;
XmlNode *xn = &m_nodes[m_numNodes++];
xn->m_nodeId = TAG_SCRIPTTEXT;//0; // TEXT NODE
xn->m_node = pstart;
xn->m_nodeLen = p - pstart;
xn->m_tagName = NULL;
xn->m_tagNameLen = 0;
xn->m_hasBackTag = false;
xn->m_hash = 0;
xn->m_isVisible = false;
xn->m_isBreaking = false;
// advance i to get to the </script> or <gbframe> etc.
i = p - &m_xml[0] ;
}
// sanity
if ( m_numNodes > m_maxNumNodes ) { char *xx=NULL;*xx=0; }
// trim off last node if empty! it is causing a core in isBackTag()
if ( m_numNodes > 0 && m_nodes[m_numNodes-1].m_nodeLen == 0 )
m_numNodes--;
// debug msg time
if ( g_conf.m_logTimingBuild )
logf(LOG_TIMING,
"build: xml: set: 4d. %"UINT64"",gettimeofdayInMilliseconds());
return true;
}
// for translating HTML entities to an iso char
#include "Entities.h"
// . replaces line-breaking html tags with 2 returns if "includeTags" is false
// . stores tags too if "includeTags" is true
// . returns # chars written to buf
// . NOTE: see XmlNode.cpp for list of tag types in "NodeType" structure
// . used to get xml subtrees as text
// . used to get <TITLE>'s
// . must write to your buf rather than just return a pointer since we may
// have to concatenate several nodes together, we may have to replace tags,..
// . TODO: nuke this in favor of Pos.cpp::filter() -- but that needs Words.cpp
int32_t Xml::getText ( char *buf ,
int32_t bufMaxSize ,
int32_t node1 ,
int32_t node2 ,
bool includeTags ,
bool visibleTextOnly ,
bool filter , // convert entities, \r's
bool filterSpaces , // filter excessive punct/spaces
bool useStopIndexTag ) { // indexable text only?
// init some vars
int32_t i = node1;
int32_t n = node2;
// truncate n to the # of nodes we have
if ( n > m_numNodes ) n = m_numNodes;
// keep a non visible tag stack
int32_t notVisible = 0;
// are we in indexable area?
bool inStopTag = false;
// the destination
char *dst = buf;
char *dstEnd = buf + bufMaxSize;
char cs = -1;
// cannot allow nested script tags, messed up our summary generator
// when a page tried to print a <SCRIPT> tag in a doWrite('<SCRIPT>')
// already in a <SCRIPT> block
//char inScript = false;
// loop through all nodes from here on until we run outta nodes...
// or until we hit a tag with the same depth as us.
for ( ; i < n ; i++ ) {
// if it's <stop index> continue until start index
if ( useStopIndexTag ) {
// is it a <stop index> tag ?
if ( m_nodes[i].m_nodeId == 115 ) {
inStopTag = true;
continue;
}
// is it a start tag?
if ( m_nodes[i].m_nodeId == 114 ) {
inStopTag = false;
continue;
}
// continue if in a stop section
if ( inStopTag ) continue;
}
// . set skipText to true if this tag has inivisble text
// . examples: <option> <script> ...
if ( m_nodes[i].isTag() && ! m_nodes[i].isVisible() &&
m_nodes[i].hasBackTag() ) {
if ( m_nodes[i].isFrontTag() ) notVisible++;
else notVisible--;
if ( notVisible < 0 ) notVisible = 0;
}
// . if it's a tag then write a \n\n or \n to the buf
// . do this only if we do not include tags
// . do it only if there's something already in the buf
if ( ! includeTags && m_nodes[i].isTag() ) {
// do nothing if buf still empty
if ( dst <= buf ) continue;
// or not a breaking tag
if ( ! m_nodes[i].isBreaking() ) continue;
// forgot this check! leave room for terminating \0
if ( dst + 3 >= dstEnd ) break;
// if we're not junk filtering just add 2 \n's
if ( ! filterSpaces ) {
*dst++='\n';
*dst++='\n';
continue;
}
// need at least 2 chars in the dst buf so far
if ( dst - 1 <= buf ) continue;
if ( cs == -1 ) continue;
// . if prev char is punct, do nothing.
// . check prev prev char to make sure not a single chr
// . TODO: fix this!
if ( is_punct_a( *(dst - cs))) continue;
//if ( is_punct_utf8( dst[-1]) ) continue;
if ( i+1 >= n ) continue;
if ( is_punct_utf8 ( &m_nodes[i+1].m_node[0] )
&& !m_nodes[i+1].isTag() ) continue;
// . watch out for punct before space(s) though
// . it also ensures that this char is the first char
// of any potential multi-byte sequence
if ( is_wspace_utf8 ( dst - cs ) ) {
// back up one before that even
char *f = dst - cs - 1;
// don't do a while loop on this
// cuz with those xls docs we can
// have a TON of spaces cuz their
// just a bunch of <td></td>&nbsp;'s
if ( f > buf && is_wspace_a ( *f ) ) f--;
if ( f > buf && is_wspace_a ( *f ) ) f--;
if ( f > buf && is_wspace_a ( *f ) ) f--;
if ( f > buf && is_ascii(*f)&&is_punct_a(*f) )
continue;
}
// ok, add the ".."
*dst++='.';
*dst++='.';
continue;
}
// if this tag/text is not visible then continue
if ( notVisible ) continue;
// . get a ptr to the node's data
// . is 1 of 3 things: a text blob, xml tag or html tag
char *nodeData = m_nodes[i].getNode ();
int nodeDataLen = m_nodes[i].getNodeLen();
// . truncate the node if it's too big
// . make sure we truncate at a non alphanumeric character
// . avoid breaking in the middle of a word
// . we cannot truncate tags
if ( dst + nodeDataLen >= dstEnd ) { // bufMaxSize ) {
// cannot truncate tags
if ( m_nodes[i].isTag() ) break;
nodeDataLen = dstEnd - dst - 2;//bufMaxSize - blen;
while ( nodeDataLen > 0 &&
! is_wspace_a(nodeData[ nodeDataLen-1 ]))
nodeDataLen--;
}
// if we truncated the whole thing just break out, we're done.
if ( nodeDataLen <= 0 ) break;
// . copy the node data into our buffer
// . translate HTML entities to iso characters
// . translate \r's into spaces
// point to it
char *src = nodeData;
char *srcEnd = nodeData + nodeDataLen;
// size of character in bytes, usually 1
//char cs ;
// copy the node @src into "dst"
for ( ; src < srcEnd ; src += cs , dst += cs ) {
// get the character size in bytes
cs = getUtf8CharSize ( src );
// no back to back spaces if we're filtering junk
if ( filterSpaces && is_wspace_utf8 ( src ) ) {
if ( dst <= buf ) {dst -= cs; continue;}
if ( dst[-1] == ' ' ) {dst -= cs; continue;}
// ok, do not filter it
//goto simplecopy;
}
// a lot of docs have ^M's in them (\r)
//if ( c == '\r' ) { buf [ blen++ ] = ' '; continue; }
// store it as-is if not filtering or not html entity
//simplecopy:
// if more than 1 byte in char, use memcpy
if ( cs > 1 ) memcpy ( dst , src , cs );
else *dst = *src;
}
// continue looping over nodes (text and tag nodes)
}
// . strip trailing spaces
// . is_wspace_utf8 will be false if it is not the first character
// of a utf8 char sequence, and i don't count any multi-byte
// spaces i guess...
while ( dst > buf && is_wspace_a ( dst[-1] ) ) dst--;
// null term it
*dst = '\0';
// return the # of bytes we've written into the buffer.
return dst - buf;
}
// just get a pointer to it
char *Xml::getMetaContentPointer ( char *field ,
int32_t fieldLen ,
char *name ,
int32_t *slen ) {
// find the first meta summary node
for ( int32_t i = 0 ; i < m_numNodes ; i++ ) {
// continue if not a meta tag
if ( m_nodes[i].m_nodeId != 68 ) continue;
// . does it have a type field that's "summary"
// . <meta name=summary content="...">
// . <meta http-equiv="refresh" content="0;URL=http://y.com/">
int32_t len;
char *s = getString ( i , name , &len );
// continue if name doesn't match field
if ( len != fieldLen ) continue;
// field can be "summary","description","keywords",...
if ( strncasecmp ( s , field , fieldLen ) != 0 ) continue;
// point to the summary itself
s = getString ( i , "content" , &len );
if ( ! s || len <= 0 ) continue;
// return the pointer (and set the length of what it points to)
*slen = len;
return s;
}
*slen = 0;
return NULL;
}
// . extract the content from a meta tag
// . null terminate it and store it into "buf"
// . field can be stuff like "summary","description","keywords",...
// . TODO: have a filter option to filter out back-to-back spaces for summary
// generation purposes in Summary class
// . "name" is usually "name" or "http-equiv"
// . if "convertHtmlEntities" is true we turn < into &lt; and > in &gt;
int32_t Xml::getMetaContent (char *buf, int32_t bufLen, char *field, int32_t fieldLen ,
char *name , bool convertHtmlEntities ,
int32_t startNode , int32_t *matchedNode ) {
// return 0 length if no buffer space
if ( bufLen <= 0 ) return 0;
// assume it's empty
buf[0] = '\0';
// assume no tag matched
if ( matchedNode ) *matchedNode = -1;
// store output into "dst"
char *dst = buf;
char *dstEnd = buf + bufLen;
// find the first meta summary node
for ( int32_t i = startNode ; i < m_numNodes ; i++ ) {
// breathe
QUICKPOLL(m_niceness);
// continue if not a meta tag
if ( m_nodes[i].m_nodeId != 68 ) continue;
// . does it have a type field that's "summary"
// . <meta name=summary content="...">
// . <meta http-equiv="refresh" content="0;URL=http://y.com/">
int32_t len;
char *s = getString ( i , name , &len );
// continue if name doesn't match field
// field can be "summary","description","keywords",...
if ( len != fieldLen ) continue;
if ( strncasecmp ( s , field , fieldLen ) != 0 ) continue;
// point to the summary itself
s = getString ( i , "content" , &len );
if ( ! s || len <= 0 ) continue;
// point to it
char *src = s;
char *srcEnd = s + len;
// size of character in bytes, usually 1
char cs ;
// bookmark
char *lastp = NULL;
// copy the node @p into "dst"
for ( ; src < srcEnd ; src+= cs ) {
// breathe
QUICKPOLL(m_niceness);
// get the character size in bytes
cs = getUtf8CharSize ( src );
// break if we are full! (save room for \0)
if ( dst + 5 >= dstEnd ) break;
// remember last punct for cutting purposes
if ( ! is_alnum_utf8 ( src ) ) lastp = dst;
// encode it as an html entity if asked to
if ( *src == '<' && convertHtmlEntities ) {
memcpy ( dst , "&lt;" , 4 );
dst += 4;
continue;
}
// encode it as an html entity if asked to
if ( *src == '>' && convertHtmlEntities ) {
memcpy ( dst , "&gt;" , 4 );
dst += 4;
continue;
}
// if more than 1 byte in char, use memcpy
if ( cs > 1 ) memcpy ( dst , src , cs );
else *dst = *src;
dst += cs;
}
// continue looping over nodes (text and tag nodes)
// do not split a word in the middle! so if we had to
// truncate, at least try to truncate at last punctuation
// mark if we had one.
if ( dst + 5 >= dstEnd && lastp ) {
*lastp = '\0';
len = lastp - buf;
}
// end at dst as well
else {
*dst = '\0';
len = dst - buf;
}
// store node number
if ( matchedNode ) *matchedNode = i;
return len;
}
return 0;
}
bool Xml::hasGigablastForm(char **url, int32_t *urlLen) {
// find the first meta summary node
for ( int32_t i = 0 ; i < m_numNodes ; i++ ) {
// continue if not a FORM tag
if ( m_nodes[i].m_nodeId != 40 ) continue;
// <form method=get action=/cgi/0.cgi name=f>
int32_t len;
char *s = getString ( i , "action" , &len );
if (url) *url = s;
if (urlLen) *urlLen = len;
// skip http://
if ( len > 10 && strncasecmp(s,"http://",7)==0 ) {
s += 7; len -= 7; }
// skip https://
if ( len > 11 && strncasecmp(s,"https://",8)==0 ) {
s += 8; len -= 8; }
// skip www.
if ( len > 4 && strncasecmp(s,"www.",4)==0) {
s += 4; len -= 4; }
// skip sitesearch.
if ( len > 11 && strncasecmp(s,"sitesearch.",11)==0) {
s += 11; len -= 11; }
// need gigablast.com
if ( len < 13 || strncasecmp(s,"gigablast.com",13)!=0)
continue;
// skip gigablast.com
s += 13; len -= 13;
// need slash
if ( len < 2 || *s != '/' ) continue;
// skip slash
s += 1; len -= 1;
// action must be http://www.gigablast.com/cgi/0.cgi EXACTLY
if ( len == 9 && strncasecmp(s,"cgi/0.cgi",9)==0) return true;
// another forms
if ( len == 6 && strncasecmp(s,"search" ,6)==0) return true;
// another form
if ( len == 9 && strncasecmp(s,"index.php",9)==0) return true;
}
return false;
}
// TEST CASES:
//. this is NOT rss, but has an rdf:rdf tag in it!
// http://www.silverstripe.com/silverstripe-adds-a-touch-of-design-and-a-whole-lot-more/
// http://government.zdnet.com/?p=4245
int32_t Xml::isRSSFeed ( ) {
// must have atleast one rss.channel.item.link node
//int32_t rssLink = getNodeNum ( "rss.channel.item.link" );
//if ( rssLink >= 0 )
// return true;
// rdf: must have atleast one rss.channel.item.link node
//rssLink = getNodeNum ( "rdf.channel.item.link" );
//if ( rssLink >= 0 )
// return true;
//bool hasTag = false;
int32_t type = 0;
int32_t tag = 0;
int32_t i;
for ( i = 0; i < m_numNodes; i++ ) {
// skip text nodes (nodeId is 0)
if ( m_nodes[i].m_nodeId == TAG_TEXTNODE ) continue;
// check for RSS/FEED/RDF node
if ( m_nodes[i].m_nodeId == TAG_RDF ) {
tag = TAG_RDF; type = 1; }
if ( m_nodes[i].m_nodeId == TAG_RSS ) {
tag = TAG_RSS; type = 1; }
if ( m_nodes[i].m_nodeId == TAG_FEED ) {
tag = TAG_FEED; type = 6; }
if ( tag ) break;
}
// if no such tag we are definitely not rss
if ( ! tag ) return 0;
// i have only seen rdf tags embedded in html
if ( tag != TAG_RDF ) return type;
// . now check for a <channel>, <item> or <link> tag
// . we need one of those to be useful
for ( i = 0; i < m_numNodes; i++ ) {
if ( m_nodes[i].m_nodeId == TAG_CHANNEL ) return type;
if ( m_nodes[i].m_nodeId == TAG_ITEM ) return type;
if ( m_nodes[i].m_nodeId == TAG_ENTRY ) return type;
//if ( m_nodes[i].m_nodeId == TAG_LINK ) return type;
}
return 0;
}
char *Xml::getRSSTitle ( int32_t *titleLen , bool *isHtmlEncoded ) const {
// assume it is html encoded (i.e. <'s are encoded as &lt;'s)
*isHtmlEncoded = true;
// . extract the RSS/Atom title
// rss/rdf
int32_t tLen;
//char *title = getString ( "item.title",
// &tLen ,
// true );
char *title = getString ( "title" ,
&tLen ,
true );
// atom
//if ( ! title )
// title = getString ( "entry.title",
// &tLen ,
// true );
// watch out for <![CDATA[]]> block
if ( tLen >= 12 && strncasecmp(title, "<![CDATA[", 9) == 0 ) {
title += 9;
tLen -= 12;
*isHtmlEncoded = false;
}
// return
*titleLen = tLen;
return title;
}
char *Xml::getRSSDescription ( int32_t *descLen , bool *isHtmlEncoded ) {
// assume it is html encoded (i.e. <'s are encoded as &lt;'s)
*isHtmlEncoded = true;
// . extract the RSS/Atom description
// rss/rdf
int32_t dLen;
char *desc = getString ( "description", // "item.description",
&dLen ,
true );
// get content first, it is usually more inclusive than the summary
if ( ! desc )
desc = getString ( "content" , // "entry.content",
&dLen ,
true );
// atom
if ( ! desc )
desc = getString ( "summary" , // "entry.summary",
&dLen ,
true );
// watch out for <![CDATA[]]> block
if ( dLen >= 12 && strncasecmp(desc, "<![CDATA[", 9) == 0 ) {
desc += 9;
dLen -= 12;
*isHtmlEncoded = false;
}
// return
*descLen = dLen;
return desc;
}
// get a link to an RSS feed for this page
char *Xml::getRSSPointer ( int32_t *length, int32_t startNode, int32_t *matchedNode ) {
// assume no tag matched
if ( matchedNode ) *matchedNode = -1;
*length = 0;
// find the first meta summary node
for ( int32_t i = startNode ; i < m_numNodes ; i++ ) {
// continue if not a <link> tag
if ( m_nodes[i].m_nodeId != TAG_LINK ) continue;
// . check for rel="alternate"
int32_t len;
char *s = getString ( i, "rel", &len );
// continue if name doesn't match field
// field can be "summary","description","keywords",...
if ( len != 9 ) continue;
if ( strncasecmp ( s , "alternate" , 9 ) != 0 )
continue;
// . check for valid type:
// type="application/atom+xml" (atom)
// type="application/rss+xml" (RSS 1.0/2.0)
// type="application/rdf+xml" (RDF)
// type="text/xml" (RSS .92) support?
s = getString ( i, "type", &len );
if (len == 20 && !strncasecmp(s, "application/atom+xml", 20))
goto isRSS;
if (len == 19 && !strncasecmp(s, "application/rss+xml", 19) )
goto isRSS;
if (len == 19 && !strncasecmp(s, "application/rdf+xml", 19) )
goto isRSS;
// not RSS
continue;
isRSS:
// . extract the link from href=""
s = getString ( i, "href", length );
if ( matchedNode ) *matchedNode = i;
return s;
}
return NULL;
}
// get the link pointed to by this RSS item
char *Xml::getItemLink ( int32_t *linkLen ) {
char *link = NULL;
*linkLen = 0;
// find the first meta summary node
for ( int32_t i = 0; i < m_numNodes ; i++ ) {
// skip node if not an xml tag node
//if ( m_nodes[i].m_nodeId != 1 ) continue;
if ( m_nodes[i].m_nodeId == TAG_ENCLOSURE ) {
link = (char *) getString ( i, "url", linkLen );
if ( link ) return link;
}
// skip if not a <link> tag
else if ( m_nodes[i].m_nodeId != TAG_LINK )
continue;
// do we have an "enclosure" tag? used for multimedia.
/*
if ( m_nodes[i].m_tagNameLen == 9 &&
memcmp(m_nodes[i].m_tagName,"enclosure",9) == 0 )
link = (char *) getString ( i, "url", linkLen );
if ( link ) return link;
// skip if not a <link> tag
if ( m_nodes[i].m_tagNameLen != 4 ) continue;
if (memcmp(m_nodes[i].m_tagName,"link",4)!=0) continue;
}
*/
// check for href string in the <link> tag... wierd...
link = getString ( i, "href", linkLen );
if ( link ) return link;
// if not in href, get the following text node
char *node = m_nodes[i].m_node;
int32_t nodeLen = m_nodes[i].m_nodeLen;
// must not end in "/>"
if (node[nodeLen-2] == '/' ) continue;
// expect <link> url </link>
if ( i + 2 >= m_numNodes ) continue;
if ( !isBackTag(i+2) ) continue;
// get the url
link = m_nodes[i+1].m_node;
*linkLen = m_nodes[i+1].m_nodeLen;
if ( link && &linkLen > 0 ) return link;
}
// no link found, return NULL
*linkLen = 0;
return NULL;
}
int32_t Xml::findNodeNum(char *nodeText) {
// do a binary search to find the node whose text begins at
// this pointer
int32_t a = 0;
int32_t b = m_numNodes - 1;
if ( nodeText < m_nodes[a].m_node ) return -1;
if ( nodeText > m_nodes[b].m_node ) return -1;
while (a < b) {
int32_t mid = ((b-a)>>1) + a;
if ( nodeText == m_nodes[a].m_node ) return a;
if ( nodeText == m_nodes[b].m_node ) return b;
if ( nodeText == m_nodes[mid].m_node ) return mid;
if ( nodeText < m_nodes[mid].m_node ) b = mid - 1;
else a = mid + 1;
}
return -1;
}
// returns -1 if no count found
int32_t Xml::getPingServerCount ( ) {
for ( int32_t i = 0 ; i < m_numNodes && i < 40 ; i++ ) {
if ( ! m_nodes[i].isXmlTag() ) continue;
// must be "weblogUpdates"
int32_t slen = 0;
char *s = getString ( i,"count",&slen);
if ( ! s || slen <= 0 ) continue;
int32_t count = atoi(s);
if ( count == -1 ) continue;
// got it
return count;
}
// no count
return -1;
}
Reference in New Issue View Git Blame Copy Permalink