gigablast/open-source-search-engine
1
0
Fork 0
mirror of https://github.com/gigablast/open-source-search-engine.git synced 2024-10-04 20:27:43 +03:00
Code Issues Projects Releases Wiki Activity
734973bb81
open-source-search-engine/Summary.cpp
mwells b0caf3eb00 get summary "ns" parm and collectionrec 
knobs for summary gen working.
2014-07-03 07:29:44 -07:00

1736 lines
52 KiB
C++
Raw Blame History

#include "Summary.h"
#include "Speller.h"
#include "Words.h"
//#include "AppendingWordsWindow.h"
#include "Sections.h"
Summary::Summary()
: m_summaryLocs(m_summaryLocBuf,
MAX_SUMMARY_LOCS*sizeof(uint64_t)),
m_summaryLocsPops(m_summaryLocPopsBuf,
MAX_SUMMARY_LOCS*sizeof(long)) {
//m_buf = NULL;
m_bitScoresBuf = NULL;
m_bitScoresBufSize = 0;
reset();
}
Summary::~Summary() { reset(); }
void Summary::reset() {
//if ( m_buf && m_freeBuf )
// mfree ( m_buf, m_bufMaxLen, "Summary" );
if ( m_bitScoresBuf ){
mfree ( m_bitScoresBuf, m_bitScoresBufSize,
"SummaryBitScore" );
m_bitScoresBuf = NULL;
m_bitScoresBufSize = 0;
}
m_summaryLen = 0;
//m_bufMaxLen = 0;
//m_bufLen = 0;
//m_buf = NULL;
m_isNormalized = false;
//m_freeBuf = true;
m_numExcerpts = 0;
m_summaryLocs.reset();
m_summaryLocsPops.reset();
}
//////////////////////////////////////////////////////////////////
//
// THE NEW SUMMARY GENERATOR
//
//////////////////////////////////////////////////////////////////
// returns false and sets g_errno on error
bool Summary::set2 ( Xml *xml ,
Words *words ,
Bits *bits ,
Sections *sections ,
Pos *pos ,
Query *q ,
long long *termFreqs ,
float *affWeights , // 1-1 with qterms
//char *coll ,
//long collLen ,
bool doStemming ,
long maxSummaryLen ,
long maxNumLines ,
long numDisplayLines ,
long maxNumCharsPerLine ,
//long bigSampleRadius ,
//long bigSampleMaxLen ,
bool ratInSummary ,
//TitleRec *tr ,
Url *f ,
//bool allowPunctInPhrase ,
//bool excludeLinkText ,
//bool excludeMetaText ,
//bool hackFixWords ,
//bool hackFixPhrases ,
//float *queryProximityScore,
Matches *matches ,
char *titleBuf ,
long titleBufLen ) {
//m_proximityScore = -1;
// pointless, possibly caller in Msg20 is just interested in
// Msg20Request::m_computeLinkInfo or m_setLinkInfo. NO! we need
// to see if it has all the query terms...
//if ( maxNumLines <= 0 ) return true;
m_numDisplayLines = numDisplayLines;
m_displayLen = 0;
//m_useDateLists = useDateLists;
//m_exclDateList = exclDateList;
//m_begPubDateList = begPubDateList;
//m_endPubDateList = endPubDateList;
//m_diversity = 1.0;
// long long start = gettimeofdayInMilliseconds();
// assume we got maxnumlines of summary
if ( (maxNumCharsPerLine+6)*maxNumLines > maxSummaryLen ) {
//maxNumCharsPerLine = (maxSummaryLen-10)/maxNumLines;
if ( maxNumCharsPerLine < 10 ) maxNumCharsPerLine = 10;
static char s_flag = 1;
if ( s_flag ) {
s_flag = 0;
log("query: Warning. "
"Max summary excerpt length decreased to "
"%li chars because max summary excerpts and "
"max summary length are too big.",
maxNumCharsPerLine);
}
}
// . sanity check
// . summary must fit in m_summary[]
// . leave room for tailing \0
if ( maxSummaryLen >= MAX_SUMMARY_LEN ) {
g_errno = EBUFTOOSMALL;
return log("query: Summary too big to hold in buffer of %li "
"bytes.",(long)MAX_SUMMARY_LEN);
}
// . hash query word ids into a small hash table
// . we use this to see what words in the document are query terms
//long qscores [ MAX_QUERY_TERMS ];
// and if we found each query term or not
//long nt = q->getNumNonFieldedSingletonTerms();
//long nqt = q->getNumTerms();
// do not overrun the final*[] buffers
if ( maxNumLines > 256 ) {
g_errno = EBUFTOOSMALL;
return log("query: More than 256 summary lines requested.");
}
// . MORE BIG HACK
// . since we're working with fielded query terms we must check BIG
// HACK here in case the fielded query term is the ONLY query
// term.
// . LOGIC MOVED INTO MATCHES.CPP
// Nothing to match...print beginning of content as summary
if ( matches->m_numMatches == 0 && maxNumLines > 0 )
return getDefaultSummary ( xml,
words,
sections, // scores,
pos,
//bigSampleRadius,
maxSummaryLen );
/*long long end = gettimeofdayInMilliseconds();
if ( end - start > 2 )
log ( LOG_WARN,"summary: took %lli ms to finish big hack",
end - start );
start = gettimeofdayInMilliseconds();*/
//
// zero out all word weights
for ( long i = 0 ; i < q->m_numWords; i++ )
m_wordWeights[i] = 0.0;
// query terms
long numTerms = q->getNumTerms();
// . compute our word weights wrt each query. words which are more rare
// have a higher weight. We use this to weight the terms importance
// when generating the summary.
// . used by the proximity algo
// . used in setSummaryScores() for scoring summaries
if ( termFreqs && q->m_numWords > 1 ) {
float maxTermFreq = 0;
for ( long i = 0 ; i < numTerms ; i++ ) {
// www.abc.com --> treat www.abc as same term freq
// 'www.infonavit.gob.mx do de carne? mxa'
//if(q->m_qterms[i].m_isPhrase) continue;
if(termFreqs[i] > maxTermFreq)
maxTermFreq = termFreqs[i];
}
maxTermFreq++; //don't div by 0!
for ( long i = 0 ; i < numTerms ; i++ ) {
//if(q->m_qterms[i].m_isPhrase) continue;
// if this is a phrase the other words following
// the first word will have a word weight of 0
// so should be ignored for that...
long ndx = q->m_qterms[i].m_qword - q->m_qwords;
// oh it is already complemented up here
m_wordWeights[ndx] = 1.0 -
((float)termFreqs[i] / maxTermFreq);
//make sure everything has a little weight:
if(m_wordWeights[ndx] < .10) m_wordWeights[ndx] = .10;
//log(LOG_WARN,
//"query word num %li termnum %li freq %f max %f",
//ndx,i,m_wordWeights[ndx],maxTermFreq);
}
}
else {
for ( long i = 0 ; i < q->m_numWords; i++ )
m_wordWeights[i] = 1.0;
}
// convenience
m_maxNumCharsPerLine = maxNumCharsPerLine;
//m_qscores = qscores;
m_q = q;
//m_proximityScore = 0;
bool hadEllipsis = false;
// set the max excerpt len to the max summary excerpt len
long maxExcerptLen = m_maxNumCharsPerLine;
long lastNumFinal = 0;
long maxLoops = 1024;
char *p, *pend;
// if just computing absScore2...
if ( maxNumLines <= 0 )//&& bigSampleRadius <= 0 )
return true;//return matches->m_hasAllQueryTerms;
p = m_summary;
pend = m_summary + maxSummaryLen;
m_numExcerpts = 0;
// . the "maxGotIt" count vector accumulates into "retired"
// . that is how we keep track of what query words we used for previous
// summary excerpts so we try to get diversified excerpts with
// different query terms/words in them
char retired [ MAX_QUERY_WORDS ];
memset ( retired, 0, m_q->m_numWords * sizeof(char) );
// some query words are already matched in the title
for ( long i = 0 ; i < m_q->m_numWords ; i++ )
if ( matches->m_qwordFlags[i] & MF_TITLEGEN )
retired [ i ] = 1;
//
// Loop over all words that match a query term. The matching words
// could be from any one of the 3 Words arrays above. Find the
// highest scoring window around each term. And then find the highest
// of those over all the matching terms.
//
long numFinal;
for ( numFinal = 0; numFinal < maxNumLines; numFinal++ ){
if ( numFinal == m_numDisplayLines )
m_displayLen = p - m_summary;
// reset these at the top of each loop
Match *maxm;
long long maxScore = 0;
long maxa = 0;
long maxb = 0;
long maxi = -1;
long lasta = -1;
char maxGotIt [ MAX_QUERY_WORDS ];
if(lastNumFinal == numFinal) {
if(maxLoops-- <= 0) {
log(LOG_WARN, "query: got infinite loop "
"bug, query is %s url is %s",
m_q->m_orig,
f->getUrl());
break;
}
}
lastNumFinal = numFinal;
// long long stget = gettimeofdayInMilliseconds();
// does the max that we found have a new query word that was
// not already in the summary?
//long maxFoundNew = 0;
// loop through all the matches and see which is best
for ( long i = 0 ; i < matches->m_numMatches ; i++ ) {
long a , b;
// reset lasta if we changed words class
if ( i > 0 && matches->m_matches[i-1].m_words !=
matches->m_matches[i].m_words )
lasta = -1;
// only use matches in title, etc.
mf_t flags = matches->m_matches[i].m_flags;
bool skip = true;
if ( flags & MF_METASUMM ) skip = false;
if ( flags & MF_METADESC ) skip = false;
if ( flags & MF_BODY ) skip = false;
if ( flags & MF_DMOZSUMM ) skip = false;
if ( flags & MF_RSSDESC ) skip = false;
if ( skip ) continue;
// ask him for the query words he matched
char gotIt [ MAX_QUERY_WORDS ];
// clear it for him
memset ( gotIt, 0, m_q->m_numWords * sizeof(char) );
// . get score of best window around this match
// . do not allow left post of window to be <= lasta to
// avoid repeating the same window.
long long score = getBestWindow (matches,
i,
&lasta,
&a, &b,
gotIt ,
retired ,
maxExcerptLen);
// USE THIS BUF BELOW TO DEBUG THE ABOVE CODE.
// PRINTS OUT THE SUMMARY
/*
//if ( score >=12000 ) {
char buf[10*1024];
char *xp = buf;
if ( i == 0 )
log (LOG_WARN,"=-=-=-=-=-=-=-=-=-=-=-=-=-=-=");
sprintf(xp, "score=%08li a=%05li b=%05li ",
(long)score,(long)a,(long)b);
xp += gbstrlen(xp);
for ( long j = a; j < b; j++ ){
//long s = scores->m_scores[j];
long s = 0;
if ( s < 0 ) continue;
char e = 1;
long len = words->getWordLen(j);
for(long k=0;k<len;k +=e){
char c = words->m_words[j][k];
//if ( is_binary( c ) ) continue;
*xp = c;
xp++;
}
//p += gbstrlen(p);
if ( s == 0 ) continue;
sprintf ( xp ,"(%li)",s);
xp += gbstrlen(xp);
}
log (LOG_WARN,"query: summary: %s", buf);
//}
*/
// prints out the best window with the score
/*
char buf[MAX_SUMMARY_LEN];
char *bufPtr = buf;
char *bufPtrEnd = p + MAX_SUMMARY_LEN;
if ( i == 0 )
log (LOG_WARN,"=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=");
long len = 0;
Words *ww = matches->m_matches[i].m_words;
//Sections *ss = matches->m_matches[i].m_sections;
//if ( ss->m_numSections <= 0 ) ss = NULL;
//len=pos->filter(bufPtr, bufPtrEnd, ww, a, b, NULL);
//log(LOG_WARN,"summary: %li) %s - %lli",i,bufPtr,
//score);
log(LOG_WARN,"summary: %li) %s - %lli",i,bufPtr,
score);
*/
// skip if was in title or something
if ( score <= 0 ) continue;
// skip if not a winner
if ( maxi >= 0 && score <= maxScore ) continue;
// we got a new winner
maxi = i;
maxa = a;
maxb = b;
maxScore = score;
// save this too
memcpy ( maxGotIt , gotIt , m_q->m_numWords );
}
// retire the query words in the winning summary
//log( LOG_WARN,"summary: took %lli ms to finish getbestwindo",
// gettimeofdayInMilliseconds() - stget );
// all done if no winner was made
if ( maxi == -1 ) break;
// sanity check
//if ( maxa == -1 || maxb == -1 ) { char *xx = NULL; *xx = 0; }
if ( maxa == -1 ) break;
if ( maxb == -1 ) break;
// who is the winning match?
maxm = &matches->m_matches[maxi];
Words *ww = maxm->m_words;
Sections *ss = maxm->m_sections;
// we now use "m_swbits" for the summary bits since they are
// of size sizeof(swbit_t), a short at this point
swbit_t *bb = maxm->m_bits->m_swbits;
// this should be impossible
if ( maxa > ww->m_numWords || maxb > ww->m_numWords ){
log ( LOG_WARN,"query: summary starts or ends after "
"document is over! maxa=%li maxb=%li nw=%li",
maxa, maxb, ww->m_numWords );
maxa = ww->m_numWords - 1;
maxb = ww->m_numWords;
//char *xx = NULL; *xx = 0;
}
// assume we do not preceed with ellipsis "..."
bool needEllipsis = true;
// rule of thumb, don't use ellipsis if the first letter is
// capital, or a non letter
char *c = ww->m_words[maxa]+0;
if ( ! is_alpha_utf8(c) ) needEllipsis = false;
else if ( is_upper_utf8(c) ) needEllipsis = false;
// is punct word before us pair acrossable? if so then we
// probably are not the start of a sentence.
if ( bb[maxa] & D_STARTS_SENTENCE ) needEllipsis = false;
// or if into the sample and previous excerpt had an ellipsis
// do not bother using one for us.
if ( p > m_summary && hadEllipsis ) needEllipsis = false;
if ( needEllipsis ) {
// break out if no room for "..."
//long elen;
if ( p + 4 + 2 > pend ) break;
// space first?
if ( p > m_summary ) *p++ = ' ';
memcpy ( p , "... " , 4 );
p += 4;
}
// separate summary excerpts with a single space.
if ( p > m_summary ) {
if ( p + 2 > pend ) break;
*p++ = ' ';
}
// assume we need a trailing ellipsis
needEllipsis = true;
// so next excerpt does not need to have an ellipsis if we
// have one at the end of this excerpt
hadEllipsis = needEllipsis;
// start with quote?
if ( (bb[maxa] & D_IN_QUOTES) && p + 1 < pend ) {
// preceed with quote
*p++ = '\"';
}
// . filter the words into p
// . removes back to back spaces
// . converts html entities
// . filters in stores words in [a,b) interval
long len = pos->filter(p, pend, ww, maxa, maxb, ss);
// break out if did not fit
if ( len == 0 ) break;
// don't consider it if it is a substring of the title
if ( len == titleBufLen &&
strncasestr(titleBuf, p, titleBufLen, len) ) {
// don't consider this one
numFinal--;
goto skip;
}
// don't consider it if the length wasn't anything nice
if ( len < 5 ){
numFinal--;
goto skip;
}
// otherwise, keep going
p += len;
// now we just indicate which query terms we got
for ( long i = 0 ; i < m_q->m_numWords ; i++ ) {
// do not breach
if ( retired[i] >= 100 ) continue;
retired [ i ] += maxGotIt [ i ];
}
// add all the scores of the excerpts to the doc summary score.
// zero out scores of the winning sample so we don't get them
// again. use negative one billion to ensure that we don't get
// them again
for ( long j = maxa ; j < maxb ; j++ )
// mark it as used
bb[j] |= D_USED;
// if we ended on punct that can be paired across we need
// to add an ellipsis
if ( needEllipsis ) {
if ( p + 4 + 2 > pend ) break;
memcpy ( p , " ..." , 4 );
p += 4;
}
// try to put in a small summary excerpt if we have atleast
// half of the normal excerpt length left
if ( maxExcerptLen == m_maxNumCharsPerLine &&
//pos->m_pos[maxb] - pos->m_pos[maxa]
len <= ( m_maxNumCharsPerLine / 2 + 1 ) ){
maxExcerptLen = m_maxNumCharsPerLine / 2;
// don't count it in the finals since we try to get a
// small excerpt
numFinal--;
}
else if ( m_numExcerpts < MAX_SUMMARY_EXCERPTS &&
m_numExcerpts >= 0 ) {
m_summaryExcerptLen[m_numExcerpts] = p - m_summary;
m_numExcerpts++;
// also reset maxExcerptLen
maxExcerptLen = m_maxNumCharsPerLine;
}
skip:
// zero out the scores so they will not be used in others
for ( long j = maxa ; j < maxb ; j++ )
// mark it
bb[j] |= D_USED;
}
if ( numFinal <= m_numDisplayLines )
m_displayLen = p - m_summary;
/*end = gettimeofdayInMilliseconds();
if ( end - start > 10 )
log ( LOG_WARN,"summary: took %llims to finish doing summary "
"numMatches=%li maxNumLines=%li url=%s", end - start,
matches.m_numMatches, maxNumLines, f->m_url );
start = gettimeofdayInMilliseconds();*/
// If we still didn't find a summary, directly use whats given in the
// meta summary or description.
if ( p == m_summary ){
Words *wp;
Pos *pp;
Sections *ss;
// get it from the summary
if ( matches->getMatchGroup(MF_METASUMM ,&wp,&pp,&ss) )
p += pp->filter(p,pend, wp, 0, wp->m_numWords, ss );
else if ( matches->getMatchGroup(MF_METADESC,&wp,&pp,&ss) )
p += pp->filter(p,pend, wp, 0, wp->m_numWords, ss );
if ( p != m_summary ){
m_summaryExcerptLen[0] = p - m_summary;
m_numExcerpts = 1;
}
// in this case we only have one summary line
if ( m_numDisplayLines > 0 )
m_displayLen = p - m_summary;
}
// If we still didn't find a summary, get the default summary
if ( p == m_summary ) {
// then return the default summary
bool status = getDefaultSummary ( xml,
words,
sections,
pos,
//bigSampleRadius,
maxSummaryLen );
if ( m_numDisplayLines > 0 )
m_displayLen = m_summaryLen;
return status;
}
// if we don't find a summary, theres no need to NULL terminate
if ( p != m_summary ) *p++ = '\0';
// set length
m_summaryLen = p - m_summary;
if ( m_summaryLen > 50000 ) { char*xx=NULL;*xx=0; }
// it may not have all query terms if rat=0 (Require All Terms=false)
// so use Matches::m_matchesQuery instead of Matches::m_hasAllQTerms
//if ( ! matches->m_matchesQuery )
// log("query: msg20: doc %s missing query terms for q=%s",
// f->getUrl(),m_q->m_orig );
return true;
}
// . usually we get more summary lines than displayed so that the summary
// deduped, XmlDoc::getSummaryVector(), has adequate sample space
// . "max excerpts". we truncate the summary if we need to.
// XmlDoc.cpp::getSummary(), likes to request more excerpts than are
// actually displayed so it has a bigger summary for deduping purposes.
long Summary::getSummaryLen ( long maxLines ) {
long len = 0;
for ( long i = 0 ; i < m_numExcerpts && i < maxLines ; i++ )
len += m_summaryExcerptLen[i];
return len;
}
// MDW: this logic moved mostly to Bits::setForSummary() and
// Summary::set2(). See the gigawiki url to see the rules for summary
// generation: http://10.5.1.202:237/eng_wiki/index.php/Eng:Projects
// i removed this whole function so use git diff to see it later if you
// need to. setSummaryScores() is obsoleted.
// . return the score of the highest-scoring window containing match #m
// . window is defined by the half-open interval [a,b) where a and b are
// word #'s in the Words array indicated by match #m
// . return -1 and set g_errno on error
long long Summary::getBestWindow ( Matches *matches ,
long mm ,
long *lasta ,
long *besta ,
long *bestb ,
char *gotIt ,
char *retired ,
long maxExcerptLen ) {
// get the window around match #mm
Match *m = &matches->m_matches[mm];
// what is the word # of match #mm?
long matchWordNum = m->m_wordNum;
// what Words/Pos/Bits classes is this match in?
Words *words = m->m_words;
Section **sp = NULL;
long *pos = m->m_pos->m_pos;
// use "m_swbits" not "m_bits", that is what Bits::setForSummary() uses
swbit_t *bb = m->m_bits->m_swbits;
// shortcut
if ( m->m_sections ) sp = m->m_sections->m_sectionPtrs;
long nw = words->getNumWords();
long long *wids = words->getWordIds();
nodeid_t *tids = words->getTagIds();
// . sanity check
// . this prevents a core i've seen
if ( matchWordNum >= nw ) {
log("summary: got overflow condition for q=%s",m_q->m_orig);
// assume no best window
*besta = -1;
*bestb = -1;
*lasta = matchWordNum;
return 0;
}
// . we NULLify the section ptrs if we already used the word in another
// summary.
// . google seems to index SEC_MARQUEE, so i took that out of here
long badFlags = SEC_SCRIPT|SEC_STYLE|SEC_SELECT|SEC_IN_TITLE;
if ( (bb[matchWordNum] & D_USED) ||
( sp && (sp[matchWordNum]->m_flags & badFlags) ) ) {
// assume no best window
*besta = -1;
*bestb = -1;
*lasta = matchWordNum;
return 0;
}
// . "a" is the left fence post of the window (it is a word # in Words)
// . go to the left as far as we can
// . thus we decrement "a"
long a = matchWordNum;
// "posa" is the character position of the END of word #a
long posa = pos[a+1];
long firstFrag = -1;
bool startOnQuote = false;
bool goodStart = false;
long wordCount = 0;
// . decrease "a" as long as we stay within maxNumCharsPerLine
// . avoid duplicating windows by using "lasta", the last "a" of the
// previous call to getBestWindow(). This can happen if our last
// central query term was close to this one.
for ( ; a > 0 && posa - pos[a-1] < maxExcerptLen && a > *lasta; a-- ) {
// . don't include any "dead zone",
// . dead zones have already been used for the summary, and
// we are getting a second/third/... excerpt here now then
//if ( wscores[a-1] == -1000000000 ||
if ( (bb[a-1]&D_USED) ||
// stop on a title word as well
//wscores[a-1] == -20000000 ||
// stop if its the start of a sentence, too
bb[a] & D_STARTS_SENTENCE ){
goodStart = true;
break;
}
// stop before title word
if ( bb[a-1] & D_IN_TITLE ) {
goodStart = true;
break;
}
// don't go beyond an LI, TR, P tag
if ( tids && ( tids[a-1] == TAG_LI ||
tids[a-1] == TAG_TR ||
tids[a-1] == TAG_P ||
tids[a-1] == TAG_DIV ) ){
goodStart = true;
break;
}
// stop if its the start of a quoted sentence
if ( a+1<nw && (bb[a+1] & D_IN_QUOTES) &&
words->m_words[a][0] == '\"' ){
startOnQuote = true;
goodStart = true;
break;
}
// find out the first instance of a fragment (comma, etc)
// watch out! because frag also means 's' in there's
if ( ( bb[a] & D_STARTS_FRAG ) &&
!(bb[a-1] & D_IS_STRONG_CONNECTOR) && firstFrag == -1 )
firstFrag = a;
if ( wids[a] ) wordCount++;
}
// if didn't find a good start, then start at the start of the frag
if ( !goodStart && firstFrag != -1 )
a = firstFrag;
// don't let punct or tag word start a line, unless a quote
if ( a < matchWordNum && !wids[a] && words->m_words[a][0] != '\"' ){
while ( a < matchWordNum && !wids[a] ) a++;
// do not break right after a "strong connector", like
// apostrophe
while ( a < matchWordNum && a > 0 &&
( bb[a-1] & D_IS_STRONG_CONNECTOR ) )
a++;
// don't let punct or tag word start a line
while ( a < matchWordNum && !wids[a] ) a++;
}
// remember, b is not included in the summary, the summary is [a,b-1]
// remember to include all words in a matched phrase
long b = matchWordNum + m->m_numWords ;
long endQuoteWordNum = -1;
long numTagsCrossed = 0;
for ( ; b <= nw; b++ ){
if ( b == nw ) break;
if ( pos[b+1] - pos[a] >= maxExcerptLen ) break;
if ( startOnQuote && words->m_words[b][0] == '\"' )
endQuoteWordNum = b;
// don't include any dead zone, those are already-used samples
//if ( wscores[b] == -1000000000 ) break;
if ( bb[b]&D_USED ) break;
// stop on a title word
//if ( wscores[b] == -20000000 ) break;
// stop on a title word
if ( bb[b] & D_IN_TITLE ) break;
if ( wids[b] ) wordCount++;
// don't go beyond an LI or TR backtag
if ( tids && ( tids[b] == (BACKBIT|TAG_LI) ||
tids[b] == (BACKBIT|TAG_TR) ) ){
numTagsCrossed++;
// try to have atleast 10 words in the summary
if ( wordCount > 10 )
break;
}
// go beyond a P or DIV backtag in case the earlier char is a
// ':'. This came from a special case for wikipedia pages
// eg. http://en.wikipedia.org/wiki/Flyover
if ( tids && ( tids[b] == (BACKBIT|TAG_P) ||
tids[b] == (BACKBIT|TAG_DIV) )){
numTagsCrossed++;
// try to have atleast 10 words in the summary
if ( wordCount > 10 && words->m_words[b-1][0] != ':' )
break;
}
}
// don't end on a lot of punct words
if ( b > matchWordNum && !wids[b-1]){
// remove more than one punct words. if we're ending on a quote
// keep it
while ( b > matchWordNum && !wids[b-2] &&
endQuoteWordNum != -1 && b > endQuoteWordNum )
b--;
// do not break right after a "strong connector", like
// apostrophe
while ( b > matchWordNum && (bb[b-2] & D_IS_STRONG_CONNECTOR) )
b--;
}
// a shortcut
Match *ms = matches->m_matches;
// make m_matches.m_matches[mi] the first match in our [a,b) window
long mi ;
// . the match at the center of the window is match #"mm", so that
// matches->m_matches[mm] is the Match class
// . set "mi" to it and back up "mi" as long as >= a
for ( mi = mm ; mi >= 0 && ms[mi-1].m_wordNum >=a ; mi-- ) ;
// now get the score of this excerpt. Also mark all the represented
// query words. Mark the represented query words in the array that
// comes to us. also mark how many times the same word is repeated in
// this summary.
long long score = 0LL;
// is a url contained in the summary, that looks bad! punish!
bool hasUrl = false;
// the word count we did above was just an approximate. count it right
wordCount = 0;
// for debug
char buf[5000];
char *xp = buf;
// wtf?
if ( b > nw ) b = nw;
// first score from the starting match down to a, including match
for ( long i = a ; i < b ; i++ ) {
// debug print out
if ( g_conf.m_logDebugSummary ) {
long len = words->getWordLen(i);
char cs;
for(long k=0;k<len; k+=cs ) {
char *c = words->m_words[i]+k;
cs = getUtf8CharSize(c);
if ( is_binary_utf8 ( c ) ) continue;
memcpy ( xp , c , cs );
xp += cs;
}
}
//if ( wscores[i] < 0 ) continue;
// skip if in bad section, marquee, select, script, style
if ( sp && (sp[i]->m_flags & badFlags) ) continue;
// don't count just numeric words
if ( words->isNum(i) ) continue;
// check if there is a url. best way to check for '://'
if ( !wids[i] ){
char *wrd = words->m_words[i];
long wrdLen = words->m_wordLens[i];
if ( wrdLen == 3 &&
wrd[0] == ':' && wrd[1] == '/' && wrd[2] == '/' )
hasUrl = true;
}
// get the score
//long t = wscores[i];
// just make every word 100 pts
long t = 100;
// penalize it if in one of these sections
if ( bb[i] & ( D_IN_PARENS |
D_IN_HYPERLINK |
D_IN_LIST |
D_IN_SUP |
D_IN_BLOCKQUOTE ) )
//t /= 3;
// backoff since posbd has best window
// in some links, etc.
//t *= .85;
t *= 1;
// boost it if in bold or italics
if ( bb[i] & D_IN_BOLDORITALICS ) t *= 2;
// add the score for this word
score += t;
// print the score, "t"
if ( g_conf.m_logDebugSummary ) {
sprintf ( xp ,"(%li)",t);
xp += gbstrlen(xp);
}
// skip if not wid
if ( ! wids[i] ) continue;
// count the alpha words we got
wordCount++;
// if no matches left, skip
if ( mi >= matches->m_numMatches ) continue;
// get the match
Match *next = &ms[mi];
// skip if not a match
if ( i != next->m_wordNum ) continue;
// must be a match in this class
if ( next->m_words != words ) continue;
// advance it
mi++;
// which query word # does it match
long qwn = next->m_qwordNum;
if ( qwn < 0 || qwn >= m_q->m_numWords ){char*xx=NULL;*xx=0;}
// undo old score
score -= t;
// add 100000 per match
t = 100000;
// weight based on tf, goes from 0.1 to 1.0
t = (long)((float)t * m_wordWeights [ qwn ]);
// if it is a query stop word, make it 10000 pts
if ( m_q->m_qwords[qwn].m_isQueryStopWord ) t = 0;//10000;
// have we matched it in this [a,b) already?
if ( gotIt[qwn] > 0 ) t /= 15;
// have we matched it already in a winning window?
else if ( retired [qwn] > 0 ) t /= 12;
// add it back
score += t;
if ( g_conf.m_logDebugSummary ) {
sprintf ( xp ,"[%li]",t);
xp += gbstrlen(xp);
}
// inc the query word count for this window
if ( gotIt[qwn] < 100 ) gotIt[qwn]++;
}
long oldScore = score;
// apply the bonus if it starts or a sentence
// only apply if the score is positive and if the wordcount is decent
if ( score > 0 && wordCount > 7 ){
// a match can give us 10k to 100k pts based on the tf weights
// so we don't want to overwhelm that too much, so let's make
// this a 20k bonus if it starts a sentence
if ( bb[a] & D_STARTS_SENTENCE ) score += 8000;
// likewise, a fragment, like after a comma
else if ( bb[a] & D_STARTS_FRAG ) score += 4000;
// 1k if the match word is very close to the
// start of a sentence, lets say 3 alphawords
if ( matchWordNum - a < 7 ) score += 1000;
// 20M in case of meta stuff, and rss description, which
// should be the best summary. so give a huge boost
if ( ! tids ) score += 20000000;
}
// a summary isn't really a summary if its less than 7 words.
// reduce the score, but still give it a decent score.
// minus 5M.
if ( wordCount < 7 ) score -= 20000;
// summaries that cross a lot of tags are usually bad, penalize them
if ( numTagsCrossed > 1 ) score -= (numTagsCrossed * 20000);
if ( hasUrl ) score -= 8000;
// show it
if ( g_conf.m_logDebugSummary )
logf(LOG_DEBUG,"score=%08li prescore=%08li a=%05li b=%05li %s",
(long)score,oldScore,(long)a,(long)b,buf);
// set lasta, besta, bestb
*lasta = a;
*besta = a;
*bestb = b;
return score;
}
// get summary when no search terms could be found
bool Summary::getDefaultSummary ( Xml *xml,
Words *words,
Sections *sections,
Pos *pos,
long maxSummaryLen ){
char *p = m_summary;
if (MAX_SUMMARY_LEN < maxSummaryLen)
maxSummaryLen = MAX_SUMMARY_LEN;
// null it out
m_summaryLen = 0;
// try the meta summary tag
if ( m_summaryLen <= 0 )
m_summaryLen = xml->getMetaContent ( p , maxSummaryLen ,
"summary",7);
// the meta descr
if ( m_summaryLen <= 0 )
m_summaryLen = xml->getMetaContent(p,maxSummaryLen,
"description",11);
if ( m_summaryLen > 0 ) {
m_summaryExcerptLen[0] = m_summaryLen;
m_numExcerpts = 1;
return true;
}
bool inTitle = false;
//bool inHeader = false;
bool inTable = false;
bool inList = false;
bool inLink = false;
bool inStyle = false;
int scoreMult = 1;
char *pend = m_summary + maxSummaryLen - 2;
long start = -1, numConsecutive = 0;
long bestStart = -1, bestEnd = -1, longestConsecutive = 0;
long lastAlnum = -1;
// google seems to index SEC_MARQUEE, so i took that out of here
long badFlags = SEC_SCRIPT|SEC_STYLE|SEC_SELECT|SEC_IN_TITLE;
// shortcut
nodeid_t *tids = words->m_tagIds;
long long *wids = words->getWordIds();
// get the section ptr array 1-1 with the words, "sp"
Section **sp = NULL;
if ( sections ) sp = sections->m_sectionPtrs;
for (long i = 0;i < words->getNumWords(); i++){
// skip if in bad section
if ( sp && (sp[i]->m_flags & badFlags) ) continue;
if (start > 0 && bestStart == start &&
( words->m_words[i] - words->m_words[start] ) >=
( maxSummaryLen - 8 )){
longestConsecutive = numConsecutive;
bestStart = start;
bestEnd = lastAlnum;//i-1;
break;
}
if (words->isAlnum(i) ) {
// (scores->getScore(i) * scoreMult) > 0){
if (!inLink)
numConsecutive++;
lastAlnum = i;
if (start < 0) start = i;
continue;
}
nodeid_t tid = tids[i] & BACKBITCOMP;
// we gotta tag?
if ( tid ) {
// ignore <p> tags
if ( tid == TAG_P ) continue;
// is it a front tag?
if ( tid && ! (tids[i] & BACKBIT) ) {
if ( tid == TAG_STYLE )
inStyle = true;
else if ( tid == TAG_TITLE )
inTitle = true;
else if ( tid == TAG_OL || tid == TAG_UL )
inList = true;
else if ( tid == TAG_A )
inLink = true;
}
else if ( tid ) {
if ( tid == TAG_STYLE )
inStyle = false;
else if ( tid == TAG_TITLE )
inTitle = false;
else if ( tid == TAG_OL || tid == TAG_UL )
inList = false;
else if ( tid == TAG_A )
inLink = false;
}
if (inTitle||inList||inTable||inStyle) scoreMult = -1;
else scoreMult = 1;
if ( ! isBreakingTagId(tid) )
continue;
}
else if ( ! wids[i] ) continue;
// end of consecutive words
if ( numConsecutive > longestConsecutive ) {
longestConsecutive = numConsecutive;
bestStart = start;
bestEnd = i-1;
}
start = -1;
numConsecutive = 0;
}
if (bestStart >= 0 && bestEnd > bestStart){
long len = pos->filter(p, pend-10, words,
bestStart,
bestEnd,
sections);//cores);
p += len;
if ( len > 0 && p + 3 + 2 < pend ){
// space first?
if ( p > m_summary ) *p++ = ' ';
memcpy ( p , "..." , 3 );
p += 3;
}
// NULL terminate
*p++ = '\0';
// set length
m_summaryLen = p - m_summary;
if ( m_summaryLen > 50000 ) { char*xx=NULL;*xx=0; }
return true;
}
return true;
}
/*
bool Summary::scanForLocations ( ) {
m_summaryLocs.reset();
m_summaryLocsPops.reset();
Words words;
if ( ! words.set( m_buf, m_bufLen, TITLEREC_CURRENT_VERSION,
false, // computeIds
false // hasHtmlEntities
) )
return false;
char locBuf[1024];
AppendingWordsWindow ww;
if ( ! ww.set( &words,
1, // minWindowSize
5, // maxWindowSize
1024, // buf size
locBuf // buf
) )
return false;
// find all phrases between length of 1 and 5
for (ww.processFirstWindow(); !ww.isDone(); ww.processNextWindow()) {
ww.act();
char *phrasePtr = ww.getPhrasePtr();
long phraseLen = ww.getPhraseLen();
long numPhraseWords = ww.getNumWords();
if ( numPhraseWords == 0 ) continue;
// see if buf phrase is a place
long placePop = getPlacePop( phrasePtr, phraseLen);
if ( placePop > 50000 ) {
uint64_t place = hash64d( phrasePtr, phraseLen);
if (place == 0) continue;
log(LOG_DEBUG, "query: found place:'%s' (len:%ld) in "
"summary -- h:%llu pop:%ld",
phrasePtr, phraseLen, place, placePop);
if (!m_summaryLocs.safeMemcpy((char *)&place,
sizeof(uint64_t)))
return false;
if (!m_summaryLocsPops.safeMemcpy((char *)&placePop,
sizeof(long)))
return false;
}
}
// sanity check - should have same # of locs as loc pops
if (m_summaryLocs.length()/sizeof(uint64_t) !=
m_summaryLocsPops.length()/sizeof(long)) {
char *xx = NULL; *xx = 0;
}
return true;
}
*/
///////////////
//
// YE OLDE SUMMARY GENERATOR of LORE
//
///////////////
// i upped this from 300 to 3000 to better support the BIG HACK
#define MAX_TO_MATCH 3000
bool Summary::set0 ( char *doc , long docLen , Query *q, Msg20Request *mr ) {
return set1 ( doc ,
docLen ,
q ,
mr->m_summaryMaxLen ,
mr->m_numSummaryLines,
mr->m_maxNumCharsPerLine ,
mr->m_bigSampleRadius ,
mr->m_bigSampleMaxLen ,
NULL , // bigSampleLen ptr!
NULL ,
(long long *)mr->ptr_termFreqs );
}
// . doc must be NULL terminated
// . returns false and sets g_errno on error
// . CAUTION: writes into "doc"
bool Summary::set1 ( char *doc ,
long docLen ,
Query *q ,
long maxSummaryLen ,
long maxNumLines ,
long maxNumCharsPerLine ,
long bigSampleRadius ,
long bigSampleMaxLen ,
long *bigSampleLen ,
char *foundTermVector ,
long long *termFreqs ) {
// reset summary
m_summaryLen = 0;
m_summary[0]='\0';
// boundary check
if ( MAX_SUMMARY_LEN < maxNumCharsPerLine * maxNumLines ) {
g_errno = EBUFTOOSMALL;
return log("query: Summary too big to hold in buffer of %li "
"bytes.",(long)MAX_SUMMARY_LEN);
}
// query terms
long numTerms = q->getNumTerms();
// . now assign scores based on term frequencies
// . highest score is 10000, then 9900, 9800, 9700, ...
long ptrs [ MAX_QUERY_TERMS ];
for ( long i = 0 ; i < numTerms ; i++ ) ptrs[i] = i;
// convenience var
long long *freqs = termFreqs; // q->getTermFreqs();
// . this is taken from IndexTable.cpp
// . bubble sort so lower freqs (rare terms) are on top
bool flag = true;
while ( flag ) {
flag = false;
for ( long i = 1 ; i < numTerms ; i++ ) {
if ( freqs[i] >= freqs[i-1] ) continue;
long tmp = freqs[i];
freqs[i ] = freqs[i-1];
freqs[i-1] = tmp;
tmp = ptrs[i];
ptrs [i ] = ptrs [i-1];
ptrs [i-1] = tmp;
flag = true;
}
}
// assign scores, give rarest terms highest score
long scores [ MAX_QUERY_TERMS ];
for ( long i = 0 ; i < numTerms ; i++ )
scores[ptrs[i]] = 10000000 - (i*100);
// force QUERY stop words to have much lower scores at most 10000
for ( long i = 0 ; i < numTerms ; i++ )
if ( q->isQueryStopWord(i) && q->getTermSign(i) == '\0' )
//scores[i] /= 100000;
scores[i] = 0;
// . don't bother with ignored terms (mostly stop words) but could be
// word from a compound word like cd-rom or some_file
// . typically they will just be represented by a phrase termId
// . we need to include so we can match on those words
//for ( long i = 0 ; i < numTerms ; i++ )
// if ( q->m_ignore[i] ) scores[i] = 0;
// don't include if no word representation to match (like phrases)
for ( long i = 0 ; i < numTerms ; i++ )
if ( q->isPhrase(i) ) scores[i] = 0;
// don't highlight '-' terms (or boolean terms in a NOT clause)
for ( long i = 0 ; i < numTerms ; i++ ) {
if ( q->getTermSign(i) == '-' ) scores[i] = -1000000;
//if ( q->m_qterms[i].m_underNOT ) scores[i] = -1000000;
// don't highlight stuff in fields
if ( q->m_qterms[i].m_qword->m_fieldCode) scores[i] = -1000000;
}
// . set the "m" array
// . it helps us avoid excessive use of strcmp()
// . m [c] lets us know if a query term begins with the letter c
// . m2[c] lets us know if a query term's 2nd letter is c
char m [256];
char m2[256];
memset ( m , 0 , 256 );
memset ( m2 , 0 , 256 );
// populate
for ( long i = 0 ; i < numTerms ; i++ ) {
if ( scores[i] <= 0 ) continue;
long tlen = q->getTermLen ( i );
char *t = q->getTerm ( i );
// bitch if NULL!!!
if ( ! t || tlen <= 0 ) continue;
char t0 = t[0];
// count both upper and lower case!
if ( is_ascii(t0) ) {
m[(unsigned char)(to_upper_a(t0))] = 1;
m[(unsigned char)(to_lower_a(t0))] = 1;
}
else {
m[(unsigned char)t0] = 1;
}
// if we convert all chars to ascii beforing hashing, watch out
if ( tlen <= 2 ) { m2[0]=1; continue; }
char t1 = t[1];
// c++ et al are special cases
// but do we really need to call it '0'???
//if ( ! is_alnum_a(t1) ) { m2[0] = 1; continue; }
if ( is_ascii(t1) ) {
m2[(unsigned char)(to_upper_a((t1)))] = 1;
m2[(unsigned char)(to_lower_a((t1)))] = 1;
}
else {
m2[(unsigned char)t1] = 1;
}
}
// . score of each word matching a query term in doc
// . divide by 2 since we don't match on punctuation words, only alnum
// . wordPtrs pts into "doc" to the matching word
char *wordPtrs [MAX_TO_MATCH];
long qterms [MAX_TO_MATCH];
long numMatches = 0;
// . now find the matches by using strncasecmp()
// . we make sure the first 2 chars match before call strncasecmp()
// . we set the scores[] array
unsigned char *s = (unsigned char *)doc;
long i = 0;
long j;
unsigned char c;
// this flag is used to ensure we do phrases correctly.
// without it, the query "business development center" (in quotes)
// would match a doc with "business development" and
// "development center" as two separate phrases.
char cflag = 0;
while ( s[i] ) {
// skip non-alnum chars
// while ( s[i] && ! is_alnum(s[i]) ) i++;
for ( ; ! is_alnum_utf8 (s+i ) ; i += getUtf8CharSize(s+i) ) {
// if we hit start of a tag, skip the whole tag
//if ( s[i] == '<' ) i = skipTag ( s , i );
// else i += getUtf8CharSize(s+i);
}
// get length
j = i;
// while ( is_alnum (s[j] ) ) j++;
for ( ; is_alnum_utf8 (s+j ) ; j += getUtf8CharSize(s+j) );
// if no alnum after, bail
if ( j == i ) break;
// . does this word match a query word?
// . continue if first char matches no query term
if ( ! m[s[i]] ) { i = j; cflag = 0; continue; }
// get 2nd char
c = s[i+1];
// . if not alnum use \0
// . do we need this???
//if ( ! is_alnum_a ( c ) ) c = '\0';
// does 2nd char match a query term?
if ( ! m2[c] ) { i = j; cflag = 0; continue; }
// add in + or ++ (from Words.cpp)
if ( s[j] == '+' ) {
if ( s[j+1]=='+' && !is_alnum_utf8(s+j+2) ) j += 2;
else if ( !is_alnum_utf8(s+j+1) ) j++;
}
// c#
if ( s[j] == '#' && !is_alnum_utf8(s+j+1) ) j++;
// . check all the way here, it's probably a match
// . TODO: what about phrases?
long k ;
for ( k = 0 ; k < numTerms ; k++ ) {
if ( scores[k] <= 0 ) continue;
if ( q->getTermLen(k) != (j-i)<<1 ) continue;
// . watch out for foreign chars on this compare
// . advance over first 2 letters which we know match
// . no, they could match different words!!! fixed!
unsigned char *s1 = &s[i] ;
unsigned char *s2 = (unsigned char *)q->getTerm(k) ;
//long len = j - i ;
unsigned char *s1end = s1 + j - i;
char size1 ;
char size2 ;
// compare them independent of case in utf8
for ( ; s1 < s1end ; ) {
size1 = getUtf8CharSize(s1);
size2 = getUtf8CharSize(s2);
if ( size1 != size2 ) break;
long low1 = to_lower_utf8_32 ( (char *)s1 );
long low2 = to_lower_utf8_32 ( (char *)s2 );
if ( low1 != low2 ) break;
s1 += size1;
s2 += size2;
}
// if no match, try next term
if ( s1 < s1end ) continue;
// if it's matching a term involved in a compound
// phrase then we must have matched the prev word
if ( q->m_qterms[k].m_phrasePart >= 0 &&
! q->m_hasDupWords ) {
//if ( cflag > 0 && k == 7 )
// log("hey");
//if ( cflag > 0 && k == 6 )
// log("hey");
// are we the first in a compound phrase?
if ( k == 0 ||
q->m_qterms[k-1].m_isPhrase ||
q->m_qterms[k-1].m_phrasePart !=
q->m_qterms[k+0].m_phrasePart )
cflag = k;
// are we not the first in a compound phrase?
else if ( cflag == k-1 &&
q->m_qterms[k+0].m_phrasePart ==
q->m_qterms[k-1].m_phrasePart )
cflag = k;
// if query has dup words, do a strncmp!
//else if (strncasecmp(q->m_qterms[k].m_term,
// (char *)s1,j-i)==0)
// cflag = k;
// otherwise the phrase chain was broken
else {
cflag = 0;
// do not count as a match even
continue;
}
}
// set term vector for the BIG HACK
if ( foundTermVector ) foundTermVector[k] = 1;
// skip this if we got too many, but we still go
// through the ropes for the BIG HACK
if ( numMatches >= MAX_TO_MATCH ) continue;
// we got a match for sure
wordPtrs [ numMatches ] = (char *)&s[i];
qterms [ numMatches ] = k;
numMatches++;
//if ( numMatches >= MAX_TO_MATCH ) goto combine;
break;
}
if ( k == numTerms ) cflag = 0;
// advance to j now
i = j;
}
combine:
// if no summary request, we're done
if ( maxNumLines <= 0 || maxSummaryLen <= 0 ) goto getsample;
{
// combine neighbors scores to yours
long score;
long radius = maxNumCharsPerLine / 2 - 5;
// min of one
if ( radius <= 0 ) radius = 1;
// if a match is within maxNumCharsPerLine chars of it, add it in
long a , b ;
long ascore ;
long qterm;
long max = 0;
long maxi = -1;
long maxa = 0;
long maxb = 0;
char gotIt [ MAX_QUERY_TERMS ];
char *maxleft = NULL;
char *maxright = NULL;
for ( long i = 0 ; i < numMatches ; i++ ) {
// if word already used, skip it
if ( qterms[i] == -1 ) continue;
// set totalScore base
score = scores[qterms[i]];
// use this so we can decrease score of repeated query terms
for ( long j = 0 ; j < numTerms ; j++ ) gotIt[j] = 0;
// add a got it for us
gotIt [qterms[i]] = 1;
// add in our left neighbors
a = i ;
while ( --a >= 0 ) {
// get distance from center
long dist = wordPtrs[i] - wordPtrs[a] ;
// break out if too far away
if ( dist > radius ) break;
// stop if we hit start of sentence
// if we hit a term already used, stop
if ( qterms[a] == -1 ) break;
// date terms are required so make the score huge, 2B
if ( qterms[a] < 0 ) {
score = 2000000000;
continue;
}
// it's score
ascore = scores[qterms[a]];
// it's query term #
qterm = qterms[a];
// reduce score of this term if we already have it
if ( gotIt[qterm] ) ascore /= 100;
// reduce by how far away we are from center
ascore -= (ascore / radius * dist) / 2 ;
// ensure a min of 1
if ( ascore <= 0 ) ascore = 1;
// add it in
score += ascore;
// in case we get it again
gotIt[qterm]++;
}
// inc a so we're on the word to be included
a++;
// for summaries, keep going back until we hit some punctuation
// that delimits the sentence... if any.
char *pp = wordPtrs[a];
char *ppmin = pp - 2*radius;
if ( ppmin < doc ) ppmin = doc;
char sent = 0;
for ( ; pp > ppmin ; pp-- ) {
if ( pp[-1] == '.' ) { sent = 1; break; }
if ( pp[-1] == '?' ) { sent = 1; break; }
if ( pp[-1] == '!' ) { sent = 1; break; }
if ( pp[-1] == ':' ) { sent = 1; break; }
// Xml::getText() replaces breaking tags with double
// \n's, so assume it will also break a sentence.
if ( pp[-1] == '\n' &&
pp+2 > doc &&
pp[-2] == '\n' ) { sent = 1; break; }
}
// samples that start with a sentence beginning get more points
if ( sent || pp == doc ) score *= 2;
// otherwise, don't worry about it
else pp = wordPtrs[a];
// skip back over punct
// while ( ! is_alnum(*pp) && pp < wordPtrs[a] ) pp++;
for ( ; ! is_alnum_utf8(pp) && pp < wordPtrs[a] ;
pp += getUtf8CharSize(pp) );
// this may be smaller than normal if we had to extend the
// left radius to make sure it started at the beginning of
// a sentence.
long bradius = 2*radius - (wordPtrs[a] - pp);
// do not go over doc end
if ( pp + bradius > doc + docLen ) bradius = doc + docLen - pp;
// add in our right neighbors
b = i ;
while ( ++b < numMatches ) {
// get distnace from center
long dist = wordPtrs[b] - wordPtrs[i] ;
// break out if too far away
//if ( dist > radius ) break;
if ( dist > bradius ) break;
// if we hit a term already used, stop
if ( qterms[b] == -1 ) break;
// it's score
ascore = scores[qterms[b]];
// it's query term #
qterm = qterms[b];
// reduce score of this term if we already have it
if ( gotIt[qterm] ) ascore /= 100;
// reduce by how far away we are from center
ascore -= (ascore / radius * dist) / 2 ;
// ensure a min of 1
if ( ascore <= 0 ) ascore = 1;
// add it in
score += ascore;
// in case we get it again
gotIt[qterm]++;
}
// samples with extra punctuation cruft are bad
char *s = pp;
char *send = wordPtrs[i] + bradius;
char ssize;
for ( ; s < send ; s += ssize ) {
ssize = getUtf8CharSize(s);
if ( !is_alnum_utf8(s) &&
*s!=',' &&
!is_alnum_utf8(s+ssize) &&
*(s+ssize)!='\"' )
score >>= 1;
}
// is this the new max? continue, if not
if ( score <= max && maxi >= 0 ) continue;
// otherwise, we got a winner
max = score;
maxi = i;
maxa = a;
maxb = b;
maxleft = pp;
maxright = wordPtrs[i] + bradius;
}
// if no matches, return
if ( maxi == -1 ) return true;
// the winning word, whose neighborhood scored the highest
//char *center = wordPtrs[maxi];
// set excerpt boundaries
//char *left = center - radius;
char *left = maxleft - 1;
if ( left < doc ) left = doc;
char *docLast = doc + docLen - 1;
//char *right = center + radius;
char *right = maxright;
if ( right > docLast ) right = docLast;
// don't let excerpt ptrs break a word
//while ( is_alnum (*left ) && left > doc ) left++;
//while ( is_alnum (*right) && right < docLast ) right--;
for ( ; is_alnum_utf8 (left ) && left > doc ; )
left += getUtf8CharSize(left);
for ( ; is_alnum_utf8 (right) && right < docLast ; )
// back up over all of utf8 char
for ( ; (*right & 0xc0) == 0x80 ; right-- );
// skip the over initial or ending non-alnum chars
//while ( ! is_alnum (*left ) ) left++;
//while ( ! is_alnum (*right) ) right--;
for ( ; ! is_alnum_utf8 (left ) ; )
left += getUtf8CharSize(left);
for ( ; ! is_alnum_utf8 (right) ; )
// back up over all of utf8 char
for ( ; (*right & 0xc0) == 0x80 ; right-- );
// get excerpt length
long elen = right - left + 1;
// if 0 or less, no summary
if ( elen <= 0 ) return true;
// . store in m_summary[]
// . filter out \n \t \r (and multiple sequential spaces later?)
// . convert < and > to &lt; and &gt; respectively
char *p = m_summary + m_summaryLen;
// leave room for NULL termination and any html entities we insert
char *pend = m_summary + MAX_SUMMARY_LEN - 6;
char *pstart = p;
for ( long i = 0 ; i < elen && p < pend ; i++ ) {
if ( left[i] == '<' ) {*p++='&';*p++='l';*p++='t';*p=';';}
else if ( left[i] == '>' ) {*p++='&';*p++='g';*p++='t';*p=';';}
else if ( left[i] == '\t' ) { *p=' '; }
else if ( left[i] == '\n' ) { *p=' '; }
else if ( left[i] == '\r' ) { *p=' '; }
else { *p = left[i]; }
// don't add it if it was a space and there's a space before it
if ( *p==' ' && p > pstart && *(p-1)==' ' ) continue;
// officially add it
p++;
}
// NULL terminate
*p++ = '\0';
// set m_summaryLen
m_summaryLen = p - m_summary;
// . now reduce the scores by what's in gotIt, so those terms are less
// likely to be matched again, it gives others a chance
// . clear the gotIt array
for ( long j = 0 ; j < numTerms ; j++ ) gotIt[j] = 0;
// reduce scores of query terms included in this summary excerpt
for ( long j = maxa ; j < maxb ; j++ ) {
qterm = qterms[j];
if ( gotIt[qterm] != 0 ) continue;
gotIt[qterm] = 1;
scores [qterm] /= 8;
}
// remove winning matches from our 2 arrays so we don't do again
for ( long j = maxa ; j < maxb ; j++ ) qterms[j] = -1;
// clear out from "doc" so we don't dup any of summary, too
memset ( left , ' ' , elen );
// . do we have enough excerpts?
// . if not keep looping
if ( --maxNumLines > 0 ) goto combine;
}
getsample:
char *docEnd = doc + docLen;
char *p = doc;
char *oldright = (char *)0x7fffffff;
char *oldleft = NULL;
// if no big sample request, skip this part
if ( bigSampleRadius <= 0 || bigSampleMaxLen <= 0 ) return true;
// get text within a radius of bigSampleRadius words of every
// query term for generating related topics and what not
for ( long i = 0 ; i < numMatches ; i++ ) {
// if it is a stop word or ignored, skip it, unless forced
// with a plus sign
long qt = qterms[i];
if ( q->isQueryStopWord(qt) && q->getTermSign(qt) == '\0' )
continue;
// point to left extreme
char *left = wordPtrs[i] - bigSampleRadius ;
if ( left < doc ) left = doc;
char *right = wordPtrs[i] + bigSampleRadius ;
if ( right > docEnd ) right = docEnd;
// increase left to avoid splitting words
//while(is_alnum(*left ) && left > doc && is_alnum(left[-1] ))
// left--;
// decrease right to avoid splitting words
//while(is_alnum(*right) && right > doc && is_alnum(right[-1]))
// right--;
// don't let excerpt ptrs break a word
for ( ; is_alnum_utf8 (left ) && left > doc ; ) {
// get char to left
char *pre = left -1;
// back up over all of utf8 char
for ( ; (*pre & 0xc0) == 0x80 ; pre-- );
// stop if not alnum
if ( ! is_alnum_utf8(pre) ) break;
// back up left otherwise
left = pre;
}
for ( ; is_alnum_utf8 (right ) && right > doc ; ) {
// get char to right
char *pre = right -1;
// back up over all of utf8 char
for ( ; (*pre & 0xc0) == 0x80 ; pre-- );
// stop if not alnum
if ( ! is_alnum_utf8(pre) ) break;
// back up right otherwise
right = pre;
}
// if no previous sample claim it all
if ( oldright == (char *)0x7fffffff ) {
oldleft = left;
oldright = right;
}
// if disjoint with previous sample, write previous sample
else if ( left > oldright ) {
long size = oldright - oldleft;
if ( p + size + 1 < docEnd ) {
memcpy ( p , oldleft , size );
p += size ;
*p++ = '\0';
}
// we become the old left and right now
oldleft = left;
oldright = right;
// break out if we got enough
if ( p - doc >= bigSampleMaxLen ) break;
}
// otherwise merge with previous sample
else oldright = right;
}
// write out the last one here
if ( oldright != (char *)0x7fffffff ) {
long size = oldright - oldleft;
if ( p + size + 1 < docEnd ) {
memcpy ( p , oldleft , size );
p += size ;
*p++ = '\0';
}
}
// back up if we exceeded limit
if ( p > doc + bigSampleMaxLen ) p = doc + bigSampleMaxLen;
// don't split last word
//while ( p > doc && is_alnum(*p) && is_alnum(p[-1]) ) p--;
for ( ; p > doc && is_alnum_utf8 (p ) ; ) {
// get char to p
char *pre = p -1;
// back up over all of utf8 char
for ( ; (*pre & 0xc0) == 0x80 ; pre-- );
// stop if not alnum
if ( ! is_alnum_utf8(pre) ) break;
// back up p otherwise
p = pre;
}
// NULL terminate
//*p = '\0';
// debug msg
// print it all out
/*
char *tt = doc;
char *ttend = tt + (p - doc);
while ( tt < ttend ) {
log("%s",tt);
tt += gbstrlen(tt) + 1;
}
*/
// set sample length
*bigSampleLen = p - doc;
// success
return true;
}
Reference in New Issue View Git Blame Copy Permalink