gigablast/open-source-search-engine
1
0
Fork 0
mirror of https://github.com/gigablast/open-source-search-engine.git synced 2024-10-04 12:17:35 +03:00
Code Issues Projects Releases Wiki Activity

fix diffbot smoke tests. do not index spider replies

Browse Source 
for custom crawls.
This commit is contained in:
Matt Wells 2014-05-12 15:14:11 -07:00
parent 0a2523f361
commit 5f7bbe7523
2 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Hostdb.cpp
View File

@ -2666,7 +2666,7 @@ long *getLocalIps ( ) {
// skip if loopback we stored above
if ( ip == loopback ) continue;
// skip bogus ones
if ( ip == 0 || ip == 1 || ip == 2 ) continue;
if ( (unsigned long)ip <= 10 ) continue;
// show it
//log("host: detected local ip %s",iptoa(ip));
// otherwise store it

10
XmlDoc.cpp
View File

@ -20345,7 +20345,10 @@ char *XmlDoc::getMetaList ( bool forDelete ) {
// i guess it is safe to do this after getting the spiderreply
SafeBuf *spiderReplyMetaList = NULL;
if ( cr->m_indexSpiderReplies && m_useSpiderdb ) {
if ( cr->m_indexSpiderReplies &&
m_useSpiderdb &&
// doing it for diffbot throws off smoketests
! cr->m_isCustomCrawl ) {
// get the spiderreply ready to be added
spiderReplyMetaList = getSpiderReplyMetaList ( newsr );
// error?
@ -20882,7 +20885,10 @@ char *XmlDoc::getMetaList ( bool forDelete ) {
// i guess it is safe to do this after getting the spiderreply
SafeBuf *spiderReplyMetaList = NULL;
if ( cr->m_indexSpiderReplies && m_useSpiderdb ) {
if ( cr->m_indexSpiderReplies &&
m_useSpiderdb &&
// doing it for diffbot throws off smoketests
! cr->m_isCustomCrawl ) {
// get the spiderreply ready to be added to the rdbs w/ msg4
spiderReplyMetaList = getSpiderReplyMetaList ( newsr );
// block?