gigablast/open-source-search-engine
1
0
Fork 0
mirror of https://github.com/gigablast/open-source-search-engine.git synced 2024-10-04 12:17:35 +03:00
Code Issues Projects Releases Wiki Activity

more fixes to pausing spiders if too many incoming udp slots.

Browse Source 
raised limit from 200 to 300.
This commit is contained in:
Matt Wells 2015-09-02 07:43:18 -07:00
parent aeb5039470
commit 1dc68de0fa
2 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
PageHosts.cpp
View File

@ -566,7 +566,8 @@ skipReplaceHost:
h->m_pingInfo.m_udpSlotsInUseIncoming ) {
char *f1 = "";
char *f2 = "";
if ( h->m_pingInfo.m_udpSlotsInUseIncoming >= 200 ) {
// MAXUDPSLOTS in Spider.cpp is 300 right now
if ( h->m_pingInfo.m_udpSlotsInUseIncoming >= 300 ) {
f1 = "<b>";
f2 = "</b>";
}

11
Spider.cpp
View File

@ -2835,6 +2835,9 @@ static void gotSpiderdbListWrapper2( void *state , RdbList *list,Msg5 *msg5) {
THIS->populateWaitingTreeFromSpiderdb ( true );
}
// lower from 1300 to 300
#define MAXUDPSLOTS 300
//////////////////
//////////////////
//
@ -2867,6 +2870,8 @@ void SpiderColl::populateWaitingTreeFromSpiderdb ( bool reentry ) {
if ( m_deleteMyself ) { char *xx=NULL;*xx=0; }
// skip if spiders off
if ( ! m_cr->m_spideringEnabled ) return;
// skip if udp table is full
if ( g_udpServer.getNumUsedSlotsIncoming() >= MAXUDPSLOTS ) return;
// if entering for the first time, we need to read list from spiderdb
if ( ! reentry ) {
// just return if we should not be doing this yet
@ -3117,6 +3122,9 @@ void SpiderColl::populateDoledbFromWaitingTree ( ) { // bool reentry ) {
// this.
if ( ! g_conf.m_spideringEnabled ) return;
// skip if udp table is full
if ( g_udpServer.getNumUsedSlotsIncoming() >= MAXUDPSLOTS ) return;
// try skipping!!!!!!!!!!!
// yeah, this makes us scream. in addition to calling
// Doledb::m_rdb::addRecord() below
@ -5945,9 +5953,6 @@ void SpiderLoop::startLoop ( ) {
log("build: failed to register updatecrawlinfowrapper");
}
// lower from 1300 to 200
#define MAXUDPSLOTS 200
// call this every 50ms it seems to try to spider urls and populate doledb
// from the waiting tree
void doneSleepingWrapperSL ( int fd , void *state ) {