gigablast/open-source-search-engine
1
0
Fork 0
mirror of https://github.com/gigablast/open-source-search-engine.git synced 2024-10-04 20:27:43 +03:00
Code Issues Projects Releases Wiki Activity

fix spider core

Browse Source
This commit is contained in:
Matt Wells 2013-12-27 15:28:44 -07:00
parent 5fcfff6729
commit 5cdb73bc70
1 changed files with 17 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Msg13.cpp
View File

@ -2186,16 +2186,25 @@ void gotIframeExpandedContent ( void *state ) {
// we respect crawl delay for sure
void scanHammerQueue ( int fd , void *state ) {
Msg13Request *r = s_hammerQueueHead;
if ( ! r ) return;
if ( ! s_hammerQueueHead ) return;
long long nowms = gettimeofdayInMilliseconds();
top:
Msg13Request *r = s_hammerQueueHead;
if ( ! r ) return;
Msg13Request *prev = NULL;
long long waited = -1LL;
Msg13Request *nextLink = NULL;
// scan down the linked list of queued of msg13 requests
for ( ; r ; prev = r , r = r->m_nextLink ) {
for ( ; r ; prev = r , r = nextLink ) {
// downloadTheDocForReals() could free "r" so save this here
nextLink = r->m_nextLink;
long long last;
last = s_hammerCache.getLongLong(0,r->m_firstIp,30,true);
// is one from this ip outstanding?
@ -2224,6 +2233,11 @@ void scanHammerQueue ( int fd , void *state ) {
if ( s_hammerQueueTail == r )
s_hammerQueueTail = prev;
// if "r" was freed by downloadTheDocForReals() then
// in the next iteration of this loop, "prev" will point
// to a freed memory area, so start from the top again
goto top;
// try to download some more i guess...
}
}