From 5cdb73bc700e130f73ba23095a02f47ec9dfb0d0 Mon Sep 17 00:00:00 2001 From: Matt Wells Date: Fri, 27 Dec 2013 15:28:44 -0700 Subject: [PATCH] fix spider core --- Msg13.cpp | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/Msg13.cpp b/Msg13.cpp index 416ae476..9ce2ed57 100644 --- a/Msg13.cpp +++ b/Msg13.cpp @@ -2186,16 +2186,25 @@ void gotIframeExpandedContent ( void *state ) { // we respect crawl delay for sure void scanHammerQueue ( int fd , void *state ) { - Msg13Request *r = s_hammerQueueHead; - if ( ! r ) return; + if ( ! s_hammerQueueHead ) return; long long nowms = gettimeofdayInMilliseconds(); + top: + + Msg13Request *r = s_hammerQueueHead; + if ( ! r ) return; + Msg13Request *prev = NULL; long long waited = -1LL; + Msg13Request *nextLink = NULL; // scan down the linked list of queued of msg13 requests - for ( ; r ; prev = r , r = r->m_nextLink ) { + for ( ; r ; prev = r , r = nextLink ) { + + // downloadTheDocForReals() could free "r" so save this here + nextLink = r->m_nextLink; + long long last; last = s_hammerCache.getLongLong(0,r->m_firstIp,30,true); // is one from this ip outstanding? @@ -2224,6 +2233,11 @@ void scanHammerQueue ( int fd , void *state ) { if ( s_hammerQueueTail == r ) s_hammerQueueTail = prev; + // if "r" was freed by downloadTheDocForReals() then + // in the next iteration of this loop, "prev" will point + // to a freed memory area, so start from the top again + goto top; + // try to download some more i guess... } }