gigablast/open-source-search-engine
1
0
Fork 0
mirror of https://github.com/gigablast/open-source-search-engine.git synced 2024-10-04 12:17:35 +03:00
Code Issues Projects Releases Wiki Activity

more spider proxy fixes

Browse Source
This commit is contained in:
mwells 2014-06-02 16:53:06 -07:00
parent 38854e44f3
commit 2582a487a5
1 changed files with 16 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
SpiderProxy.cpp
View File

@ -115,7 +115,7 @@ bool buildProxyTable ( ) {
// scan the user inputted space-separated list of ip:ports // scan the user inputted space-separated list of ip:ports
for ( ; *p ; ) { for ( ; *p ; ) {
// skip white space // skip white space
if ( is_wspace_a(*p) ) continue; if ( is_wspace_a(*p) ) { p++; continue; }
// scan in an ip:port // scan in an ip:port
char *s = p; char *portStr = NULL; char *s = p; char *portStr = NULL;
long dc = 0, pc = 0, gc = 0, bc = 0; long dc = 0, pc = 0, gc = 0, bc = 0;
@ -166,7 +166,7 @@ bool buildProxyTable ( ) {
// . we got a legit ip:port // . we got a legit ip:port
// . see if already in our table // . see if already in our table
unsigned long long ipKey = ip; unsigned long long ipKey = (unsigned long)ip;
ipKey <<= 16; ipKey <<= 16;
ipKey |= (unsigned short)(port & 0xffff); ipKey |= (unsigned short)(port & 0xffff);
@ -219,7 +219,16 @@ bool saveSpiderProxyStats ( ) {
bool loadSpiderProxyStats ( ) { bool loadSpiderProxyStats ( ) {
// save hash table // save hash table
return s_iptab.load(g_hostdb.m_dir,"spiderproxystats.dat"); if ( ! s_iptab.load(g_hostdb.m_dir,"spiderproxystats.dat") )
return false;
// unset some flags
for ( long i = 0 ; i < s_iptab.getNumSlots() ; i++ ) {
// skip empty slots
if ( ! s_iptab.m_flags[i] ) continue;
SpiderProxy *sp = (SpiderProxy *)s_iptab.getValueFromSlot(i);
sp->m_isWaiting = false;
}
return true;
} }
long getNumLoadPoints ( SpiderProxy *sp ) { long getNumLoadPoints ( SpiderProxy *sp ) {
@ -373,14 +382,14 @@ void gotTestUrlReplyWrapper ( void *state , TcpSocket *s ) {
//mfree ( ss , sizeof(spip) ,"spip" ); //mfree ( ss , sizeof(spip) ,"spip" );
// note it // note it
log("sproxy: got test url reply: %s", log("sproxy: got test url reply (%s): %s",
s->m_readBuf); mstrerror(g_errno),s->m_readBuf);
// we can get the spider proxy ip/port from the socket because // we can get the spider proxy ip/port from the socket because
// we sent this url download request to that spider proxy // we sent this url download request to that spider proxy
unsigned long long key = (unsigned long)s->m_ip; unsigned long long key = (unsigned long)s->m_ip;
key <<= 16; key <<= 16;
key |= (unsigned long)s->m_port; key |= (unsigned short)(s->m_port & 0xffff);
SpiderProxy *sp = (SpiderProxy *)s_iptab.getValue ( &key ); SpiderProxy *sp = (SpiderProxy *)s_iptab.getValue ( &key );
@ -515,7 +524,7 @@ void handleRequest54 ( UdpSlot *udpSlot , long niceness ) {
LoadBucket *lb; LoadBucket *lb;
lb = (LoadBucket *)s_loadTable.getValueFromSlot(i); lb = (LoadBucket *)s_loadTable.getValueFromSlot(i);
// get the spider proxy this load point was for // get the spider proxy this load point was for
unsigned long long key = lb->m_proxyIp; unsigned long long key = (unsigned long)lb->m_proxyIp;
key <<= 16; key <<= 16;
key |= (unsigned short)lb->m_proxyPort; key |= (unsigned short)lb->m_proxyPort;
SpiderProxy *sp = (SpiderProxy *)s_iptab.getValue(&key); SpiderProxy *sp = (SpiderProxy *)s_iptab.getValue(&key);