gigablast/open-source-search-engine
1
0
Fork 0
mirror of https://github.com/gigablast/open-source-search-engine.git synced 2024-10-04 12:17:35 +03:00
Code Issues Projects Releases Wiki Activity

emergency fixes for out of sockets core and

Browse Source 
get proxy request timing out causing spider to hang bug.
This commit is contained in:
Matt Wells 2014-10-09 07:20:04 -07:00
parent 487c6e0037
commit 8bb3545b71
3 changed files with 19 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
HttpServer.cpp
View File

@ -59,7 +59,7 @@ bool HttpServer::init ( short port,
//m_maxOpenSockets = 1000000; //m_maxOpenSockets = 1000000;
//well, not infinite //well, not infinite
m_maxOpenSockets = g_conf.m_httpMaxSockets; //m_maxOpenSockets = g_conf.m_httpMaxSockets;
m_uncompressedBytes = m_bytesDownloaded = 1; m_uncompressedBytes = m_bytesDownloaded = 1;
@ -79,7 +79,7 @@ bool HttpServer::init ( short port,
getMsgPiece , getMsgPiece ,
port , port ,
//&g_conf.m_httpMaxSockets ) ) return false; //&g_conf.m_httpMaxSockets ) ) return false;
&m_maxOpenSockets ) ) return false; &g_conf.m_httpMaxSockets ) ) return false;
//g_conf.m_httpMaxReadBufSize , //g_conf.m_httpMaxReadBufSize ,
//g_conf.m_httpMaxSendBufSize ) ) return false; //g_conf.m_httpMaxSendBufSize ) ) return false;
// set our secure TcpServer class // set our secure TcpServer class
@ -424,6 +424,8 @@ bool HttpServer::getDoc ( long ip,
// . callback will be called on completion of transaction // . callback will be called on completion of transaction
// . be sure to free "req/reqSize" in callback() somewhere // . be sure to free "req/reqSize" in callback() somewhere
// MDW: THIS IS RETURNING TRUE SOMEHOW w/o setting g_errno
if ( ! tcp->sendMsg ( ip , if ( ! tcp->sendMsg ( ip ,
port , port ,
req , req ,
@ -440,6 +442,7 @@ bool HttpServer::getDoc ( long ip,
states[n] = NULL; states[n] = NULL;
callbacks[n] = NULL; callbacks[n] = NULL;
s_numOutgoingSockets--; s_numOutgoingSockets--;
log("http: sendmsg returned true!: %s",mstrerror(g_errno));
return true; return true;
} }

6
Msg13.cpp
View File

@ -787,7 +787,7 @@ void downloadTheDocForReals2 ( Msg13Request *r ) {
NULL , NULL ,
r , // state data r , // state data
gotProxyHostReplyWrapper , gotProxyHostReplyWrapper ,
10 )){// 10 sec timeout 9999999 )){// 9999999 sec timeout
// sanity check // sanity check
if ( ! g_errno ) { char *xx=NULL;*xx=0; } if ( ! g_errno ) { char *xx=NULL;*xx=0; }
// report it // report it
@ -809,7 +809,7 @@ void gotProxyHostReplyWrapper ( void *state , UdpSlot *slot ) {
if ( g_errno ) { if ( g_errno ) {
// note it // note it
log("sproxy: got proxy request error: %s",mstrerror(g_errno)); log("sproxy: got proxy request error: %s",mstrerror(g_errno));
g_udpServer.sendErrorReply(slot,g_errno); g_udpServer.sendErrorReply(r->m_udpSlot,g_errno);
return; return;
} }
// //
@ -823,7 +823,7 @@ void gotProxyHostReplyWrapper ( void *state , UdpSlot *slot ) {
if ( replySize != sizeof(ProxyReply) ) { if ( replySize != sizeof(ProxyReply) ) {
log("sproxy: bad 54 reply size of %li != %li", log("sproxy: bad 54 reply size of %li != %li",
replySize,(long)sizeof(ProxyReply)); replySize,(long)sizeof(ProxyReply));
g_udpServer.sendErrorReply(slot,g_errno); g_udpServer.sendErrorReply(r->m_udpSlot,g_errno);
return; return;
} }

13
TcpServer.cpp
View File

@ -594,6 +594,7 @@ bool TcpServer::sendMsg ( long ip ,
// . adds socket to array for us and sets the fd non-blocking, etc. // . adds socket to array for us and sets the fd non-blocking, etc.
s = getNewSocket ( ); s = getNewSocket ( );
// return true if s is NULL and g_errno was set by getNewSocket() // return true if s is NULL and g_errno was set by getNewSocket()
// might set g_errno to EOUTOFSOCKETS
if ( ! s ) { mfree ( sendBuf , sendBufSize,"TcpServer"); return true; } if ( ! s ) { mfree ( sendBuf , sendBufSize,"TcpServer"); return true; }
// set up the new TcpSocket for connecting // set up the new TcpSocket for connecting
s->m_state = state; s->m_state = state;
@ -764,7 +765,7 @@ TcpSocket *TcpServer::getNewSocket ( ) {
long now = getTimeLocal(); long now = getTimeLocal();
static long s_last = 0; static long s_last = 0;
static long s_count = 0; static long s_count = 0;
if ( now - s_last < 5 ) if ( now - s_last < 5 && s_last )
s_count++; s_count++;
else { else {
log("tcp: Out of sockets. Max sockets = %li. " log("tcp: Out of sockets. Max sockets = %li. "
@ -779,6 +780,8 @@ TcpSocket *TcpServer::getNewSocket ( ) {
// send email alert // send email alert
g_pingServer.sendEmailMsg ( &s_lastTime , g_pingServer.sendEmailMsg ( &s_lastTime ,
"out of sockets on https"); "out of sockets on https");
// in case sendEmailMsg resets g_errno somehow
g_errno = EOUTOFSOCKETS;
return NULL; return NULL;
} }
@ -877,7 +880,7 @@ TcpSocket *TcpServer::wrapSocket ( int sd , long niceness , bool isIncoming ) {
long now = getTimeLocal(); long now = getTimeLocal();
static long s_last = 0; static long s_last = 0;
static long s_count = 0; static long s_count = 0;
if ( now - s_last < 5 ) if ( now - s_last < 5 && s_last )
s_count++; s_count++;
else { else {
log("tcp: Out of sockets. Max sockets = %li. " log("tcp: Out of sockets. Max sockets = %li. "
@ -892,6 +895,8 @@ TcpSocket *TcpServer::wrapSocket ( int sd , long niceness , bool isIncoming ) {
// send email alert // send email alert
g_pingServer.sendEmailMsg ( &s_lastTime , g_pingServer.sendEmailMsg ( &s_lastTime ,
"out of sockets on https"); "out of sockets on https");
// in case sendEmailMsg resets g_errno somehow
g_errno = EOUTOFSOCKETS;
return NULL; return NULL;
} }
// sanity check // sanity check
@ -903,6 +908,8 @@ TcpSocket *TcpServer::wrapSocket ( int sd , long niceness , bool isIncoming ) {
// send email alert // send email alert
g_pingServer.sendEmailMsg ( &s_lastTime , g_pingServer.sendEmailMsg ( &s_lastTime ,
"out of sockets on https2"); "out of sockets on https2");
// in case sendEmailMsg resets g_errno somehow
g_errno = EOUTOFSOCKETS;
return NULL; return NULL;
} }
// alloc a new TcpSocket // alloc a new TcpSocket
@ -922,6 +929,8 @@ TcpSocket *TcpServer::wrapSocket ( int sd , long niceness , bool isIncoming ) {
g_pingServer.sendEmailMsg ( &s_lastTime , g_pingServer.sendEmailMsg ( &s_lastTime ,
"out of sockets on https3"); "out of sockets on https3");
//sleep(10000); //sleep(10000);
// in case sendEmailMsg resets g_errno somehow
g_errno = EOUTOFSOCKETS;
return NULL; return NULL;
} }
// save this i guess // save this i guess