gigablast/open-source-search-engine
1
0
Fork 0
mirror of https://github.com/gigablast/open-source-search-engine.git synced 2024-10-04 12:17:35 +03:00
Code Issues Projects Releases Wiki Activity

show more spider proxy stats in table

Browse Source
This commit is contained in:
mwells 2014-06-02 17:12:25 -07:00
parent 2582a487a5
commit abbc116442
2 changed files with 32 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
SpiderProxy.cpp
View File

@ -54,6 +54,8 @@ public:
long long m_timesUsed;
long m_lastBytesDownloaded;
// special things used by LoadBucket algo to determine which
// SpiderProxy to use to download from a particular IP
long m_countForThisIp;
@ -294,6 +296,10 @@ bool printSpiderProxyTable ( SafeBuf *sb ) {
// print "FAILED" in red if it failed to download
"<td><b>test url download time</b></td>"
"<td><b>last bytes downloaded</b></td>"
"<td><b>last test url error</b></td>"
"</tr>"
, TABLE_STYLE
@ -315,6 +321,10 @@ bool printSpiderProxyTable ( SafeBuf *sb ) {
sp->m_lastDownloadTestAttemptMS>0 )
bg = "ffa6a6";
// or a perm denied error (as opposed to a timeout above)
if ( sp->m_lastDownloadError )
bg = "ffa6a6";
// print it
sb->safePrintf (
"<tr bgcolor=#%s>"
@ -362,6 +372,14 @@ bool printSpiderProxyTable ( SafeBuf *sb ) {
"<font color=red>FAILED</font>"
"</td>");
sb->safePrintf("<td>%li</td>",sp->m_lastBytesDownloaded);
if ( sp->m_lastDownloadError )
sb->safePrintf("<td><font color=red>%s</font></td>",
mstrerror(sp->m_lastDownloadError));
else
sb->safePrintf("<td>none</td>");
sb->safePrintf("</tr>\n");
}
@ -403,6 +421,18 @@ void gotTestUrlReplyWrapper ( void *state , TcpSocket *s ) {
long long took = nowms - sp->m_lastDownloadTestAttemptMS;
sp->m_lastDownloadTookMS = (long)took;
HttpMime mime;
mime.set ( s->m_readBuf , s->m_readOffset , NULL );
// tiny proxy permission denied is 403
long status = mime.getHttpStatus();
if ( status == 403 ) {
log("sproxy: got bad http status from proxy: %li",status);
g_errno = EPERMDENIED;
}
sp->m_lastBytesDownloaded = s->m_readOffset;
// ETCPTIMEDOUT?
sp->m_lastDownloadError = g_errno;

3
gb.conf
View File

@ -221,7 +221,8 @@
# List of white space-separated spider proxy IPs. Put in IP:port format.
# Example <i>1.2.3.4:80 4.5.6.7:99</i>
<proxyIps><![CDATA[127.0.0.1:8888]]></>
<proxyIps><![CDATA[127.0.0.1:8888 64.71.190.137:8888
]]></>
# Download this url every minute through each proxy listed above to ensure
# they are up. Typically you should make this a URL you own so you do not