gigablast/open-source-search-engine
1
0
Fork 0
mirror of https://github.com/gigablast/open-source-search-engine.git synced 2024-10-04 20:27:43 +03:00
Code Issues Projects Releases Wiki Activity

fix rdbcache corruption from -O2 compile bug.

Browse Source 
fix too many spiders per ip bug!
This commit is contained in:
Matt Wells 2014-02-05 16:58:21 -08:00
parent 951e9d5068
commit 5c8b9af1d3
3 changed files with 16 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Makefile
View File

@ -326,8 +326,9 @@ Rdb.o:
RdbBase.o:
$(CC) $(DEFS) $(CPPFLAGS) -O2 -c $*.cpp
RdbCache.o:
$(CC) $(DEFS) $(CPPFLAGS) -O2 -c $*.cpp
# RdbCache.cpp gets "corrupted" with -O2... like RdbTree.cpp
#RdbCache.o:
# $(CC) $(DEFS) $(CPPFLAGS) -O2 -c $*.cpp
# fast dictionary generation and spelling recommendations
#Speller.o:

16
Spider.cpp
View File

@ -6019,10 +6019,18 @@ bool SpiderLoop::gotDoledbList2 ( ) {
}
// how many spiders out for this ip now?
long outPerIp = g_spiderLoop.getNumSpidersOutPerIp ( sreq->m_firstIp ,
m_collnum );
if ( outPerIp >= maxSpidersOutPerIp ) goto hitMax;
// . how many spiders out for this ip now?
// . TODO: count locks in case twin is spidering... but it did not seem
// to work right for some reason
long ipOut = 0;
for ( long i = 0 ; i <= m_maxUsed ; i++ ) {
// get it
XmlDoc *xd = m_docs[i];
if ( ! xd ) continue;
if ( ! xd->m_firstIpValid ) continue;
if ( xd->m_firstIp == sreq->m_firstIp ) ipOut++;
}
if ( ipOut >= maxSpidersOutPerIp ) goto hitMax;
// sometimes we have it locked, but is still in doledb i guess.

2
gb.conf
View File

@ -51,7 +51,7 @@
<readOnlyMode>0</>
# Controls all spidering for all collections
<spideringEnabled>1</>
<spideringEnabled>0</>
# What is the maximum number of web pages the spider is allowed to download
# simultaneously for ALL collections PER HOST?