fix pagination for &stream=1

Collectiondb.cpp

@@ -1616,8 +1616,9 @@ bool CollectionRec::load ( char *coll , long i ) {
 	m_collLen = gbstrlen ( coll );
 	strcpy ( m_coll , coll );
 
-	log(LOG_INFO,"db: loading conf for collection %s (%li)",coll,
-	    (long)m_collnum);
+	if ( ! g_conf.m_doingCommandLine )
+		log(LOG_INFO,"db: loading conf for collection %s (%li)",coll,
+		    (long)m_collnum);
 
 	// collection name HACK for backwards compatibility
 	//if ( strcmp ( coll , "main" ) == 0 ) {
@@ -1657,11 +1658,12 @@ bool CollectionRec::load ( char *coll , long i ) {
 		// it is binary now
 		memcpy ( &m_localCrawlInfo , sb.getBufStart(),sb.length() );
 
-	
-	log("coll: loaded %s (%li) local hasurlsready=%li",
-	    m_coll,
-	    (long)m_collnum,
-	    (long)m_localCrawlInfo.m_hasUrlsReadyToSpider);
+
+	if ( ! g_conf.m_doingCommandLine )
+		log("coll: loaded %s (%li) local hasurlsready=%li",
+		    m_coll,
+		    (long)m_collnum,
+		    (long)m_localCrawlInfo.m_hasUrlsReadyToSpider);
 
 
 	// we introduced the this round counts, so don't start them at 0!!
@@ -1704,10 +1706,11 @@ bool CollectionRec::load ( char *coll , long i ) {
 		// it is binary now
 		memcpy ( &m_globalCrawlInfo , sb.getBufStart(),sb.length() );
 
-	log("coll: loaded %s (%li) global hasurlsready=%li",
-	    m_coll,
-	    (long)m_collnum,
-	    (long)m_globalCrawlInfo.m_hasUrlsReadyToSpider);
+	if ( ! g_conf.m_doingCommandLine )
+		log("coll: loaded %s (%li) global hasurlsready=%li",
+		    m_coll,
+		    (long)m_collnum,
+		    (long)m_globalCrawlInfo.m_hasUrlsReadyToSpider);
 	
 
 	////////////
@@ -2354,8 +2357,9 @@ void nukeDoledb ( collnum_t collnum );
 // . it is also called on load of the collection at startup
 bool CollectionRec::rebuildUrlFilters ( ) {
 
-	log("coll: rebuilding url filters for %s ufp=%li",m_coll,
-	    (long)m_urlFiltersProfile);
+	if ( ! g_conf.m_doingCommandLine )
+		log("coll: rebuilding url filters for %s ufp=%li",m_coll,
+		    (long)m_urlFiltersProfile);
 
 	// if not a custom crawl, and no expressions, add a default one
 	//if ( m_numRegExs == 0 && ! m_isCustomCrawl ) {

Conf.cpp

@@ -11,6 +11,7 @@ Conf g_conf;
 
 Conf::Conf ( ) {
 	m_save = true;
+	m_doingCommandLine = false;
 }
 
 // . does this requester have ROOT admin privledges???

Conf.h

@@ -196,6 +196,9 @@ class Conf {
 	//bool m_useDiffbot;
 
 	//bool  m_indexEventsOnly;
+	// are we doing a command line thing like 'gb 0 dump s ....' in
+	// which case we do not want to log certain things
+	bool m_doingCommandLine;
 
 	// linkdb for storing linking relations
 	long  m_linkdbMaxTreeMem;

Msg40.cpp

@@ -99,6 +99,7 @@ Msg40::Msg40() {
 	m_sendsOut      = 0;
 	m_sendsIn       = 0;
 	m_printi        = 0;
+	m_numDisplayed  = 0;
 	m_lastChunk     = false;
 	//m_numGigabitInfos = 0;
 }
@@ -1675,6 +1676,15 @@ bool Msg40::gotSummary ( ) {
 			    mstrerror(g_errno));
 		}
 
+		// assume we show this to the user
+		m_numDisplayed++;
+
+		// do not print it if before the &s=X start position though
+		if ( m_si && m_numDisplayed <= m_si->m_firstResultNum ){
+			log("msg40: hiding #%li (%lu)",
+			    m_printi,mr->m_contentHash32);
+			continue;
+		}
 
 		log("msg40: printing #%li (%lu)",m_printi,mr->m_contentHash32);
 

Msg40.h

@@ -264,6 +264,7 @@ class Msg40 {
 	long m_sendsOut      ;
 	long m_sendsIn       ;
 	long m_printi        ;
+	long m_numDisplayed  ;
 	long m_socketHadError;
 
 

main.cpp

@@ -2479,6 +2479,9 @@ int main2 ( int argc , char *argv[] ) {
 		long long termId  = -1;
 		char *coll = "";
 
+		// so we do not log every collection coll.conf we load
+		g_conf.m_doingCommandLine = true;
+
 		// we have to init collection db because we need to know if 
 		// the collnum is legit or not in the tree
 		if ( ! g_collectiondb.loadAllCollRecs()   ) {