From ec5c38bab5b4019125758050cc6bb23970170145 Mon Sep 17 00:00:00 2001
From: Matt Wells <mwells2@gigablast.com>
Date: Tue, 24 Nov 2015 08:51:18 -0800
Subject: [PATCH] fix urgent merge mode bug some more? limit spiders to 5 per
 custom crawl coll per shard.

---
 Collectiondb.cpp |  3 +++
 RdbBase.cpp      | 12 +++++++++---
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/Collectiondb.cpp b/Collectiondb.cpp
index 548ed53d..63d0c025 100644
--- a/Collectiondb.cpp
+++ b/Collectiondb.cpp
@@ -333,6 +333,9 @@ bool Collectiondb::addExistingColl ( char *coll, collnum_t collnum ) {
 	if ( cr->m_isCustomCrawl ) {
 		cr->m_getLinkInfo = false;
 		cr->m_computeSiteNumInlinks = false;
+		// limit each shard to 5 spiders per collection to prevent
+		// ppl from spidering the web and hogging up resources
+		cr->m_maxNumSpiders = 5;
 	}
 
 	// we need to compile the regular expressions or update the url
diff --git a/RdbBase.cpp b/RdbBase.cpp
index 04c174c7..2512fa51 100644
--- a/RdbBase.cpp
+++ b/RdbBase.cpp
@@ -1607,8 +1607,8 @@ bool RdbBase::attemptMerge ( int32_t niceness, bool forceMergeAll, bool doLog ,
 	if ( ! m_mergeUrgent && numFiles - 14 >= m_minToMerge ) {
 		m_mergeUrgent = true;
 		if ( doLog ) 
-		log(LOG_INFO,"merge: Entering urgent merge mode for %s.",
-		    m_dbname);
+		log(LOG_INFO,"merge: Entering urgent merge mode for %s "
+		    "coll=%s.", m_dbname,m_coll);
 		g_numUrgentMerges++;
 	}
 
@@ -1811,7 +1811,8 @@ void RdbBase::gotTokenForMerge ( ) {
 		m_mergeUrgent = true;
 		if ( m_doLog )
 		log(LOG_INFO,
-		    "merge: Entering urgent merge mode for %s.", m_dbname);
+		    "merge: Entering urgent merge mode (2) for %s coll=%s.", 
+		    m_dbname,m_coll);
 		g_numUrgentMerges++;
 	}
 	// tfndb has his own merge class since titledb merges write tfndb recs
@@ -1894,6 +1895,11 @@ void RdbBase::gotTokenForMerge ( ) {
 			log(LOG_LOGIC,"merge: attemptMerge: Resuming. bad "
 			    "engineer for %s coll=%s",m_dbname,m_coll);
 			//g_msg35.releaseToken();
+			if ( m_mergeUrgent ) {
+				log("merge: leaving urgent merge mode");
+				g_numUrgentMerges--;
+				m_mergeUrgent = false;
+			}
 			return false;
 		}
 		// make a log note