added jobCreationTimeUTC and jobCompletionTimeUTC

to json api

Collectiondb.cpp

@@ -467,13 +467,13 @@ bool Collectiondb::addNewColl ( char *coll ,
 		cr->m_collectiveRespiderFrequency = 0.0;
 		//cr->m_restrictDomain = true;
 		// reset the crawl stats
-		// . this will core if a host was dead and then when it came
-		//   back up host #0's parms.cpp told it to add a new coll
-		cr->m_diffbotCrawlStartTime=
-			gettimeofdayInMillisecondsGlobalNoCore();
-		cr->m_diffbotCrawlEndTime   = 0LL;
 	}
 
+	// . this will core if a host was dead and then when it came
+	//   back up host #0's parms.cpp told it to add a new coll
+	cr->m_diffbotCrawlStartTime = getTimeGlobalNoCore();
+	cr->m_diffbotCrawlEndTime   = 0;
+	
 	// . just the basics on these for now
 	// . if certain parms are changed then the url filters
 	//   must be rebuilt, as well as possibly the waiting tree!!!

Collectiondb.h

@@ -659,8 +659,9 @@ class CollectionRec {
 	long long m_maxToProcess;
 	long      m_maxCrawlRounds;
 
-	long long m_diffbotCrawlStartTime;
-	long long m_diffbotCrawlEndTime;
+	// in seconds now
+	long m_diffbotCrawlStartTime;
+	long m_diffbotCrawlEndTime;
 
 	// for testing their regexes etc...
 	//char m_isDiffbotTestCrawl;

PageCrawlBot.cpp

@@ -2355,10 +2355,13 @@ bool printCrawlDetailsInJson ( SafeBuf *sb , CollectionRec *cx ) {
 		//nomen = "job";
 	}
 
-
 	sb->safePrintf("\n\n{"
 		      "\"name\":\"%s\",\n"
 		      "\"type\":\"%s\",\n"
+
+		       "\"jobCreationTimeUTC\":%li,\n"
+		       "\"jobCompletionTimeUTC\":%li,\n"
+
 		      //"\"alias\":\"%s\",\n"
 		      //"\"crawlingEnabled\":%li,\n"
 		      "\"jobStatus\":{" // nomen = jobStatus / crawlStatus
@@ -2384,6 +2387,11 @@ bool printCrawlDetailsInJson ( SafeBuf *sb , CollectionRec *cx ) {
 		      //,cx->m_coll
 		      , cx->m_diffbotCrawlName.getBufStart()
 		      , crawlTypeStr
+
+		       , cx->m_diffbotCrawlStartTime
+		       // this is 0 if not over yet
+		       , cx->m_diffbotCrawlEndTime
+
 		      //, alias
 		      //, (long)cx->m_spideringEnabled
 		      , crawlStatus

Parms.cpp

@@ -8518,6 +8518,26 @@ void Parms::init ( ) {
 	m->m_flags = PF_DIFFBOT;
 	m++;
 
+	m->m_cgi   = "dbcrawlstarttime";
+	m->m_xml   = "diffbotCrawlStartTime";
+	m->m_off   = (char *)&cr.m_diffbotCrawlStartTime - x;
+	m->m_type  = TYPE_LONG;
+	m->m_page  = PAGE_NONE;
+	m->m_obj   = OBJ_COLL;
+	m->m_def   = "0";
+	m->m_flags = PF_DIFFBOT;
+	m++;
+
+	m->m_cgi   = "dbcrawlendtime";
+	m->m_xml   = "diffbotCrawlEndTime";
+	m->m_off   = (char *)&cr.m_diffbotCrawlEndTime - x;
+	m->m_type  = TYPE_LONG;
+	m->m_page  = PAGE_NONE;
+	m->m_obj   = OBJ_COLL;
+	m->m_def   = "0";
+	m->m_flags = PF_DIFFBOT;
+	m++;
+
 	m->m_cgi   = "dbcrawlname";
 	m->m_xml   = "diffbotCrawlName";
 	m->m_off   = (char *)&cr.m_diffbotCrawlName - x;

Spider.cpp

@@ -12252,6 +12252,8 @@ void handleRequestc1 ( UdpSlot *slot , long niceness ) {
 			ci->m_hasUrlsReadyToSpider = 0;
 			// save that!
 			cr->m_needsSave = true;
+			// set the time that this happens
+			cr->m_diffbotCrawlEndTime = getTimeGlobalNoCore();
 		}
 		
 		// save it

XmlDoc.cpp

@@ -2113,8 +2113,8 @@ bool XmlDoc::indexDoc ( ) {
 		// need to save collection rec now during auto save
 		cr->m_needsSave = true;
 		// update this just in case we are the last url crawled
-		long long now = gettimeofdayInMillisecondsGlobal();
-		cr->m_diffbotCrawlEndTime = now;
+		//long long now = gettimeofdayInMillisecondsGlobal();
+		//cr->m_diffbotCrawlEndTime = now;
 	}