checkpoint for new parm logic

to allowing syncing with newly added or deleted
collections even if a host was dead when collection
was added/deleted. also added parm change request
queueing.

CollectionRec.h

@@ -1015,6 +1015,8 @@ class CollectionRec {
 	key128_t m_timedbEndKey;
 	RdbList  m_timedbList;
 
+	// used by Parms.cpp
+	char m_hackFlag;
 
 	//long m_numEventsOnHost;
 

Makefile

@@ -36,7 +36,7 @@ OBJS =  Tfndb.o UdpSlot.o \
 	Stats.o BigFile.o Msg17.o \
 	Speller.o DiskPageCache.o \
 	PingServer.o StopWords.o TopTree.o \
-	Parms.o Pages.o Msg28.o \
+	Parms.o Pages.o \
 	Unicode.o iana_charset.o Iso8859.o \
 	SearchInput.o \
 	Categories.o Msg2a.o PageCatdb.o PageDirectory.o \
@@ -49,7 +49,7 @@ OBJS =  Tfndb.o UdpSlot.o \
 	PageSpam.o Proxy.o PageThreads.o Linkdb.o \
 	PageNetTest.o \
 	matches2.o LanguageIdentifier.o \
-	Language.o Repair.o Process.o Msg3e.o \
+	Language.o Repair.o Process.o \
 	Abbreviations.o \
 	RequestTable.o TuringTest.o Msg51.o geo_ip_table.o \
 	Msg40.o Msg4.o \

Msg4.cpp

@@ -1212,6 +1212,7 @@ bool addMetaList ( char *p , UdpSlot *slot ) {
 		    "rdbId=%li. not in repair mode. dropping.",(long)rdbId);
 		char *xx=NULL;*xx=0;
 		// drop it for now!!
+		p += recSize;
 		if ( p < pend ) goto loop;
 		// all done
 		return true;

Msg4.h

@@ -54,6 +54,7 @@ class Msg4 {
 			   char  rdbId = -1               );
 
 	// this one is faster...
+	// returns false if blocked
 	bool addMetaList ( char      *metaList                 ,
 			   long       metaListSize             ,
 			   collnum_t  collnum                  ,

PageCrawlBot.cpp

@@ -2354,6 +2354,10 @@ bool sendPageCrawlbot ( TcpSocket *socket , HttpRequest *hr ) {
 	//   pg->m_function(). even though maxtocrawl is on "PAGE_NONE" 
 	//   hopefully it will still be set
 	// . but we should take care of add/del/reset coll here.
+	// . i guess this will be handled by the new parm syncing logic
+	//   which deals with add/del coll requests
+
+	/*
 	if ( cast == 0 ) {
 		// add a new collection by default
 		if ( ! cr && name && name[0] ) 
@@ -2427,6 +2431,7 @@ bool sendPageCrawlbot ( TcpSocket *socket , HttpRequest *hr ) {
 		// this is a cast, so just return simple response
 		return g_httpServer.sendDynamicPage (socket,"OK",2);
 	}
+	*/
 
 	/////////
 	//
@@ -4431,6 +4436,10 @@ bool resetUrlFilters ( CollectionRec *cr ) {
 	return true;
 }
 
+/*
+  THIS IS NOW AUTOMATIC from new Parms.cpp broadcast logic
+*/
+
 
 bool setSpiderParmsFromHtmlRequest ( TcpSocket *socket ,
 				     HttpRequest *hr , 

Pages.cpp

@@ -17,6 +17,23 @@ Pages g_pages;
 // error message thingy used by HttpServer.cpp for logging purposes
 char *g_msg;
 
+/*
+class WebPage {
+ public:
+	char  m_pageNum;  // see enum array below for this
+	char *m_filename;
+	long  m_flen;
+	char *m_name;     // for printing the links to the pages in admin sect.
+	bool  m_cast;     // broadcast input to all hosts?
+	bool  m_usePost;  // use a POST request/reply instead of GET?
+	                  // used because GET's input is limited to a few k.
+	//char  m_perm;     // permissions, see USER_* #define's below
+	char *m_desc; // page description
+	bool (* m_function)(TcpSocket *s , HttpRequest *r);
+	long  m_niceness;
+};
+*/
+
 // . list of all dynamic pages, their path names, permissions and callback
 //   functions that generate that page
 // . IMPORTANT: these must be in the same order as the PAGE_* enum in Pages.h
@@ -397,6 +414,25 @@ long Pages::getDynamicPageNumber ( HttpRequest *r ) {
 	return -1;
 }
 
+void doneFlushingParms ( void *state ) {
+	TcpSocket *sock = (TcpSocket *)state;
+	// set another http request again
+	HttpRequest r;
+	bool status = r.set ( sock->m_readBuf , sock->m_readOffset , sock ) ;
+	// we stored the page # below
+	WebPage *pg = &s_pages[sock->m_pageNum];
+	// call the page specifc function which will send data back on socket
+	pg->m_function ( sock , &r );
+}
+
+void doneBroadcastingParms ( void *state ) {
+	TcpSocket *sock = (TcpSocket *)state;
+	// free this mem
+	sock->m_handyBuf.purge();
+	// but flush msg4 now so parms are realized on all machines
+	flushMsg4Buffers ( state , doneFlushingParms );
+}
+
 // . returns false if blocked, true otherwise
 // . send an error page on error
 bool Pages::sendDynamicReply ( TcpSocket *s , HttpRequest *r , long page ) {
@@ -538,6 +574,72 @@ bool Pages::sendDynamicReply ( TcpSocket *s , HttpRequest *r , long page ) {
 	// 		return g_httpServer.sendErrorReply(s,505,mstrerror(g_errno));
 	// 	}
 
+
+	// get safebuf stored in TcpSocket class
+	SafeBuf *parmList = &s->m_handyBuf;
+
+	// chuck this in there
+	s->m_pageNum = page;
+
+	////////
+	//
+	// the new way to set and distribute parm settings
+	//
+	////////
+	
+	// . convert http request to list of parmdb records
+	// . will only add parm recs we have permission to modify
+	if ( ! g_parms.convertHttpRequestToParmList ( r , parmList ) )
+		return g_httpServer.sendErrorReply(s,505,mstrerror(g_errno));
+		
+
+	// . add parmList using Parms::m_msg4 to all hosts!
+	// . returns true and sets g_errno on error
+	// . returns false if would block
+	// . so then doneBroadcastingParms() is called when all hosts
+	//   have received the updated parms, unless a host is dead,
+	//   in which case he should sync up when he comes back up
+	if ( ! g_parms.broadcastParmList ( parmList , 
+					   s , // state is socket i guess
+					   doneBroadcastingParms ) )
+		// this would block, so return false
+		return false;
+
+	// free the mem if we didn't block
+	s->m_handyBuf.purge();
+
+	// on error from broadcast, bail here
+	if ( g_errno )
+		return g_httpServer.sendErrorReply(s,505,mstrerror(g_errno));
+
+	// if this is a save & exit request we must log it here because it
+	// will never return in order to log it in HttpServer.cpp
+	// TODO: make this a function we can call.
+	if ( g_conf.m_logHttpRequests && page == PAGE_MASTER ) { 
+		//&& pg->m_function==CommandSaveAndExit ) {
+		// get time format: 7/23/1971 10:45:32
+		time_t tt ;//= getTimeGlobal();
+		if ( isClockInSync() ) tt = getTimeGlobal();
+		else                   tt = getTimeLocal();
+		struct tm *timeStruct = localtime ( &tt );
+		char buf[64];
+		strftime ( buf , 100 , "%b %d %T", timeStruct);
+		// what url refered user to this one?
+		char *ref = r->getReferer();
+		// skip over http:// in the referer
+		if ( strncasecmp ( ref , "http://" , 7 ) == 0 ) ref += 7;
+		// save ip in case "s" gets destroyed
+		long ip = s->m_ip;
+		logf (LOG_INFO,"http: %s %s %s %s %s",
+		      buf,iptoa(ip),r->getRequest(),ref,
+		      r->getUserAgent());
+	}
+
+	// if we did not block... maybe there were no parms to broadcast
+	return pg->m_function ( s , r );
+
+	/*
+
 	// broadcast request to ALL hosts if we should
 	// should this request be broadcasted?
 	long cast = r->getLong("cast",-1) ;
@@ -551,7 +653,7 @@ bool Pages::sendDynamicReply ( TcpSocket *s , HttpRequest *r , long page ) {
 		cast = 0;
 		if ( page == PAGE_CRAWLBOT ) cast = 1;
 	}
-
+	*/
 	// proxy can only handle certain pages. it has logic in Proxy.cpp
 	// to use the 0xfd msg type to forward certain page requests to 
 	// host #0, like 
@@ -586,6 +688,7 @@ bool Pages::sendDynamicReply ( TcpSocket *s , HttpRequest *r , long page ) {
 	//if ( g_proxy.isProxyRunning() && 
 	//   (g_conf.isMasterAdmin( s, r ) || g_hostdb.getProxyByIp(s->m_ip)) )
 	//	cast = false;
+	/*
 	if ( g_proxy.isProxy () ) cast = 0;
 	// this only returns true on error. uses msg28 to send the http request
 	// verbatim to all hosts in network, using tcpserver. the spawned msg28
@@ -615,6 +718,7 @@ bool Pages::sendDynamicReply ( TcpSocket *s , HttpRequest *r , long page ) {
 							  "collection given.");
 		}
 	}
+
 	// if this is a save & exit request we must log it here because it
 	// will never return in order to log it in HttpServer.cpp
 	if ( g_conf.m_logHttpRequests && page == PAGE_MASTER ) { 
@@ -642,6 +746,7 @@ bool Pages::sendDynamicReply ( TcpSocket *s , HttpRequest *r , long page ) {
 	// . now, so it can be responsible for calling pg->m_function
 	//if ( userType > USER_PUBLIC ) {
 	// check if user has public page access 
+	
 	if ( isLocal ) { //g_users.hasPermission( r, page , s )){
 		// . this will set various parms
 		// . we know the request came from a host in the cluster
@@ -673,8 +778,10 @@ bool Pages::sendDynamicReply ( TcpSocket *s , HttpRequest *r , long page ) {
 	// . sets g_errno on error i think
 	// . false means not called from msg28
 	return pg->m_function ( s , r );
+	*/
 }
 
+/*
 #include "Msg28.h"
 static Msg28        s_msg28;
 static TcpSocket   *s_s;
@@ -725,6 +832,7 @@ void doneWrapper ( void *state ) {
 	// . this must call g_httpServer.sendDynamicReply() eventually
 	s_pages[s_page].m_function ( s_s , &s_r );
 }
+*/
 
 // certain pages are automatically generated by the g_parms class
 // because they are menus of configurable parameters for either g_conf

Parms.h

@@ -7,8 +7,13 @@
 #ifndef _PARMS_H_
 #define _PARMS_H_
 
+#include "Rdb.h"
+
 //#include "CollectionRec.h"
 
+void handleRequest3e ( UdpSlot *slot , long niceness ) ;
+void handleRequest3f ( UdpSlot *slot , long niceness ) ;
+
 // special priorities for the priority drop down 
 // in the url filters table
 enum {
@@ -101,6 +106,10 @@ class Parm {
 	// this is 1 if NOT an array (i.e. array of only one parm).
 	// in such cases a "count" is NOT stored before the parm in 
 	// CollectionRec.h or Conf.h.
+	bool isArray() { return (m_max>1); };
+
+	long getNumInArray() ;
+
 	long  m_max;   // max elements in the array
 	// if array is fixed size, how many elements in it?
 	// this is 0 if not a FIXED size array.
@@ -120,8 +129,9 @@ class Parm {
 	char *m_icon;
 	char *m_qterm;
 	long  m_parmNum; // slot # in the m_parms[] array that we are
-	bool (*m_func)(TcpSocket *s , HttpRequest *r,
-		       bool (*cb)(TcpSocket *s , HttpRequest *r));
+	//bool (*m_func)(TcpSocket *s , HttpRequest *r,
+	//	       bool (*cb)(TcpSocket *s , HttpRequest *r));
+	bool (*m_func)(char *parmRec);
 	long  m_plen;  // offset of length for TYPE_STRINGS (m_htmlHeadLen...)
 	char  m_group; // start of a new group of controls?
 	// m_priv = 1 means gigablast's software license clients cannot see
@@ -149,6 +159,7 @@ class Parm {
 	char  m_sprpp; // propagate the cgi variable to other pages via POST?
 	bool  m_sync;  // this parm should be synced
 	long  m_hash;  // hash of "title"
+	long  m_cgiHash; // hash of m_cgi
 
 	bool   getValueAsBool   ( class SearchInput *si ) ;
 	long   getValueAsLong   ( class SearchInput *si ) ;
@@ -239,9 +250,8 @@ class Parms {
 			      bool (*callback)(TcpSocket *s , HttpRequest *r),
 			      class CollectionRec *newcr = NULL );
 	
-	void insertParm ( long i , long an , char *THIS ) ;
-
-	void removeParm ( long i , long an , char *THIS ) ;
+	//void insertParm ( long i , long an , char *THIS ) ;
+	//void removeParm ( long i , long an , char *THIS ) ;
 
 	void setParm ( char *THIS, Parm *m, long mm, long j, char *s,
 		       bool isHtmlEncoded , bool fromRequest ) ;
@@ -270,16 +280,63 @@ class Parms {
 	unsigned long calcChecksum();
 
 	// get size of serialized parms
-	long getStoredSize();
+	//long getStoredSize();
 	// . serialized to buf
 	// . if buf is NULL, just calcs size
-	bool serialize( char *buf, long *bufSize );
-	void deserialize( char *buf );
+	//bool serialize( char *buf, long *bufSize );
+	//void deserialize( char *buf );
 
 	void overlapTest ( char step ) ;
 
+
+	/////
+	//
+	// parms now in parmdb
+	//
+	/////
+
+	// all parm recs need to be in the tree
+	//Rdb m_rdb;
+
+	//
+	// new functions
+	//
+
+	bool addNewParmToList1 ( SafeBuf *parmList ,
+				 collnum_t collnum ,
+				 char *parmValString ,
+				 long  occNum ,
+				 char *parmName ) ;
+	bool addNewParmToList2 ( SafeBuf *parmList ,
+				 collnum_t collnum , 
+				 char *parmValString ,
+				 long occNum ,
+				 Parm *m ) ;
+	bool addCurrentParmToList1 ( SafeBuf *parmList ,
+				     CollectionRec *cr , 
+				     char *parmName ) ;
+	bool addCurrentParmToList2 ( SafeBuf *parmList ,
+				     collnum_t collnum , 
+				     long occNum ,
+				     Parm *m ) ;
+	bool convertHttpRequestToParmList (HttpRequest *hr,SafeBuf *parmList);
+	Parm *getParmFast2 ( long cgiHash32 ) ;
+	Parm *getParmFast1 ( char *cgi ) ;
+	bool broadcastParmList ( SafeBuf *parmList ) ;
+	bool doParmSendingLoop ( ) ;
+	bool syncParmsWithHost0 ( ) ;
+	bool makeSyncHashList ( SafeBuf *hashList ) ;
+	long getNumInArray ( collnum_t collnum ) ;
+	bool addAllParmsToList ( SafeBuf *parmList, collnum_t collnum ) ;
+	bool updateParm ( char *rec ) ;
+
+	//
+	// end new functions
+	//
+
+
 	bool m_isDefaultLoaded;
-	
+
 	Page m_pages [ 50 ];
 	long m_numPages;
 	
@@ -289,7 +346,8 @@ class Parms {
 	// just those Parms that have a m_sparm of 1
 	Parm *m_searchParms [ MAX_PARMS ];
 	long m_numSearchParms;
-	
+
+	/*
  private:
 	// these return true if overflow
 	bool serializeConfParm( Parm *m, long i, char **p, char *end, 
@@ -305,7 +363,8 @@ class Parms {
 				   bool *confChgd );
 	void deserializeCollParm( class CollectionRec *cr,
 				  Parm *m, SerParm *sp, char **p );
-	
+	*/
+
 	// for holding default.conf file for collection recs for OBJ_COLL
 	char m_buf [ MAX_XML_CONF ];
 

Process.cpp

@@ -1061,10 +1061,10 @@ void processSleepWrapper ( int fd , void *state ) {
 
 	// need to have a clock unified with host #0. i guess proxy
 	// does not sync with host #0 though
-	if ( ! isClockInSync() && ! g_hostdb.m_myHost->m_isProxy ) return;
+	//if ( ! isClockInSync() && ! g_hostdb.m_myHost->m_isProxy ) return;
 
 	// get time the day started
-	long now = getTimeGlobal();
+	long now = getTimeLocal();//GlobalNoCore();
 	// set this for the first time
 	if ( g_process.m_lastSaveTime == 0 )
 		g_process.m_lastSaveTime = now;

Process.h

@@ -10,7 +10,7 @@
 
 
 #include "HttpRequest.h"
-#include "Msg28.h"
+//#include "Msg28.h"
 
 class Process {
 
@@ -78,7 +78,7 @@ class Process {
 	bool        m_powerIsOn;
 	long long   m_powerOffTime;
 	HttpRequest m_r;
-	Msg28       m_msg28;
+	//Msg28       m_msg28;
 	bool        m_exiting;
 	bool        m_calledSave;
 

Rdb.cpp

@@ -2519,6 +2519,7 @@ Rdb *getRdbFromId ( uint8_t rdbId ) {
 		s_table9 [ RDB_STATSDB   ] = g_statsdb.getRdb();
 		s_table9 [ RDB_REVDB     ] = g_revdb.getRdb();
 		//s_table9 [ RDB_FAKEDB    ] = NULL;
+		s_table9 [ RDB_PARMDB    ] = NULL;
 
 		s_table9 [ RDB2_INDEXDB2   ] = g_indexdb2.getRdb();
 		s_table9 [ RDB2_POSDB2     ] = g_posdb2.getRdb();
@@ -2674,6 +2675,7 @@ long getDataSizeFromRdbId ( uint8_t rdbId ) {
 				  i == RDB_SERPDB ||
 				  i == RDB_MONITORDB ||
 				  i == RDB_TAGDB   ||
+				  i == RDB_PARMDB ||
 				  i == RDB_SPIDERDB ||
 				  i == RDB_DOLEDB ||
 				  i == RDB_CATDB ||

Rdb.h

@@ -37,6 +37,7 @@ enum {
 	RDB_CACHEDB, // 17
 	RDB_SERPDB, // 18
 	RDB_MONITORDB, // 19
+	RDB_PARMDB, // kind of a fake rdb for modifying collrec/g_conf parms
 	//RDB_FAKEDB, // used by spider.cpp to fake things out
 // . secondary rdbs for rebuilding done in PageRepair.cpp
 // . we add new recs into these guys and then make the original rdbs

Spider.cpp

@@ -4062,6 +4062,13 @@ void doneSendingNotification ( void *state ) {
 	// spider requests had a valid spider priority, so let's rebuild!
 	cr->m_spiderColl->m_waitingTreeNeedsRebuild = true;
 
+	// we have to send these two parms to all in cluster now
+	SafBuf parmList;
+	g_parms.addParmToList2 ( &parmList , cr , "spiderRoundNum" ); 
+	g_parms.addParmToList2 ( &parmList , cr , "spiderRoundStart" ); 
+	// this uses msg4 so parm ordering is guaranteed
+	g_parms.broadcastParmList ( &parmList , NULL , NULL );
+
 	// log it
 	log("spider: new round #%li starttime = %lu for %s"
 	    , cr->m_spiderRoundNum

TcpSocket.h

@@ -123,6 +123,10 @@ class TcpSocket {
 
 	class UdpSlot *m_udpSlot;
 
+	// stuff used by Pages.cpp
+	SafeBuf m_handyBuf;
+	long m_pageNum;
+
 	// used for debugging, PageResults.cpp sets this to the State0 ptr
 	char *m_tmp;
 };

main.cpp

@@ -82,7 +82,8 @@
 #include "Msg28.h"
 //#include "Msg30.h"
 //#include "MsgB.h"
-#include "Msg3e.h"
+//#include "Msg3e.h"
+#include "Parms.h"
 //#include "Msg50.h"
 //#include "MsgF.h"
 //#include "Msg33.h"
@@ -3159,9 +3160,15 @@ int main ( int argc , char *argv[] ) {
 	//	log("db: Failed to init Statsdb snapshot sleep callback.");
 
 	// check to make sure we have the latest parms
-	Msg3e msg3e;  
-	msg3e.checkForNewParms();
+	//Msg3e msg3e;  
+	//msg3e.checkForNewParms();
 
+	// this stuff is similar to alden's msg3e but will sync collections
+	// that were added/deleted
+	if ( ! g_parms.syncParmsWithHost0() ) {
+		log("parms: error syncing parms: %s",mstrerror(g_errno));
+		return 0;
+	}
 
 	if(recoveryMode) {
 		//now that everything is init-ed send the message.
@@ -4898,6 +4905,9 @@ bool registerMsgHandlers2(){
 	if(! g_udpServer.registerHandler(0x4f,handleRequest4f)) return false;
 	if(! g_udpServer.registerHandler(0x95,handleRequest95)) return false;
 
+	if(! g_udpServer.registerHandler(0x3e,handleRequest3e)) return false;
+	if(! g_udpServer.registerHandler(0x3f,handleRequest3f)) return false;
+
 	return true;
 
 	/*
@@ -4921,7 +4931,7 @@ bool registerMsgHandlers3(){
 	//Msg24 msg24;    if ( ! msg24.registerHandler () ) return false;
 	//Msg40 msg40;    if ( ! msg40.registerHandler () ) return false;
 	//MsgB  msgb;     if ( ! msgb.registerHandler  () ) return false;
-       	Msg3e msg3e;    if ( ! msg3e.registerHandler () ) return false;
+       	//Msg3e msg3e;    if ( ! msg3e.registerHandler () ) return false;
 	//Msg42 msg42;    if ( ! msg42.registerHandler () ) return false;
 	//Msg33 msg33;    if ( ! msg33.registerHandler () ) return false;
 	//if ( ! g_pingServer.registerHandler() ) return false;