gigablast/open-source-search-engine
1
0
Fork 0
mirror of https://github.com/gigablast/open-source-search-engine.git synced 2024-10-04 04:07:13 +03:00
Code Issues Projects Releases Wiki Activity

fix new merging algo.

Browse Source
This commit is contained in:
Matt Wells 2015-08-16 10:11:21 -07:00
parent 178721d35b
commit f8fb266844
4 changed files with 42 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Collectiondb.cpp
View File

@ -1692,6 +1692,8 @@ static CollectionRec g_default;
CollectionRec::CollectionRec() {
m_nextLink = NULL;
m_prevLink = NULL;
m_spiderCorruptCount = 0;
m_collnum = -1;
m_coll[0] = '\0';

47
Rdb.cpp
View File

@ -1671,6 +1671,27 @@ static CollectionRec *s_mergeHead = NULL;
static CollectionRec *s_mergeTail = NULL;
static bool s_needsBuild = true;
void addCollnumToLinkedListOfMergeCandidates ( collnum_t dumpCollnum ) {
// add this collection to the linked list of merge candidates
CollectionRec *cr = g_collectiondb.getRec ( dumpCollnum );
if ( ! cr ) return;
// do not double add it, if already there just return
if ( cr->m_nextLink ) return;
if ( cr->m_prevLink ) return;
if ( s_mergeTail && cr ) {
s_mergeTail->m_nextLink = cr;
cr ->m_nextLink = NULL;
cr ->m_prevLink = s_mergeTail;
s_mergeTail = cr;
}
else if ( cr ) {
cr->m_prevLink = NULL;
cr->m_nextLink = NULL;
s_mergeHead = cr;
s_mergeTail = cr;
}
}
// this is also called in Collectiondb::deleteRec2()
void removeFromMergeLinkedList ( CollectionRec *cr ) {
CollectionRec *prev = cr->m_prevLink;
@ -1691,21 +1712,6 @@ void doneDumpingCollWrapper ( void *state ) {
//RdbBase *base = THIS->getBase(THIS->m_dumpCollnum);
//if ( base ) base->m_checkedForMerge = false;
// add this collection to the linked list of merge candidates
CollectionRec *cr = g_collectiondb.getRec ( THIS->m_dumpCollnum );
if ( s_mergeTail && cr ) {
s_mergeTail->m_nextLink = cr;
cr ->m_nextLink = NULL;
cr ->m_prevLink = s_mergeTail;
s_mergeTail = cr;
}
else if ( cr ) {
cr->m_prevLink = NULL;
cr->m_nextLink = NULL;
s_mergeHead = cr;
s_mergeTail = cr;
}
// return if the loop blocked
if ( ! THIS->dumpCollLoop() ) return;
// otherwise, call big wrapper
@ -1848,6 +1854,17 @@ void attemptMergeAll2 ( ) {
removeFromMergeLinkedList ( cr );
cr = next;
}
// every 60 seconds try to merge collectionless rdbs
static int32_t s_count = 0;
if ( ++s_count == 30 ) {
s_count = 0;
// try to merge collectionless rdbs like statsdb/catdb
RdbBase *base1 = g_catdb.getRdb()->getBase(0);
if ( base1 ) base1->attemptMerge(niceness,force,true);
RdbBase *base2 = g_statsdb.getRdb()->getBase(0);
if ( base2 ) base2->attemptMerge(niceness,force,true);
}
}
// . return false and set g_errno on error

1
Rdb.h
View File

@ -16,6 +16,7 @@
bool makeTrashDir() ;
void removeFromMergeLinkedList ( class CollectionRec *cr ) ;
void addCollnumToLinkedListOfMergeCandidates ( collnum_t dumpCollnum ) ;
// . each Rdb instance has an ID
// . these ids are also return values for getIdFromRdb()

7
RdbDump.cpp
View File

@ -231,6 +231,13 @@ void RdbDump::doneDumping ( ) {
// save the map to disk. true = allDone
if ( m_map ) m_map->writeMap( true );
// now try to merge this collection/db again
// if not already in the linked list. but do not add to linked list
// if it is statsdb or catdb.
if ( m_rdb && ! m_rdb->m_isCollectionLess )
addCollnumToLinkedListOfMergeCandidates ( m_collnum );
#ifdef GBSANITYCHECK
// sanity check
log("DOING SANITY CHECK FOR MAP -- REMOVE ME");