2013-08-03 00:12:24 +04:00
|
|
|
// Matt Wells, copyright Sep 2000
|
|
|
|
|
|
|
|
// . TODO: fix it, include firstPass stuff
|
|
|
|
|
|
|
|
// . the merge file reads slots from 1 or more files and dumps to another
|
|
|
|
// . the merge file dumps those slots in order of keys to the destination file
|
|
|
|
// . slots with a zero slotSize will be removed
|
|
|
|
// . does not use any memory or disk space constraints (TODO)
|
|
|
|
// . the amount of memory it needs is mostly just from map file (is small)
|
|
|
|
// . disk space is relatively small to memory
|
|
|
|
// . TODO: create a static var so only one merge can happen at a time
|
|
|
|
|
|
|
|
// . on an index of 40 gigs of just key the map file can take 56 megs
|
|
|
|
// . as we're merging X files into one file we should free up the maps
|
|
|
|
// we're merging so we don't take too much memory
|
|
|
|
// . RdbMap should have a base page number, the page # of first page in it's
|
|
|
|
// m_keys/m_offsets/m_dataSizes array
|
|
|
|
// . shifting 50 megs down will take probably half a second or so but
|
|
|
|
// we do it to save memory and it should only be done every 10 megs, say
|
|
|
|
// . and we can also start benefiting from the merged files immediately in
|
|
|
|
// seek time
|
|
|
|
|
|
|
|
// . RdbScan/RdbGet are different now, we have to figure out a way to
|
|
|
|
// read in 1 meg or less (as close as we can get to 1 meg) from each
|
|
|
|
// rdb file... TODO
|
|
|
|
|
|
|
|
#ifndef _RDBMERGE_H_
|
|
|
|
#define _RDBMERGE_H_
|
|
|
|
|
|
|
|
#include "RdbDump.h"
|
|
|
|
#include "Msg5.h"
|
|
|
|
|
|
|
|
// . we try to read this many bytes at a time then dump to a file
|
|
|
|
// . keep it to 5 megs for now
|
|
|
|
//#define MERGE_BUF_SIZE (5*1024*1024)
|
|
|
|
// . i'm trying to improve query response time
|
|
|
|
// . while merging, it seems it is directly proportional to MERGE_BUF_SIZE
|
|
|
|
// . we also need to disable bdflush so all the writes don't spew out on
|
|
|
|
// some important reads
|
|
|
|
// . i think if we do large reads the kernel commits to it and cancelling
|
|
|
|
// the thread does not really cancel the read, so keep reads small
|
|
|
|
//#define MERGE_BUF_SIZE (1000*1000)
|
|
|
|
// . now we cancel threads, try a bigger one
|
|
|
|
// . Lars's stripe size is 1 meg i believe
|
|
|
|
//#define MERGE_BUF_SIZE (4*1024*1024)
|
|
|
|
// for debugging tiering, i made this small
|
|
|
|
//#define MERGE_BUF_SIZE (1024)
|
|
|
|
|
|
|
|
class RdbMerge {
|
|
|
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
// . selects the files to merge
|
|
|
|
// . uses keyMasks and files from the passed Rdb class
|
|
|
|
// . filter out keys where key & m_keyMask != m_maskValue
|
|
|
|
// . merge to a new file
|
|
|
|
// . new file name is stored in m_filename so Rdb can look at it
|
|
|
|
// . calls rdb->incorporateMerge() when done with merge or had error
|
|
|
|
// . "maxBufSize" is size of list to get then write (read/write buf)
|
|
|
|
bool merge ( char rdbId ,
|
2013-12-26 05:12:09 +04:00
|
|
|
//char *coll ,
|
|
|
|
collnum_t collnum ,
|
2013-08-03 00:12:24 +04:00
|
|
|
BigFile *target ,
|
|
|
|
RdbMap *targetMap ,
|
2014-11-11 01:45:11 +03:00
|
|
|
int32_t id2 ,
|
|
|
|
int32_t startFileNum ,
|
|
|
|
int32_t numFiles ,
|
|
|
|
int32_t niceness ,
|
2013-08-03 00:12:24 +04:00
|
|
|
class DiskPageCache *pc ,
|
2014-10-30 22:36:39 +03:00
|
|
|
int64_t maxTargetFileSize ,
|
2013-08-03 00:12:24 +04:00
|
|
|
char keySize );
|
|
|
|
|
|
|
|
bool isMerging ( ) { return m_isMerging; };
|
|
|
|
// if we are merging, use this to see what rdb we are merging for
|
|
|
|
//class RdbBase *getRdbBase ( ) { return m_base; };
|
|
|
|
|
|
|
|
// suspend the merging until resumeMerge() is called
|
|
|
|
void suspendMerge ( ) ;
|
|
|
|
|
|
|
|
// . return false and sets errno on error merging
|
|
|
|
// . returns true if blocked, or completed successfully
|
|
|
|
bool resumeMerge ( ) ;
|
|
|
|
|
|
|
|
// these must be public so wrappers can call
|
|
|
|
bool dumpList ( );
|
|
|
|
bool getNextList ( ) ;
|
|
|
|
bool getAnotherList ( ) ;
|
|
|
|
void doneMerging ( ) ;
|
|
|
|
|
|
|
|
RdbMerge() ;
|
|
|
|
~RdbMerge() ;
|
|
|
|
void reset();
|
|
|
|
|
|
|
|
// . called to continue merge initialization after lock is secure
|
|
|
|
// . lock is g_isMergingLock
|
|
|
|
bool gotLock ( ) ;
|
|
|
|
|
|
|
|
// set to true when m_startKey wraps back to 0
|
|
|
|
bool m_doneMerging;
|
|
|
|
|
|
|
|
// how many dup keys removed from the IndexList merge, if any?
|
2014-10-30 22:36:39 +03:00
|
|
|
int64_t getDupsRemoved ( ) { return m_dupsRemoved; };
|
2013-08-03 00:12:24 +04:00
|
|
|
|
|
|
|
void doSleep();
|
|
|
|
|
2014-11-11 01:45:11 +03:00
|
|
|
int32_t m_numThreads;
|
2013-08-03 00:12:24 +04:00
|
|
|
|
|
|
|
// private:
|
|
|
|
|
|
|
|
// . used when growing the database
|
2014-11-18 05:13:36 +03:00
|
|
|
// . removes keys that would no longer be stored by us
|
2013-08-03 00:12:24 +04:00
|
|
|
//void filterList ( RdbList *list ) ;
|
|
|
|
|
|
|
|
// . we get the units from the master and the mergees from the units
|
|
|
|
class RdbBase *m_base;
|
2014-11-11 01:45:11 +03:00
|
|
|
int32_t m_startFileNum;
|
|
|
|
int32_t m_numFiles;
|
2013-08-03 00:12:24 +04:00
|
|
|
bool m_dedup;
|
2014-11-11 01:45:11 +03:00
|
|
|
int32_t m_fixedDataSize;
|
2013-08-03 00:12:24 +04:00
|
|
|
BigFile *m_target;
|
|
|
|
RdbMap *m_targetMap;
|
|
|
|
|
|
|
|
// these are used by truncateList()
|
|
|
|
//key_t m_truncationMask;
|
|
|
|
//key_t m_lastMaskedKey;
|
|
|
|
char m_truncationMask[MAX_KEY_BYTES];
|
|
|
|
char m_lastMaskedKey[MAX_KEY_BYTES];
|
2014-11-11 01:45:11 +03:00
|
|
|
int32_t m_lastNumRecs;
|
2013-08-03 00:12:24 +04:00
|
|
|
|
|
|
|
//key_t m_startKey;
|
|
|
|
//key_t m_endKey;
|
|
|
|
char m_startKey[MAX_KEY_BYTES];
|
|
|
|
char m_endKey[MAX_KEY_BYTES];
|
|
|
|
|
|
|
|
bool m_isMerging;
|
|
|
|
bool m_isSuspended;
|
|
|
|
bool m_isReadyToSave;
|
|
|
|
|
|
|
|
// for writing to target file
|
|
|
|
RdbDump m_dump;
|
|
|
|
|
|
|
|
// a Msg5 for getting RdbLists from disk/cache
|
|
|
|
Msg5 m_msg5;
|
|
|
|
Msg5 m_msg5b;
|
|
|
|
|
|
|
|
RdbList m_list;
|
|
|
|
|
2014-11-11 01:45:11 +03:00
|
|
|
int32_t m_niceness;
|
2013-08-03 00:12:24 +04:00
|
|
|
|
|
|
|
// count for indexdb
|
2014-10-30 22:36:39 +03:00
|
|
|
int64_t m_dupsRemoved;
|
2013-08-03 00:12:24 +04:00
|
|
|
|
|
|
|
class DiskPageCache *m_pc;
|
2014-10-30 22:36:39 +03:00
|
|
|
int64_t m_maxTargetFileSize;
|
2013-08-03 00:12:24 +04:00
|
|
|
|
2014-11-11 01:45:11 +03:00
|
|
|
int32_t m_id2;
|
2013-08-03 00:12:24 +04:00
|
|
|
|
|
|
|
// for getting the RdbBase class doing the merge
|
|
|
|
uint8_t m_rdbId;
|
2013-12-26 05:12:09 +04:00
|
|
|
//char m_coll [ MAX_COLL_LEN + 1 ];
|
|
|
|
collnum_t m_collnum;
|
2013-08-03 00:12:24 +04:00
|
|
|
|
|
|
|
char m_ks;
|
|
|
|
};
|
|
|
|
|
|
|
|
#endif
|