graninas/Hydra
1
1
Fork 0
mirror of https://github.com/graninas/Hydra.git synced 2024-11-27 17:25:21 +03:00
Code Issues Projects Releases Wiki Activity

KV DB WIP

Browse Source
This commit is contained in:
Alexander Granin 2019-08-12 01:37:54 +07:00
parent 6defc6c8bb
commit bc0a0bf93f
18 changed files with 1213 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
app/Astro/Astro/Catalogue.hs
View File

@ -12,25 +12,21 @@ import Astro.Types
import Astro.KVDB.Model
import Astro.Lens
withCatalogueDB
:: AppState
-> L.KVDBL db a
-> L.LangL a
withCatalogueDB :: AppState -> L.KVDBL db a -> L.LangL a
withCatalogueDB st = L.withKVDB (st ^. catalogueDB)
-- loadMeteorsCount :: L.KVDBL CatalogueDB Int
loadMeteorsCount :: AppState -> L.LangL Int
loadMeteorsCount st = do
eTest <- withCatalogueDB st $ L.getValue "test"
pure 10
eCount <- withCatalogueDB st $ L.getValue "meteors_count"
case eCount of
Left err -> L.logError ("Failed to get meteors count: " <> show err)
Right n -> pure 10
dynamicsMonitor :: AppState -> L.LangL ()
dynamicsMonitor st = do
meteorsCount <- loadMeteorsCount st
L.logInfo $ "Meteors count: " +|| meteorsCount ||+ ""
pure ()
L.delay $ 1000
initState :: AppConfig -> L.AppL AppState
@ -60,4 +56,5 @@ astroCatalogue cfg = do
L.process $ dynamicsMonitor appSt
L.awaitAppForever
-- L.awaitAppForever
L.delay 10000000

31
app/Astro/Astro/Domain/Meteor.hs Normal file
View File

@ -0,0 +1,31 @@
{-# LANGUAGE DeriveAnyClass #-}
module Astro.Domain.Meteor where
import Hydra.Prelude
import qualified Data.Map as Map
import qualified Data.Set as Set
import Data.Time.Clock (UTCTime)
type DateTime = UTCTime
data Coords = Coords
{ _azimuth :: Int
, _altitude :: Int
}
deriving (Show, Eq, Ord, Generic, ToJSON, FromJSON)
data Meteor' k = Meteor'
{ _id :: k
, _size :: Int
, _mass :: Int
, _coords :: Coords
, _timestamp :: DateTime
}
deriving (Show, Eq, Ord, Generic, ToJSON, FromJSON)
type RawMeteor = Meteor' ()
type Meteor = Meteor' Int
-- type Meteors = D.StateVar (Set.Set Meteor)

98
app/Astro/Astro/KVDB/Entities/Meteor.hs
View File

@ -5,40 +5,61 @@ module Astro.KVDB.Entities.Meteor where
import Hydra.Prelude
import qualified Data.Aeson as A
import qualified Data.ByteString.Lazy as LBS
import qualified Data.Aeson as A
import qualified Data.ByteString.Lazy as LBS
import Text.Printf
import qualified Hydra.Domain as D
-- meteors (meteor_idx -> meteor_entity_json)
-- ------------------------------------------------------------
-- 0000000 {}
import qualified Astro.Domain.Meteor as D
-- catalogue
-- ( meteors_count_key -> int
-- , meteor_idx|0 -> meteor_entity_json
-- )
-- ------------------------------------------------------------------
-- meteors_count 1
-- 0|0000000 {...}
-- 0|0000001 {...}
data MeteorEntity
--
-- instance KVDBModelEntity MeteorsTable MeteorEntity
--
--
-- instance D.DBEntity KBlockPrevHashEntity where
-- data DBKey KBlockPrevHashEntity = KBlockPrevHashKey D.BlockNumber
-- deriving (Show, Eq, Ord)
-- data DBValue KBlockPrevHashEntity = KBlockPrevHashValue D.StringHash
-- deriving (Show, Eq, Ord, Generic, ToJSON, FromJSON)
--
-- instance D.ToDBKey KBlockPrevHashEntity D.BlockNumber where
-- toDBKey = KBlockPrevHashKey
--
-- instance D.ToDBKey KBlockPrevHashEntity D.KBlock where
-- toDBKey = KBlockPrevHashKey . D._number
--
-- instance D.ToDBValue KBlockPrevHashEntity D.KBlock where
-- toDBValue kBlock = KBlockPrevHashValue $ kBlock ^. Lens.prevHash
--
-- -- TODO: this can be made by default
-- instance D.RawDBEntity KBlocksDB KBlockPrevHashEntity where
-- toRawDBKey (KBlockPrevHashKey kBlockIdx) = encodeUtf8 $ toKBlockPrevHashEntityKeyBase kBlockIdx
-- toRawDBValue = LBS.toStrict . A.encode
-- fromRawDBValue = A.decode . LBS.fromStrict
--
data MeteorsCountEntity
-- MeteorEntity
instance D.DBEntity MeteorEntity where
data KeyEntity MeteorEntity = MeteorKey Int
deriving (Show, Eq, Ord)
data ValueEntity MeteorEntity = MeteorValue D.Meteor
deriving (Show, Eq, Ord, Generic, ToJSON, FromJSON)
instance D.AsKeyEntity MeteorEntity Int where
toKeyEntity = MeteorKey
instance D.AsKeyEntity MeteorEntity D.Meteor where
toKeyEntity = MeteorKey . D._id
instance D.AsValueEntity MeteorEntity D.Meteor where
toValueEntity = MeteorValue
fromValueEntity (MeteorValue v) = v
instance D.RawDBEntity MeteorEntity where
toDBKey (MeteorKey idx) = show $ toMeteorEntityKey idx
toDBValue = D.toDBValueJSON
fromDBValue = D.fromDBValueJSON
-- instance D.DBModelEntity CatalogueDB MeteorEntity
-- instance D.DBModelEntity CatalogueDB MeteorsCountEntity
-- instance D.RawDBEntity CatalogueDB MeteorEntity where
-- toRawDBKey (MeteorKey idx) = A.encode kBlockIdx
-- toRawDBValue = LBS.toStrict . A.encode
-- fromRawDBValue = A.decode . LBS.fromStrict
-- -- KBlock entity
--
-- instance D.DBEntity KBlockEntity where
@ -60,13 +81,10 @@ data MeteorEntity
-- toRawDBKey (KBlockKey kBlockIdx) = encodeUtf8 $ toKBlockEntityKeyBase kBlockIdx
-- toRawDBValue = LBS.toStrict . A.encode
-- fromRawDBValue = A.decode . LBS.fromStrict
--
--
-- toKBlockIdxBase :: KBlockIdx -> String
-- toKBlockIdxBase = printf "%07d"
--
-- toKBlockPrevHashEntityKeyBase :: KBlockIdx -> String
-- toKBlockPrevHashEntityKeyBase = (<> "0") . toKBlockIdxBase
--
-- toKBlockEntityKeyBase :: KBlockIdx -> String
-- toKBlockEntityKeyBase = (<> "1") . toKBlockIdxBase
toIdxBase :: Int -> String
toIdxBase = printf "%07d"
toMeteorEntityKey :: Int -> String
toMeteorEntityKey = ("0|" <>) . toIdxBase

11
app/Astro/Main.hs
View File

@ -15,10 +15,19 @@ import qualified Hydra.Runtime as R
import Astro.Types
import Astro.Catalogue
loggerCfg :: D.LoggerConfig
loggerCfg = D.LoggerConfig
{ D._format = "$prio $loggername: $msg"
, D._level = D.Debug
, D._logFilePath = ""
, D._logToConsole = True
, D._logToFile = False
}
main :: IO ()
main = do
loggerRt <- R.createVoidLoggerRuntime
loggerRt <- R.createLoggerRuntime loggerCfg
coreRt <- R.createCoreRuntime loggerRt
let cfg = AppConfig False 0

0
catalogue/000009.log Normal file
View File

1
catalogue/CURRENT Normal file
View File

@ -0,0 +1 @@
MANIFEST-000008

1
catalogue/IDENTITY Normal file
View File

@ -0,0 +1 @@
15b98768706bc3f9-120aec5d7215c312

0
catalogue/LOCK Normal file
View File

240
catalogue/LOG Normal file
View File

@ -0,0 +1,240 @@
2019/08/10-17:02:41.640123 109c9b5c0 RocksDB version: 5.18.3
2019/08/10-17:02:41.640589 109c9b5c0 Git sha rocksdb_build_git_sha:
2019/08/10-17:02:41.640592 109c9b5c0 Compile date Feb 21 2019
2019/08/10-17:02:41.640593 109c9b5c0 DB SUMMARY
2019/08/10-17:02:41.640660 109c9b5c0 CURRENT file: CURRENT
2019/08/10-17:02:41.640662 109c9b5c0 IDENTITY file: IDENTITY
2019/08/10-17:02:41.640668 109c9b5c0 MANIFEST file: MANIFEST-000005 size: 59 Bytes
2019/08/10-17:02:41.640670 109c9b5c0 SST files in catalogue dir, Total Num: 0, files:
2019/08/10-17:02:41.640671 109c9b5c0 Write Ahead Log file in catalogue: 000006.log size: 0 ;
2019/08/10-17:02:41.640673 109c9b5c0 Options.error_if_exists: 0
2019/08/10-17:02:41.640677 109c9b5c0 Options.create_if_missing: 1
2019/08/10-17:02:41.640678 109c9b5c0 Options.paranoid_checks: 0
2019/08/10-17:02:41.640679 109c9b5c0 Options.env: 0x1018392e8
2019/08/10-17:02:41.640680 109c9b5c0 Options.info_log: 0x7fd2b1421400
2019/08/10-17:02:41.640681 109c9b5c0 Options.max_file_opening_threads: 16
2019/08/10-17:02:41.640682 109c9b5c0 Options.statistics: 0x0
2019/08/10-17:02:41.640683 109c9b5c0 Options.use_fsync: 0
2019/08/10-17:02:41.640684 109c9b5c0 Options.max_log_file_size: 0
2019/08/10-17:02:41.640685 109c9b5c0 Options.max_manifest_file_size: 1073741824
2019/08/10-17:02:41.640686 109c9b5c0 Options.log_file_time_to_roll: 0
2019/08/10-17:02:41.640687 109c9b5c0 Options.keep_log_file_num: 1000
2019/08/10-17:02:41.640688 109c9b5c0 Options.recycle_log_file_num: 0
2019/08/10-17:02:41.640689 109c9b5c0 Options.allow_fallocate: 1
2019/08/10-17:02:41.640689 109c9b5c0 Options.allow_mmap_reads: 0
2019/08/10-17:02:41.640690 109c9b5c0 Options.allow_mmap_writes: 0
2019/08/10-17:02:41.640691 109c9b5c0 Options.use_direct_reads: 0
2019/08/10-17:02:41.640692 109c9b5c0 Options.use_direct_io_for_flush_and_compaction: 0
2019/08/10-17:02:41.640693 109c9b5c0 Options.create_missing_column_families: 0
2019/08/10-17:02:41.640694 109c9b5c0 Options.db_log_dir:
2019/08/10-17:02:41.640695 109c9b5c0 Options.wal_dir: catalogue
2019/08/10-17:02:41.640696 109c9b5c0 Options.table_cache_numshardbits: 6
2019/08/10-17:02:41.640697 109c9b5c0 Options.max_subcompactions: 1
2019/08/10-17:02:41.640698 109c9b5c0 Options.max_background_flushes: -1
2019/08/10-17:02:41.640699 109c9b5c0 Options.WAL_ttl_seconds: 0
2019/08/10-17:02:41.640699 109c9b5c0 Options.WAL_size_limit_MB: 0
2019/08/10-17:02:41.640700 109c9b5c0 Options.manifest_preallocation_size: 4194304
2019/08/10-17:02:41.640701 109c9b5c0 Options.is_fd_close_on_exec: 1
2019/08/10-17:02:41.640702 109c9b5c0 Options.advise_random_on_open: 1
2019/08/10-17:02:41.640703 109c9b5c0 Options.db_write_buffer_size: 0
2019/08/10-17:02:41.640704 109c9b5c0 Options.write_buffer_manager: 0x7fd2b14210c0
2019/08/10-17:02:41.640705 109c9b5c0 Options.access_hint_on_compaction_start: 1
2019/08/10-17:02:41.640706 109c9b5c0 Options.new_table_reader_for_compaction_inputs: 0
2019/08/10-17:02:41.640707 109c9b5c0 Options.random_access_max_buffer_size: 1048576
2019/08/10-17:02:41.640708 109c9b5c0 Options.use_adaptive_mutex: 0
2019/08/10-17:02:41.640708 109c9b5c0 Options.rate_limiter: 0x0
2019/08/10-17:02:41.640710 109c9b5c0 Options.sst_file_manager.rate_bytes_per_sec: 0
2019/08/10-17:02:41.640711 109c9b5c0 Options.wal_recovery_mode: 2
2019/08/10-17:02:41.640711 109c9b5c0 Options.enable_thread_tracking: 0
2019/08/10-17:02:41.640712 109c9b5c0 Options.enable_pipelined_write: 0
2019/08/10-17:02:41.640740 109c9b5c0 Options.allow_concurrent_memtable_write: 1
2019/08/10-17:02:41.640742 109c9b5c0 Options.enable_write_thread_adaptive_yield: 1
2019/08/10-17:02:41.640743 109c9b5c0 Options.write_thread_max_yield_usec: 100
2019/08/10-17:02:41.640744 109c9b5c0 Options.write_thread_slow_yield_usec: 3
2019/08/10-17:02:41.640744 109c9b5c0 Options.row_cache: None
2019/08/10-17:02:41.640745 109c9b5c0 Options.wal_filter: None
2019/08/10-17:02:41.640746 109c9b5c0 Options.avoid_flush_during_recovery: 0
2019/08/10-17:02:41.640747 109c9b5c0 Options.allow_ingest_behind: 0
2019/08/10-17:02:41.640748 109c9b5c0 Options.preserve_deletes: 0
2019/08/10-17:02:41.640749 109c9b5c0 Options.two_write_queues: 0
2019/08/10-17:02:41.640750 109c9b5c0 Options.manual_wal_flush: 0
2019/08/10-17:02:41.640751 109c9b5c0 Options.max_background_jobs: 2
2019/08/10-17:02:41.640752 109c9b5c0 Options.max_background_compactions: -1
2019/08/10-17:02:41.640753 109c9b5c0 Options.avoid_flush_during_shutdown: 0
2019/08/10-17:02:41.640754 109c9b5c0 Options.writable_file_max_buffer_size: 1048576
2019/08/10-17:02:41.640755 109c9b5c0 Options.delayed_write_rate : 16777216
2019/08/10-17:02:41.640756 109c9b5c0 Options.max_total_wal_size: 0
2019/08/10-17:02:41.640757 109c9b5c0 Options.delete_obsolete_files_period_micros: 21600000000
2019/08/10-17:02:41.640758 109c9b5c0 Options.stats_dump_period_sec: 600
2019/08/10-17:02:41.640759 109c9b5c0 Options.max_open_files: 256
2019/08/10-17:02:41.640759 109c9b5c0 Options.bytes_per_sync: 0
2019/08/10-17:02:41.640760 109c9b5c0 Options.wal_bytes_per_sync: 0
2019/08/10-17:02:41.640761 109c9b5c0 Options.compaction_readahead_size: 0
2019/08/10-17:02:41.640762 109c9b5c0 Compression algorithms supported:
2019/08/10-17:02:41.640763 109c9b5c0 kZSTD supported: 0
2019/08/10-17:02:41.640764 109c9b5c0 kZlibCompression supported: 1
2019/08/10-17:02:41.640765 109c9b5c0 kXpressCompression supported: 0
2019/08/10-17:02:41.640766 109c9b5c0 kSnappyCompression supported: 1
2019/08/10-17:02:41.640767 109c9b5c0 kZSTDNotFinalCompression supported: 0
2019/08/10-17:02:41.640768 109c9b5c0 kLZ4HCCompression supported: 1
2019/08/10-17:02:41.640769 109c9b5c0 kLZ4Compression supported: 1
2019/08/10-17:02:41.640770 109c9b5c0 kBZip2Compression supported: 1
2019/08/10-17:02:41.640774 109c9b5c0 Fast CRC32 supported: Supported on x86
2019/08/10-17:02:41.640948 109c9b5c0 [/version_set.cc:3508] Recovering from manifest file: MANIFEST-000005
2019/08/10-17:02:41.640999 109c9b5c0 [/column_family.cc:474] --------------- Options for column family [default]:
2019/08/10-17:02:41.641004 109c9b5c0 Options.comparator: leveldb.BytewiseComparator
2019/08/10-17:02:41.641005 109c9b5c0 Options.merge_operator: None
2019/08/10-17:02:41.641006 109c9b5c0 Options.compaction_filter: None
2019/08/10-17:02:41.641007 109c9b5c0 Options.compaction_filter_factory: None
2019/08/10-17:02:41.641008 109c9b5c0 Options.memtable_factory: SkipListFactory
2019/08/10-17:02:41.641009 109c9b5c0 Options.table_factory: BlockBasedTable
2019/08/10-17:02:41.641027 109c9b5c0 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7fd2b1420530)
cache_index_and_filter_blocks: 0
cache_index_and_filter_blocks_with_high_priority: 0
pin_l0_filter_and_index_blocks_in_cache: 0
pin_top_level_index_and_filter: 1
index_type: 0
hash_index_allow_collision: 1
checksum: 1
no_block_cache: 0
block_cache: 0x7fd2b1420578
block_cache_name: LRUCache
block_cache_options:
capacity : 8388608
num_shard_bits : 4
strict_capacity_limit : 0
memory_allocator : None
high_pri_pool_ratio: 0.000
block_cache_compressed: 0x0
persistent_cache: 0x0
block_size: 4096
block_size_deviation: 10
block_restart_interval: 16
index_block_restart_interval: 1
metadata_block_size: 4096
partition_filters: 0
use_delta_encoding: 1
filter_policy: nullptr
whole_key_filtering: 1
verify_compression: 0
read_amp_bytes_per_bit: 0
format_version: 2
enable_index_compression: 1
block_align: 0
2019/08/10-17:02:41.641053 109c9b5c0 Options.write_buffer_size: 4194304
2019/08/10-17:02:41.641055 109c9b5c0 Options.max_write_buffer_number: 2
2019/08/10-17:02:41.641058 109c9b5c0 Options.compression: Snappy
2019/08/10-17:02:41.641059 109c9b5c0 Options.bottommost_compression: Disabled
2019/08/10-17:02:41.641060 109c9b5c0 Options.prefix_extractor: nullptr
2019/08/10-17:02:41.641061 109c9b5c0 Options.memtable_insert_with_hint_prefix_extractor: nullptr
2019/08/10-17:02:41.641062 109c9b5c0 Options.num_levels: 7
2019/08/10-17:02:41.641063 109c9b5c0 Options.min_write_buffer_number_to_merge: 1
2019/08/10-17:02:41.641064 109c9b5c0 Options.max_write_buffer_number_to_maintain: 0
2019/08/10-17:02:41.641065 109c9b5c0 Options.bottommost_compression_opts.window_bits: -14
2019/08/10-17:02:41.641066 109c9b5c0 Options.bottommost_compression_opts.level: 32767
2019/08/10-17:02:41.641067 109c9b5c0 Options.bottommost_compression_opts.strategy: 0
2019/08/10-17:02:41.641068 109c9b5c0 Options.bottommost_compression_opts.max_dict_bytes: 0
2019/08/10-17:02:41.641069 109c9b5c0 Options.bottommost_compression_opts.zstd_max_train_bytes: 0
2019/08/10-17:02:41.641070 109c9b5c0 Options.bottommost_compression_opts.enabled: false
2019/08/10-17:02:41.641071 109c9b5c0 Options.compression_opts.window_bits: -14
2019/08/10-17:02:41.641072 109c9b5c0 Options.compression_opts.level: 32767
2019/08/10-17:02:41.641073 109c9b5c0 Options.compression_opts.strategy: 0
2019/08/10-17:02:41.641074 109c9b5c0 Options.compression_opts.max_dict_bytes: 0
2019/08/10-17:02:41.641075 109c9b5c0 Options.compression_opts.zstd_max_train_bytes: 0
2019/08/10-17:02:41.641075 109c9b5c0 Options.compression_opts.enabled: false
2019/08/10-17:02:41.641076 109c9b5c0 Options.level0_file_num_compaction_trigger: 4
2019/08/10-17:02:41.641077 109c9b5c0 Options.level0_slowdown_writes_trigger: 20
2019/08/10-17:02:41.641078 109c9b5c0 Options.level0_stop_writes_trigger: 36
2019/08/10-17:02:41.641079 109c9b5c0 Options.target_file_size_base: 67108864
2019/08/10-17:02:41.641080 109c9b5c0 Options.target_file_size_multiplier: 1
2019/08/10-17:02:41.641081 109c9b5c0 Options.max_bytes_for_level_base: 268435456
2019/08/10-17:02:41.641082 109c9b5c0 Options.level_compaction_dynamic_level_bytes: 0
2019/08/10-17:02:41.641083 109c9b5c0 Options.max_bytes_for_level_multiplier: 10.000000
2019/08/10-17:02:41.641085 109c9b5c0 Options.max_bytes_for_level_multiplier_addtl[0]: 1
2019/08/10-17:02:41.641086 109c9b5c0 Options.max_bytes_for_level_multiplier_addtl[1]: 1
2019/08/10-17:02:41.641087 109c9b5c0 Options.max_bytes_for_level_multiplier_addtl[2]: 1
2019/08/10-17:02:41.641088 109c9b5c0 Options.max_bytes_for_level_multiplier_addtl[3]: 1
2019/08/10-17:02:41.641089 109c9b5c0 Options.max_bytes_for_level_multiplier_addtl[4]: 1
2019/08/10-17:02:41.641090 109c9b5c0 Options.max_bytes_for_level_multiplier_addtl[5]: 1
2019/08/10-17:02:41.641090 109c9b5c0 Options.max_bytes_for_level_multiplier_addtl[6]: 1
2019/08/10-17:02:41.641091 109c9b5c0 Options.max_sequential_skip_in_iterations: 8
2019/08/10-17:02:41.641092 109c9b5c0 Options.max_compaction_bytes: 1677721600
2019/08/10-17:02:41.641093 109c9b5c0 Options.arena_block_size: 524288
2019/08/10-17:02:41.641094 109c9b5c0 Options.soft_pending_compaction_bytes_limit: 68719476736
2019/08/10-17:02:41.641095 109c9b5c0 Options.hard_pending_compaction_bytes_limit: 274877906944
2019/08/10-17:02:41.641110 109c9b5c0 Options.rate_limit_delay_max_milliseconds: 100
2019/08/10-17:02:41.641112 109c9b5c0 Options.disable_auto_compactions: 0
2019/08/10-17:02:41.641113 109c9b5c0 Options.compaction_style: kCompactionStyleLevel
2019/08/10-17:02:41.641114 109c9b5c0 Options.compaction_pri: kByCompensatedSize
2019/08/10-17:02:41.641115 109c9b5c0 Options.compaction_options_universal.size_ratio: 1
2019/08/10-17:02:41.641116 109c9b5c0 Options.compaction_options_universal.min_merge_width: 2
2019/08/10-17:02:41.641117 109c9b5c0 Options.compaction_options_universal.max_merge_width: 4294967295
2019/08/10-17:02:41.641118 109c9b5c0 Options.compaction_options_universal.max_size_amplification_percent: 200
2019/08/10-17:02:41.641119 109c9b5c0 Options.compaction_options_universal.compression_size_percent: -1
2019/08/10-17:02:41.641120 109c9b5c0 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize
2019/08/10-17:02:41.641121 109c9b5c0 Options.compaction_options_fifo.max_table_files_size: 1073741824
2019/08/10-17:02:41.641122 109c9b5c0 Options.compaction_options_fifo.allow_compaction: 0
2019/08/10-17:02:41.641123 109c9b5c0 Options.compaction_options_fifo.ttl: 0
2019/08/10-17:02:41.641124 109c9b5c0 Options.table_properties_collectors:
2019/08/10-17:02:41.641125 109c9b5c0 Options.inplace_update_support: 0
2019/08/10-17:02:41.641126 109c9b5c0 Options.inplace_update_num_locks: 10000
2019/08/10-17:02:41.641127 109c9b5c0 Options.memtable_prefix_bloom_size_ratio: 0.000000
2019/08/10-17:02:41.641128 109c9b5c0 Options.memtable_huge_page_size: 0
2019/08/10-17:02:41.641129 109c9b5c0 Options.bloom_locality: 0
2019/08/10-17:02:41.641130 109c9b5c0 Options.max_successive_merges: 0
2019/08/10-17:02:41.641130 109c9b5c0 Options.optimize_filters_for_hits: 0
2019/08/10-17:02:41.641131 109c9b5c0 Options.paranoid_file_checks: 0
2019/08/10-17:02:41.641132 109c9b5c0 Options.force_consistency_checks: 0
2019/08/10-17:02:41.641133 109c9b5c0 Options.report_bg_io_stats: 0
2019/08/10-17:02:41.641134 109c9b5c0 Options.ttl: 0
2019/08/10-17:02:41.641207 109c9b5c0 [/version_set.cc:3724] Recovered from manifest file:catalogue/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0
2019/08/10-17:02:41.641210 109c9b5c0 [/version_set.cc:3732] Column family [default] (ID 0), log number is 4
2019/08/10-17:02:41.641298 109c9b5c0 EVENT_LOG_v1 {"time_micros": 1565431361641290, "job": 1, "event": "recovery_started", "log_files": [6]}
2019/08/10-17:02:41.641306 109c9b5c0 [/db_impl_open.cc:578] Recovering log #6 mode 2
2019/08/10-17:02:41.641358 109c9b5c0 [/version_set.cc:3037] Creating manifest 8
2019/08/10-17:02:41.642197 109c9b5c0 EVENT_LOG_v1 {"time_micros": 1565431361642193, "job": 1, "event": "recovery_finished"}
2019/08/10-17:02:41.645351 109c9b5c0 [/db_impl_open.cc:1314] DB pointer 0x7fd2b1904e00
2019/08/10-17:02:41.645524 70000e3b9000 [WARN] [/db_impl.cc:669] ------- DUMPING STATS -------
2019/08/10-17:02:41.645529 70000e3b9000 [WARN] [/db_impl.cc:670]
** DB Stats **
Uptime(secs): 0.0 total, 0.0 interval
Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s
Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s
Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent
Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s
Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s
Interval stall: 00:00:0.000 H:M:S, 0.0 percent
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
----------------------------------------------------------------------------------------------------------------------------------------------------------
Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0
Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0
Uptime(secs): 0.0 total, 0.0 interval
Flush(GB): cumulative 0.000, interval 0.000
AddFile(GB): cumulative 0.000, interval 0.000
AddFile(Total Files): cumulative 0, interval 0
AddFile(L0 Files): cumulative 0, interval 0
AddFile(Keys): cumulative 0, interval 0
Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
** File Read Latency Histogram By Level [default] **
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
----------------------------------------------------------------------------------------------------------------------------------------------------------
Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0
Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0
Uptime(secs): 0.0 total, 0.0 interval
Flush(GB): cumulative 0.000, interval 0.000
AddFile(GB): cumulative 0.000, interval 0.000
AddFile(Total Files): cumulative 0, interval 0
AddFile(L0 Files): cumulative 0, interval 0
AddFile(Keys): cumulative 0, interval 0
Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
** File Read Latency Histogram By Level [default] **

234
catalogue/LOG.old.1565431269860905 Normal file
View File

@ -0,0 +1,234 @@
2019/08/10-17:00:28.118940 10b3645c0 RocksDB version: 5.18.3
2019/08/10-17:00:28.119471 10b3645c0 Git sha rocksdb_build_git_sha:
2019/08/10-17:00:28.119474 10b3645c0 Compile date Feb 21 2019
2019/08/10-17:00:28.119476 10b3645c0 DB SUMMARY
2019/08/10-17:00:28.119518 10b3645c0 SST files in catalogue dir, Total Num: 0, files:
2019/08/10-17:00:28.119521 10b3645c0 Write Ahead Log file in catalogue:
2019/08/10-17:00:28.119522 10b3645c0 Options.error_if_exists: 0
2019/08/10-17:00:28.119525 10b3645c0 Options.create_if_missing: 1
2019/08/10-17:00:28.119526 10b3645c0 Options.paranoid_checks: 0
2019/08/10-17:00:28.119527 10b3645c0 Options.env: 0x10913d2e8
2019/08/10-17:00:28.119528 10b3645c0 Options.info_log: 0x7fb58fc089b0
2019/08/10-17:00:28.119529 10b3645c0 Options.max_file_opening_threads: 16
2019/08/10-17:00:28.119530 10b3645c0 Options.statistics: 0x0
2019/08/10-17:00:28.119531 10b3645c0 Options.use_fsync: 0
2019/08/10-17:00:28.119532 10b3645c0 Options.max_log_file_size: 0
2019/08/10-17:00:28.119533 10b3645c0 Options.max_manifest_file_size: 1073741824
2019/08/10-17:00:28.119534 10b3645c0 Options.log_file_time_to_roll: 0
2019/08/10-17:00:28.119534 10b3645c0 Options.keep_log_file_num: 1000
2019/08/10-17:00:28.119535 10b3645c0 Options.recycle_log_file_num: 0
2019/08/10-17:00:28.119536 10b3645c0 Options.allow_fallocate: 1
2019/08/10-17:00:28.119537 10b3645c0 Options.allow_mmap_reads: 0
2019/08/10-17:00:28.119538 10b3645c0 Options.allow_mmap_writes: 0
2019/08/10-17:00:28.119538 10b3645c0 Options.use_direct_reads: 0
2019/08/10-17:00:28.119539 10b3645c0 Options.use_direct_io_for_flush_and_compaction: 0
2019/08/10-17:00:28.119540 10b3645c0 Options.create_missing_column_families: 0
2019/08/10-17:00:28.119541 10b3645c0 Options.db_log_dir:
2019/08/10-17:00:28.119542 10b3645c0 Options.wal_dir: catalogue
2019/08/10-17:00:28.119543 10b3645c0 Options.table_cache_numshardbits: 6
2019/08/10-17:00:28.119543 10b3645c0 Options.max_subcompactions: 1
2019/08/10-17:00:28.119544 10b3645c0 Options.max_background_flushes: -1
2019/08/10-17:00:28.119545 10b3645c0 Options.WAL_ttl_seconds: 0
2019/08/10-17:00:28.119546 10b3645c0 Options.WAL_size_limit_MB: 0
2019/08/10-17:00:28.119547 10b3645c0 Options.manifest_preallocation_size: 4194304
2019/08/10-17:00:28.119548 10b3645c0 Options.is_fd_close_on_exec: 1
2019/08/10-17:00:28.119548 10b3645c0 Options.advise_random_on_open: 1
2019/08/10-17:00:28.119549 10b3645c0 Options.db_write_buffer_size: 0
2019/08/10-17:00:28.119550 10b3645c0 Options.write_buffer_manager: 0x7fb58fc08950
2019/08/10-17:00:28.119551 10b3645c0 Options.access_hint_on_compaction_start: 1
2019/08/10-17:00:28.119552 10b3645c0 Options.new_table_reader_for_compaction_inputs: 0
2019/08/10-17:00:28.119553 10b3645c0 Options.random_access_max_buffer_size: 1048576
2019/08/10-17:00:28.119553 10b3645c0 Options.use_adaptive_mutex: 0
2019/08/10-17:00:28.119554 10b3645c0 Options.rate_limiter: 0x0
2019/08/10-17:00:28.119555 10b3645c0 Options.sst_file_manager.rate_bytes_per_sec: 0
2019/08/10-17:00:28.119556 10b3645c0 Options.wal_recovery_mode: 2
2019/08/10-17:00:28.119557 10b3645c0 Options.enable_thread_tracking: 0
2019/08/10-17:00:28.119558 10b3645c0 Options.enable_pipelined_write: 0
2019/08/10-17:00:28.119559 10b3645c0 Options.allow_concurrent_memtable_write: 1
2019/08/10-17:00:28.119559 10b3645c0 Options.enable_write_thread_adaptive_yield: 1
2019/08/10-17:00:28.119580 10b3645c0 Options.write_thread_max_yield_usec: 100
2019/08/10-17:00:28.119582 10b3645c0 Options.write_thread_slow_yield_usec: 3
2019/08/10-17:00:28.119583 10b3645c0 Options.row_cache: None
2019/08/10-17:00:28.119583 10b3645c0 Options.wal_filter: None
2019/08/10-17:00:28.119584 10b3645c0 Options.avoid_flush_during_recovery: 0
2019/08/10-17:00:28.119585 10b3645c0 Options.allow_ingest_behind: 0
2019/08/10-17:00:28.119586 10b3645c0 Options.preserve_deletes: 0
2019/08/10-17:00:28.119587 10b3645c0 Options.two_write_queues: 0
2019/08/10-17:00:28.119588 10b3645c0 Options.manual_wal_flush: 0
2019/08/10-17:00:28.119588 10b3645c0 Options.max_background_jobs: 2
2019/08/10-17:00:28.119589 10b3645c0 Options.max_background_compactions: -1
2019/08/10-17:00:28.119590 10b3645c0 Options.avoid_flush_during_shutdown: 0
2019/08/10-17:00:28.119591 10b3645c0 Options.writable_file_max_buffer_size: 1048576
2019/08/10-17:00:28.119592 10b3645c0 Options.delayed_write_rate : 16777216
2019/08/10-17:00:28.119593 10b3645c0 Options.max_total_wal_size: 0
2019/08/10-17:00:28.119594 10b3645c0 Options.delete_obsolete_files_period_micros: 21600000000
2019/08/10-17:00:28.119594 10b3645c0 Options.stats_dump_period_sec: 600
2019/08/10-17:00:28.119595 10b3645c0 Options.max_open_files: 256
2019/08/10-17:00:28.119596 10b3645c0 Options.bytes_per_sync: 0
2019/08/10-17:00:28.119597 10b3645c0 Options.wal_bytes_per_sync: 0
2019/08/10-17:00:28.119598 10b3645c0 Options.compaction_readahead_size: 0
2019/08/10-17:00:28.119599 10b3645c0 Compression algorithms supported:
2019/08/10-17:00:28.119600 10b3645c0 kZSTD supported: 0
2019/08/10-17:00:28.119601 10b3645c0 kZlibCompression supported: 1
2019/08/10-17:00:28.119602 10b3645c0 kXpressCompression supported: 0
2019/08/10-17:00:28.119603 10b3645c0 kSnappyCompression supported: 1
2019/08/10-17:00:28.119604 10b3645c0 kZSTDNotFinalCompression supported: 0
2019/08/10-17:00:28.119604 10b3645c0 kLZ4HCCompression supported: 1
2019/08/10-17:00:28.119605 10b3645c0 kLZ4Compression supported: 1
2019/08/10-17:00:28.119606 10b3645c0 kBZip2Compression supported: 1
2019/08/10-17:00:28.119607 10b3645c0 Fast CRC32 supported: Supported on x86
2019/08/10-17:00:28.119723 10b3645c0 [/db_impl_open.cc:226] Creating manifest 1
2019/08/10-17:00:28.121131 10b3645c0 [/version_set.cc:3508] Recovering from manifest file: MANIFEST-000001
2019/08/10-17:00:28.121208 10b3645c0 [/column_family.cc:474] --------------- Options for column family [default]:
2019/08/10-17:00:28.121214 10b3645c0 Options.comparator: leveldb.BytewiseComparator
2019/08/10-17:00:28.121215 10b3645c0 Options.merge_operator: None
2019/08/10-17:00:28.121216 10b3645c0 Options.compaction_filter: None
2019/08/10-17:00:28.121217 10b3645c0 Options.compaction_filter_factory: None
2019/08/10-17:00:28.121218 10b3645c0 Options.memtable_factory: SkipListFactory
2019/08/10-17:00:28.121219 10b3645c0 Options.table_factory: BlockBasedTable
2019/08/10-17:00:28.121234 10b3645c0 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7fb58fc07070)
cache_index_and_filter_blocks: 0
cache_index_and_filter_blocks_with_high_priority: 0
pin_l0_filter_and_index_blocks_in_cache: 0
pin_top_level_index_and_filter: 1
index_type: 0
hash_index_allow_collision: 1
checksum: 1
no_block_cache: 0
block_cache: 0x7fb58fc07e78
block_cache_name: LRUCache
block_cache_options:
capacity : 8388608
num_shard_bits : 4
strict_capacity_limit : 0
memory_allocator : None
high_pri_pool_ratio: 0.000
block_cache_compressed: 0x0
persistent_cache: 0x0
block_size: 4096
block_size_deviation: 10
block_restart_interval: 16
index_block_restart_interval: 1
metadata_block_size: 4096
partition_filters: 0
use_delta_encoding: 1
filter_policy: nullptr
whole_key_filtering: 1
verify_compression: 0
read_amp_bytes_per_bit: 0
format_version: 2
enable_index_compression: 1
block_align: 0
2019/08/10-17:00:28.121260 10b3645c0 Options.write_buffer_size: 4194304
2019/08/10-17:00:28.121262 10b3645c0 Options.max_write_buffer_number: 2
2019/08/10-17:00:28.121265 10b3645c0 Options.compression: Snappy
2019/08/10-17:00:28.121266 10b3645c0 Options.bottommost_compression: Disabled
2019/08/10-17:00:28.121267 10b3645c0 Options.prefix_extractor: nullptr
2019/08/10-17:00:28.121268 10b3645c0 Options.memtable_insert_with_hint_prefix_extractor: nullptr
2019/08/10-17:00:28.121268 10b3645c0 Options.num_levels: 7
2019/08/10-17:00:28.121269 10b3645c0 Options.min_write_buffer_number_to_merge: 1
2019/08/10-17:00:28.121270 10b3645c0 Options.max_write_buffer_number_to_maintain: 0
2019/08/10-17:00:28.121271 10b3645c0 Options.bottommost_compression_opts.window_bits: -14
2019/08/10-17:00:28.121272 10b3645c0 Options.bottommost_compression_opts.level: 32767
2019/08/10-17:00:28.121273 10b3645c0 Options.bottommost_compression_opts.strategy: 0
2019/08/10-17:00:28.121273 10b3645c0 Options.bottommost_compression_opts.max_dict_bytes: 0
2019/08/10-17:00:28.121274 10b3645c0 Options.bottommost_compression_opts.zstd_max_train_bytes: 0
2019/08/10-17:00:28.121275 10b3645c0 Options.bottommost_compression_opts.enabled: false
2019/08/10-17:00:28.121276 10b3645c0 Options.compression_opts.window_bits: -14
2019/08/10-17:00:28.121277 10b3645c0 Options.compression_opts.level: 32767
2019/08/10-17:00:28.121277 10b3645c0 Options.compression_opts.strategy: 0
2019/08/10-17:00:28.121278 10b3645c0 Options.compression_opts.max_dict_bytes: 0
2019/08/10-17:00:28.121279 10b3645c0 Options.compression_opts.zstd_max_train_bytes: 0
2019/08/10-17:00:28.121280 10b3645c0 Options.compression_opts.enabled: false
2019/08/10-17:00:28.121281 10b3645c0 Options.level0_file_num_compaction_trigger: 4
2019/08/10-17:00:28.121281 10b3645c0 Options.level0_slowdown_writes_trigger: 20
2019/08/10-17:00:28.121282 10b3645c0 Options.level0_stop_writes_trigger: 36
2019/08/10-17:00:28.121283 10b3645c0 Options.target_file_size_base: 67108864
2019/08/10-17:00:28.121284 10b3645c0 Options.target_file_size_multiplier: 1
2019/08/10-17:00:28.121284 10b3645c0 Options.max_bytes_for_level_base: 268435456
2019/08/10-17:00:28.121285 10b3645c0 Options.level_compaction_dynamic_level_bytes: 0
2019/08/10-17:00:28.121286 10b3645c0 Options.max_bytes_for_level_multiplier: 10.000000
2019/08/10-17:00:28.121292 10b3645c0 Options.max_bytes_for_level_multiplier_addtl[0]: 1
2019/08/10-17:00:28.121293 10b3645c0 Options.max_bytes_for_level_multiplier_addtl[1]: 1
2019/08/10-17:00:28.121294 10b3645c0 Options.max_bytes_for_level_multiplier_addtl[2]: 1
2019/08/10-17:00:28.121295 10b3645c0 Options.max_bytes_for_level_multiplier_addtl[3]: 1
2019/08/10-17:00:28.121296 10b3645c0 Options.max_bytes_for_level_multiplier_addtl[4]: 1
2019/08/10-17:00:28.121296 10b3645c0 Options.max_bytes_for_level_multiplier_addtl[5]: 1
2019/08/10-17:00:28.121297 10b3645c0 Options.max_bytes_for_level_multiplier_addtl[6]: 1
2019/08/10-17:00:28.121298 10b3645c0 Options.max_sequential_skip_in_iterations: 8
2019/08/10-17:00:28.121299 10b3645c0 Options.max_compaction_bytes: 1677721600
2019/08/10-17:00:28.121300 10b3645c0 Options.arena_block_size: 524288
2019/08/10-17:00:28.121300 10b3645c0 Options.soft_pending_compaction_bytes_limit: 68719476736
2019/08/10-17:00:28.121301 10b3645c0 Options.hard_pending_compaction_bytes_limit: 274877906944
2019/08/10-17:00:28.121302 10b3645c0 Options.rate_limit_delay_max_milliseconds: 100
2019/08/10-17:00:28.121303 10b3645c0 Options.disable_auto_compactions: 0
2019/08/10-17:00:28.121331 10b3645c0 Options.compaction_style: kCompactionStyleLevel
2019/08/10-17:00:28.121334 10b3645c0 Options.compaction_pri: kByCompensatedSize
2019/08/10-17:00:28.121336 10b3645c0 Options.compaction_options_universal.size_ratio: 1
2019/08/10-17:00:28.121336 10b3645c0 Options.compaction_options_universal.min_merge_width: 2
2019/08/10-17:00:28.121337 10b3645c0 Options.compaction_options_universal.max_merge_width: 4294967295
2019/08/10-17:00:28.121338 10b3645c0 Options.compaction_options_universal.max_size_amplification_percent: 200
2019/08/10-17:00:28.121339 10b3645c0 Options.compaction_options_universal.compression_size_percent: -1
2019/08/10-17:00:28.121341 10b3645c0 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize
2019/08/10-17:00:28.121341 10b3645c0 Options.compaction_options_fifo.max_table_files_size: 1073741824
2019/08/10-17:00:28.121342 10b3645c0 Options.compaction_options_fifo.allow_compaction: 0
2019/08/10-17:00:28.121343 10b3645c0 Options.compaction_options_fifo.ttl: 0
2019/08/10-17:00:28.121344 10b3645c0 Options.table_properties_collectors:
2019/08/10-17:00:28.121345 10b3645c0 Options.inplace_update_support: 0
2019/08/10-17:00:28.121346 10b3645c0 Options.inplace_update_num_locks: 10000
2019/08/10-17:00:28.121346 10b3645c0 Options.memtable_prefix_bloom_size_ratio: 0.000000
2019/08/10-17:00:28.121348 10b3645c0 Options.memtable_huge_page_size: 0
2019/08/10-17:00:28.121348 10b3645c0 Options.bloom_locality: 0
2019/08/10-17:00:28.121349 10b3645c0 Options.max_successive_merges: 0
2019/08/10-17:00:28.121350 10b3645c0 Options.optimize_filters_for_hits: 0
2019/08/10-17:00:28.121351 10b3645c0 Options.paranoid_file_checks: 0
2019/08/10-17:00:28.121352 10b3645c0 Options.force_consistency_checks: 0
2019/08/10-17:00:28.121352 10b3645c0 Options.report_bg_io_stats: 0
2019/08/10-17:00:28.121353 10b3645c0 Options.ttl: 0
2019/08/10-17:00:28.121419 10b3645c0 [/version_set.cc:3724] Recovered from manifest file:catalogue/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0
2019/08/10-17:00:28.121422 10b3645c0 [/version_set.cc:3732] Column family [default] (ID 0), log number is 0
2019/08/10-17:00:28.124295 10b3645c0 [/db_impl_open.cc:1314] DB pointer 0x7fb590005200
2019/08/10-17:00:28.124449 700002b73000 [WARN] [/db_impl.cc:669] ------- DUMPING STATS -------
2019/08/10-17:00:28.124454 700002b73000 [WARN] [/db_impl.cc:670]
** DB Stats **
Uptime(secs): 0.0 total, 0.0 interval
Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s
Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s
Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent
Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s
Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s
Interval stall: 00:00:0.000 H:M:S, 0.0 percent
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
----------------------------------------------------------------------------------------------------------------------------------------------------------
Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0
Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0
Uptime(secs): 0.0 total, 0.0 interval
Flush(GB): cumulative 0.000, interval 0.000
AddFile(GB): cumulative 0.000, interval 0.000
AddFile(Total Files): cumulative 0, interval 0
AddFile(L0 Files): cumulative 0, interval 0
AddFile(Keys): cumulative 0, interval 0
Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
** File Read Latency Histogram By Level [default] **
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
----------------------------------------------------------------------------------------------------------------------------------------------------------
Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0
Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0
Uptime(secs): 0.0 total, 0.0 interval
Flush(GB): cumulative 0.000, interval 0.000
AddFile(GB): cumulative 0.000, interval 0.000
AddFile(Total Files): cumulative 0, interval 0
AddFile(L0 Files): cumulative 0, interval 0
AddFile(Keys): cumulative 0, interval 0
Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
** File Read Latency Histogram By Level [default] **

240
catalogue/LOG.old.1565431361639591 Normal file
View File

@ -0,0 +1,240 @@
2019/08/10-17:01:09.861388 11765d5c0 RocksDB version: 5.18.3
2019/08/10-17:01:09.861825 11765d5c0 Git sha rocksdb_build_git_sha:
2019/08/10-17:01:09.861828 11765d5c0 Compile date Feb 21 2019
2019/08/10-17:01:09.861829 11765d5c0 DB SUMMARY
2019/08/10-17:01:09.861895 11765d5c0 CURRENT file: CURRENT
2019/08/10-17:01:09.861897 11765d5c0 IDENTITY file: IDENTITY
2019/08/10-17:01:09.861903 11765d5c0 MANIFEST file: MANIFEST-000001 size: 13 Bytes
2019/08/10-17:01:09.861905 11765d5c0 SST files in catalogue dir, Total Num: 0, files:
2019/08/10-17:01:09.861906 11765d5c0 Write Ahead Log file in catalogue: 000003.log size: 0 ;
2019/08/10-17:01:09.861908 11765d5c0 Options.error_if_exists: 0
2019/08/10-17:01:09.861911 11765d5c0 Options.create_if_missing: 1
2019/08/10-17:01:09.861912 11765d5c0 Options.paranoid_checks: 0
2019/08/10-17:01:09.861913 11765d5c0 Options.env: 0x10c9f22e8
2019/08/10-17:01:09.861914 11765d5c0 Options.info_log: 0x7f9d42d02030
2019/08/10-17:01:09.861915 11765d5c0 Options.max_file_opening_threads: 16
2019/08/10-17:01:09.861916 11765d5c0 Options.statistics: 0x0
2019/08/10-17:01:09.861917 11765d5c0 Options.use_fsync: 0
2019/08/10-17:01:09.861918 11765d5c0 Options.max_log_file_size: 0
2019/08/10-17:01:09.861919 11765d5c0 Options.max_manifest_file_size: 1073741824
2019/08/10-17:01:09.861920 11765d5c0 Options.log_file_time_to_roll: 0
2019/08/10-17:01:09.861921 11765d5c0 Options.keep_log_file_num: 1000
2019/08/10-17:01:09.861922 11765d5c0 Options.recycle_log_file_num: 0
2019/08/10-17:01:09.861923 11765d5c0 Options.allow_fallocate: 1
2019/08/10-17:01:09.861924 11765d5c0 Options.allow_mmap_reads: 0
2019/08/10-17:01:09.861925 11765d5c0 Options.allow_mmap_writes: 0
2019/08/10-17:01:09.861926 11765d5c0 Options.use_direct_reads: 0
2019/08/10-17:01:09.861927 11765d5c0 Options.use_direct_io_for_flush_and_compaction: 0
2019/08/10-17:01:09.861927 11765d5c0 Options.create_missing_column_families: 0
2019/08/10-17:01:09.861928 11765d5c0 Options.db_log_dir:
2019/08/10-17:01:09.861929 11765d5c0 Options.wal_dir: catalogue
2019/08/10-17:01:09.861930 11765d5c0 Options.table_cache_numshardbits: 6
2019/08/10-17:01:09.861931 11765d5c0 Options.max_subcompactions: 1
2019/08/10-17:01:09.861932 11765d5c0 Options.max_background_flushes: -1
2019/08/10-17:01:09.861933 11765d5c0 Options.WAL_ttl_seconds: 0
2019/08/10-17:01:09.861934 11765d5c0 Options.WAL_size_limit_MB: 0
2019/08/10-17:01:09.861935 11765d5c0 Options.manifest_preallocation_size: 4194304
2019/08/10-17:01:09.861936 11765d5c0 Options.is_fd_close_on_exec: 1
2019/08/10-17:01:09.861937 11765d5c0 Options.advise_random_on_open: 1
2019/08/10-17:01:09.861938 11765d5c0 Options.db_write_buffer_size: 0
2019/08/10-17:01:09.861939 11765d5c0 Options.write_buffer_manager: 0x7f9d42d01cf0
2019/08/10-17:01:09.861940 11765d5c0 Options.access_hint_on_compaction_start: 1
2019/08/10-17:01:09.861941 11765d5c0 Options.new_table_reader_for_compaction_inputs: 0
2019/08/10-17:01:09.861942 11765d5c0 Options.random_access_max_buffer_size: 1048576
2019/08/10-17:01:09.861942 11765d5c0 Options.use_adaptive_mutex: 0
2019/08/10-17:01:09.861943 11765d5c0 Options.rate_limiter: 0x0
2019/08/10-17:01:09.861945 11765d5c0 Options.sst_file_manager.rate_bytes_per_sec: 0
2019/08/10-17:01:09.861946 11765d5c0 Options.wal_recovery_mode: 2
2019/08/10-17:01:09.861947 11765d5c0 Options.enable_thread_tracking: 0
2019/08/10-17:01:09.861948 11765d5c0 Options.enable_pipelined_write: 0
2019/08/10-17:01:09.861970 11765d5c0 Options.allow_concurrent_memtable_write: 1
2019/08/10-17:01:09.861972 11765d5c0 Options.enable_write_thread_adaptive_yield: 1
2019/08/10-17:01:09.861973 11765d5c0 Options.write_thread_max_yield_usec: 100
2019/08/10-17:01:09.861974 11765d5c0 Options.write_thread_slow_yield_usec: 3
2019/08/10-17:01:09.861975 11765d5c0 Options.row_cache: None
2019/08/10-17:01:09.861976 11765d5c0 Options.wal_filter: None
2019/08/10-17:01:09.861977 11765d5c0 Options.avoid_flush_during_recovery: 0
2019/08/10-17:01:09.861978 11765d5c0 Options.allow_ingest_behind: 0
2019/08/10-17:01:09.861978 11765d5c0 Options.preserve_deletes: 0
2019/08/10-17:01:09.861979 11765d5c0 Options.two_write_queues: 0
2019/08/10-17:01:09.861980 11765d5c0 Options.manual_wal_flush: 0
2019/08/10-17:01:09.861981 11765d5c0 Options.max_background_jobs: 2
2019/08/10-17:01:09.861982 11765d5c0 Options.max_background_compactions: -1
2019/08/10-17:01:09.861983 11765d5c0 Options.avoid_flush_during_shutdown: 0
2019/08/10-17:01:09.861984 11765d5c0 Options.writable_file_max_buffer_size: 1048576
2019/08/10-17:01:09.861985 11765d5c0 Options.delayed_write_rate : 16777216
2019/08/10-17:01:09.861986 11765d5c0 Options.max_total_wal_size: 0
2019/08/10-17:01:09.861987 11765d5c0 Options.delete_obsolete_files_period_micros: 21600000000
2019/08/10-17:01:09.861988 11765d5c0 Options.stats_dump_period_sec: 600
2019/08/10-17:01:09.861989 11765d5c0 Options.max_open_files: 256
2019/08/10-17:01:09.861990 11765d5c0 Options.bytes_per_sync: 0
2019/08/10-17:01:09.861991 11765d5c0 Options.wal_bytes_per_sync: 0
2019/08/10-17:01:09.861991 11765d5c0 Options.compaction_readahead_size: 0
2019/08/10-17:01:09.861992 11765d5c0 Compression algorithms supported:
2019/08/10-17:01:09.861993 11765d5c0 kZSTD supported: 0
2019/08/10-17:01:09.861994 11765d5c0 kZlibCompression supported: 1
2019/08/10-17:01:09.861996 11765d5c0 kXpressCompression supported: 0
2019/08/10-17:01:09.861997 11765d5c0 kSnappyCompression supported: 1
2019/08/10-17:01:09.861997 11765d5c0 kZSTDNotFinalCompression supported: 0
2019/08/10-17:01:09.861998 11765d5c0 kLZ4HCCompression supported: 1
2019/08/10-17:01:09.861999 11765d5c0 kLZ4Compression supported: 1
2019/08/10-17:01:09.862000 11765d5c0 kBZip2Compression supported: 1
2019/08/10-17:01:09.862004 11765d5c0 Fast CRC32 supported: Supported on x86
2019/08/10-17:01:09.862180 11765d5c0 [/version_set.cc:3508] Recovering from manifest file: MANIFEST-000001
2019/08/10-17:01:09.862231 11765d5c0 [/column_family.cc:474] --------------- Options for column family [default]:
2019/08/10-17:01:09.862236 11765d5c0 Options.comparator: leveldb.BytewiseComparator
2019/08/10-17:01:09.862237 11765d5c0 Options.merge_operator: None
2019/08/10-17:01:09.862238 11765d5c0 Options.compaction_filter: None
2019/08/10-17:01:09.862239 11765d5c0 Options.compaction_filter_factory: None
2019/08/10-17:01:09.862240 11765d5c0 Options.memtable_factory: SkipListFactory
2019/08/10-17:01:09.862241 11765d5c0 Options.table_factory: BlockBasedTable
2019/08/10-17:01:09.862263 11765d5c0 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7f9d42d01160)
cache_index_and_filter_blocks: 0
cache_index_and_filter_blocks_with_high_priority: 0
pin_l0_filter_and_index_blocks_in_cache: 0
pin_top_level_index_and_filter: 1
index_type: 0
hash_index_allow_collision: 1
checksum: 1
no_block_cache: 0
block_cache: 0x7f9d42d011a8
block_cache_name: LRUCache
block_cache_options:
capacity : 8388608
num_shard_bits : 4
strict_capacity_limit : 0
memory_allocator : None
high_pri_pool_ratio: 0.000
block_cache_compressed: 0x0
persistent_cache: 0x0
block_size: 4096
block_size_deviation: 10
block_restart_interval: 16
index_block_restart_interval: 1
metadata_block_size: 4096
partition_filters: 0
use_delta_encoding: 1
filter_policy: nullptr
whole_key_filtering: 1
verify_compression: 0
read_amp_bytes_per_bit: 0
format_version: 2
enable_index_compression: 1
block_align: 0
2019/08/10-17:01:09.862291 11765d5c0 Options.write_buffer_size: 4194304
2019/08/10-17:01:09.862293 11765d5c0 Options.max_write_buffer_number: 2
2019/08/10-17:01:09.862296 11765d5c0 Options.compression: Snappy
2019/08/10-17:01:09.862297 11765d5c0 Options.bottommost_compression: Disabled
2019/08/10-17:01:09.862298 11765d5c0 Options.prefix_extractor: nullptr
2019/08/10-17:01:09.862299 11765d5c0 Options.memtable_insert_with_hint_prefix_extractor: nullptr
2019/08/10-17:01:09.862300 11765d5c0 Options.num_levels: 7
2019/08/10-17:01:09.862301 11765d5c0 Options.min_write_buffer_number_to_merge: 1
2019/08/10-17:01:09.862302 11765d5c0 Options.max_write_buffer_number_to_maintain: 0
2019/08/10-17:01:09.862303 11765d5c0 Options.bottommost_compression_opts.window_bits: -14
2019/08/10-17:01:09.862304 11765d5c0 Options.bottommost_compression_opts.level: 32767
2019/08/10-17:01:09.862305 11765d5c0 Options.bottommost_compression_opts.strategy: 0
2019/08/10-17:01:09.862306 11765d5c0 Options.bottommost_compression_opts.max_dict_bytes: 0
2019/08/10-17:01:09.862307 11765d5c0 Options.bottommost_compression_opts.zstd_max_train_bytes: 0
2019/08/10-17:01:09.862308 11765d5c0 Options.bottommost_compression_opts.enabled: false
2019/08/10-17:01:09.862309 11765d5c0 Options.compression_opts.window_bits: -14
2019/08/10-17:01:09.862310 11765d5c0 Options.compression_opts.level: 32767
2019/08/10-17:01:09.862311 11765d5c0 Options.compression_opts.strategy: 0
2019/08/10-17:01:09.862311 11765d5c0 Options.compression_opts.max_dict_bytes: 0
2019/08/10-17:01:09.862312 11765d5c0 Options.compression_opts.zstd_max_train_bytes: 0
2019/08/10-17:01:09.862313 11765d5c0 Options.compression_opts.enabled: false
2019/08/10-17:01:09.862314 11765d5c0 Options.level0_file_num_compaction_trigger: 4
2019/08/10-17:01:09.862315 11765d5c0 Options.level0_slowdown_writes_trigger: 20
2019/08/10-17:01:09.862316 11765d5c0 Options.level0_stop_writes_trigger: 36
2019/08/10-17:01:09.862317 11765d5c0 Options.target_file_size_base: 67108864
2019/08/10-17:01:09.862318 11765d5c0 Options.target_file_size_multiplier: 1
2019/08/10-17:01:09.862319 11765d5c0 Options.max_bytes_for_level_base: 268435456
2019/08/10-17:01:09.862320 11765d5c0 Options.level_compaction_dynamic_level_bytes: 0
2019/08/10-17:01:09.862321 11765d5c0 Options.max_bytes_for_level_multiplier: 10.000000
2019/08/10-17:01:09.862323 11765d5c0 Options.max_bytes_for_level_multiplier_addtl[0]: 1
2019/08/10-17:01:09.862324 11765d5c0 Options.max_bytes_for_level_multiplier_addtl[1]: 1
2019/08/10-17:01:09.862325 11765d5c0 Options.max_bytes_for_level_multiplier_addtl[2]: 1
2019/08/10-17:01:09.862326 11765d5c0 Options.max_bytes_for_level_multiplier_addtl[3]: 1
2019/08/10-17:01:09.862327 11765d5c0 Options.max_bytes_for_level_multiplier_addtl[4]: 1
2019/08/10-17:01:09.862327 11765d5c0 Options.max_bytes_for_level_multiplier_addtl[5]: 1
2019/08/10-17:01:09.862328 11765d5c0 Options.max_bytes_for_level_multiplier_addtl[6]: 1
2019/08/10-17:01:09.862329 11765d5c0 Options.max_sequential_skip_in_iterations: 8
2019/08/10-17:01:09.862330 11765d5c0 Options.max_compaction_bytes: 1677721600
2019/08/10-17:01:09.862331 11765d5c0 Options.arena_block_size: 524288
2019/08/10-17:01:09.862332 11765d5c0 Options.soft_pending_compaction_bytes_limit: 68719476736
2019/08/10-17:01:09.862333 11765d5c0 Options.hard_pending_compaction_bytes_limit: 274877906944
2019/08/10-17:01:09.862349 11765d5c0 Options.rate_limit_delay_max_milliseconds: 100
2019/08/10-17:01:09.862351 11765d5c0 Options.disable_auto_compactions: 0
2019/08/10-17:01:09.862352 11765d5c0 Options.compaction_style: kCompactionStyleLevel
2019/08/10-17:01:09.862353 11765d5c0 Options.compaction_pri: kByCompensatedSize
2019/08/10-17:01:09.862354 11765d5c0 Options.compaction_options_universal.size_ratio: 1
2019/08/10-17:01:09.862355 11765d5c0 Options.compaction_options_universal.min_merge_width: 2
2019/08/10-17:01:09.862356 11765d5c0 Options.compaction_options_universal.max_merge_width: 4294967295
2019/08/10-17:01:09.862357 11765d5c0 Options.compaction_options_universal.max_size_amplification_percent: 200
2019/08/10-17:01:09.862358 11765d5c0 Options.compaction_options_universal.compression_size_percent: -1
2019/08/10-17:01:09.862359 11765d5c0 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize
2019/08/10-17:01:09.862360 11765d5c0 Options.compaction_options_fifo.max_table_files_size: 1073741824
2019/08/10-17:01:09.862361 11765d5c0 Options.compaction_options_fifo.allow_compaction: 0
2019/08/10-17:01:09.862362 11765d5c0 Options.compaction_options_fifo.ttl: 0
2019/08/10-17:01:09.862363 11765d5c0 Options.table_properties_collectors:
2019/08/10-17:01:09.862364 11765d5c0 Options.inplace_update_support: 0
2019/08/10-17:01:09.862365 11765d5c0 Options.inplace_update_num_locks: 10000
2019/08/10-17:01:09.862366 11765d5c0 Options.memtable_prefix_bloom_size_ratio: 0.000000
2019/08/10-17:01:09.862367 11765d5c0 Options.memtable_huge_page_size: 0
2019/08/10-17:01:09.862368 11765d5c0 Options.bloom_locality: 0
2019/08/10-17:01:09.862369 11765d5c0 Options.max_successive_merges: 0
2019/08/10-17:01:09.862369 11765d5c0 Options.optimize_filters_for_hits: 0
2019/08/10-17:01:09.862370 11765d5c0 Options.paranoid_file_checks: 0
2019/08/10-17:01:09.862371 11765d5c0 Options.force_consistency_checks: 0
2019/08/10-17:01:09.862372 11765d5c0 Options.report_bg_io_stats: 0
2019/08/10-17:01:09.862373 11765d5c0 Options.ttl: 0
2019/08/10-17:01:09.862443 11765d5c0 [/version_set.cc:3724] Recovered from manifest file:catalogue/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0
2019/08/10-17:01:09.862446 11765d5c0 [/version_set.cc:3732] Column family [default] (ID 0), log number is 0
2019/08/10-17:01:09.862535 11765d5c0 EVENT_LOG_v1 {"time_micros": 1565431269862527, "job": 1, "event": "recovery_started", "log_files": [3]}
2019/08/10-17:01:09.862544 11765d5c0 [/db_impl_open.cc:578] Recovering log #3 mode 2
2019/08/10-17:01:09.862597 11765d5c0 [/version_set.cc:3037] Creating manifest 5
2019/08/10-17:01:09.863284 11765d5c0 EVENT_LOG_v1 {"time_micros": 1565431269863280, "job": 1, "event": "recovery_finished"}
2019/08/10-17:01:09.866266 11765d5c0 [/db_impl_open.cc:1314] DB pointer 0x7f9d44002a00
2019/08/10-17:01:09.866451 70000bd0e000 [WARN] [/db_impl.cc:669] ------- DUMPING STATS -------
2019/08/10-17:01:09.866460 70000bd0e000 [WARN] [/db_impl.cc:670]
** DB Stats **
Uptime(secs): 0.0 total, 0.0 interval
Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s
Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s
Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent
Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s
Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s
Interval stall: 00:00:0.000 H:M:S, 0.0 percent
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
----------------------------------------------------------------------------------------------------------------------------------------------------------
Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0
Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0
Uptime(secs): 0.0 total, 0.0 interval
Flush(GB): cumulative 0.000, interval 0.000
AddFile(GB): cumulative 0.000, interval 0.000
AddFile(Total Files): cumulative 0, interval 0
AddFile(L0 Files): cumulative 0, interval 0
AddFile(Keys): cumulative 0, interval 0
Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
** File Read Latency Histogram By Level [default] **
** Compaction Stats [default] **
Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
----------------------------------------------------------------------------------------------------------------------------------------------------------
Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0
Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0
Uptime(secs): 0.0 total, 0.0 interval
Flush(GB): cumulative 0.000, interval 0.000
AddFile(GB): cumulative 0.000, interval 0.000
AddFile(Total Files): cumulative 0, interval 0
AddFile(L0 Files): cumulative 0, interval 0
AddFile(Keys): cumulative 0, interval 0
Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
** File Read Latency Histogram By Level [default] **

BIN
catalogue/MANIFEST-000008 Normal file
View File

Binary file not shown.

150
catalogue/OPTIONS-000008 Normal file
View File

@ -0,0 +1,150 @@
# This is a RocksDB option file.
#
# For detailed file format spec, please refer to the example file
# in examples/rocksdb_option_file_example.ini
#
[Version]
rocksdb_version=5.18.3
options_file_version=1.1
[DBOptions]
allow_mmap_writes=false
base_background_compactions=-1
new_table_reader_for_compaction_inputs=false
db_log_dir=
wal_recovery_mode=kPointInTimeRecovery
use_direct_reads=false
write_thread_max_yield_usec=100
max_manifest_file_size=1073741824
allow_2pc=false
allow_fallocate=true
fail_if_options_file_error=false
allow_ingest_behind=false
allow_mmap_reads=false
skip_log_error_on_recovery=false
recycle_log_file_num=0
delete_obsolete_files_period_micros=21600000000
compaction_readahead_size=0
use_direct_io_for_flush_and_compaction=false
log_file_time_to_roll=0
create_missing_column_families=false
advise_random_on_open=true
max_log_file_size=0
stats_dump_period_sec=600
enable_thread_tracking=false
use_adaptive_mutex=false
create_if_missing=true
is_fd_close_on_exec=true
max_background_flushes=-1
manifest_preallocation_size=4194304
error_if_exists=false
skip_stats_update_on_db_open=false
max_open_files=256
random_access_max_buffer_size=1048576
use_fsync=false
max_background_jobs=2
two_write_queues=false
max_background_compactions=-1
max_file_opening_threads=16
table_cache_numshardbits=6
keep_log_file_num=1000
avoid_flush_during_shutdown=false
db_write_buffer_size=0
max_total_wal_size=0
wal_dir=catalogue
max_subcompactions=1
atomic_flush=false
WAL_size_limit_MB=0
paranoid_checks=false
allow_concurrent_memtable_write=true
writable_file_max_buffer_size=1048576
WAL_ttl_seconds=0
delayed_write_rate=16777216
bytes_per_sync=0
wal_bytes_per_sync=0
enable_pipelined_write=false
enable_write_thread_adaptive_yield=true
write_thread_slow_yield_usec=3
access_hint_on_compaction_start=NORMAL
info_log_level=INFO_LEVEL
dump_malloc_stats=false
avoid_flush_during_recovery=false
preserve_deletes=false
manual_wal_flush=false
[CFOptions "default"]
report_bg_io_stats=false
inplace_update_support=false
max_compaction_bytes=1677721600
disable_auto_compactions=false
write_buffer_size=4194304
bloom_locality=0
max_bytes_for_level_multiplier=10.000000
compaction_filter_factory=nullptr
optimize_filters_for_hits=false
target_file_size_base=67108864
max_write_buffer_number_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
paranoid_file_checks=false
memtable_prefix_bloom_size_ratio=0.000000
force_consistency_checks=false
max_write_buffer_number=2
ttl=0
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
level0_slowdown_writes_trigger=20
level_compaction_dynamic_level_bytes=false
compaction_options_fifo={allow_compaction=false;ttl=0;max_table_files_size=1073741824;}
inplace_update_num_locks=10000
level0_file_num_compaction_trigger=4
compression=kSnappyCompression
level0_stop_writes_trigger=36
num_levels=7
table_factory=BlockBasedTable
compression_per_level=
target_file_size_multiplier=1
min_write_buffer_number_to_merge=1
arena_block_size=524288
max_successive_merges=0
memtable_huge_page_size=0
compaction_pri=kByCompensatedSize
soft_pending_compaction_bytes_limit=68719476736
max_bytes_for_level_base=268435456
comparator=leveldb.BytewiseComparator
max_sequential_skip_in_iterations=8
bottommost_compression=kDisableCompressionOption
prefix_extractor=nullptr
memtable_insert_with_hint_prefix_extractor=nullptr
memtable_factory=SkipListFactory
compaction_filter=nullptr
compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;}
merge_operator=nullptr
compaction_style=kCompactionStyleLevel
[TableOptions/BlockBasedTable "default"]
format_version=2
whole_key_filtering=true
verify_compression=false
partition_filters=false
enable_index_compression=true
checksum=kCRC32c
index_block_restart_interval=1
pin_top_level_index_and_filter=true
block_align=false
block_size=4096
index_type=kBinarySearch
filter_policy=nullptr
metadata_block_size=4096
no_block_cache=false
block_size_deviation=10
data_block_index_type=kDataBlockBinarySearch
data_block_hash_table_util_ratio=0.750000
read_amp_bytes_per_bit=8589934592
cache_index_and_filter_blocks=false
block_restart_interval=16
pin_l0_filter_and_index_blocks_in_cache=false
hash_index_allow_collision=true
cache_index_and_filter_blocks_with_high_priority=false
flush_block_policy_factory=FlushBlockBySizePolicyFactory

150
catalogue/OPTIONS-000011 Normal file
View File

@ -0,0 +1,150 @@
# This is a RocksDB option file.
#
# For detailed file format spec, please refer to the example file
# in examples/rocksdb_option_file_example.ini
#
[Version]
rocksdb_version=5.18.3
options_file_version=1.1
[DBOptions]
allow_mmap_writes=false
base_background_compactions=-1
new_table_reader_for_compaction_inputs=false
db_log_dir=
wal_recovery_mode=kPointInTimeRecovery
use_direct_reads=false
write_thread_max_yield_usec=100
max_manifest_file_size=1073741824
allow_2pc=false
allow_fallocate=true
fail_if_options_file_error=false
allow_ingest_behind=false
allow_mmap_reads=false
skip_log_error_on_recovery=false
recycle_log_file_num=0
delete_obsolete_files_period_micros=21600000000
compaction_readahead_size=0
use_direct_io_for_flush_and_compaction=false
log_file_time_to_roll=0
create_missing_column_families=false
advise_random_on_open=true
max_log_file_size=0
stats_dump_period_sec=600
enable_thread_tracking=false
use_adaptive_mutex=false
create_if_missing=true
is_fd_close_on_exec=true
max_background_flushes=-1
manifest_preallocation_size=4194304
error_if_exists=false
skip_stats_update_on_db_open=false
max_open_files=256
random_access_max_buffer_size=1048576
use_fsync=false
max_background_jobs=2
two_write_queues=false
max_background_compactions=-1
max_file_opening_threads=16
table_cache_numshardbits=6
keep_log_file_num=1000
avoid_flush_during_shutdown=false
db_write_buffer_size=0
max_total_wal_size=0
wal_dir=catalogue
max_subcompactions=1
atomic_flush=false
WAL_size_limit_MB=0
paranoid_checks=false
allow_concurrent_memtable_write=true
writable_file_max_buffer_size=1048576
WAL_ttl_seconds=0
delayed_write_rate=16777216
bytes_per_sync=0
wal_bytes_per_sync=0
enable_pipelined_write=false
enable_write_thread_adaptive_yield=true
write_thread_slow_yield_usec=3
access_hint_on_compaction_start=NORMAL
info_log_level=INFO_LEVEL
dump_malloc_stats=false
avoid_flush_during_recovery=false
preserve_deletes=false
manual_wal_flush=false
[CFOptions "default"]
report_bg_io_stats=false
inplace_update_support=false
max_compaction_bytes=1677721600
disable_auto_compactions=false
write_buffer_size=4194304
bloom_locality=0
max_bytes_for_level_multiplier=10.000000
compaction_filter_factory=nullptr
optimize_filters_for_hits=false
target_file_size_base=67108864
max_write_buffer_number_to_maintain=0
hard_pending_compaction_bytes_limit=274877906944
paranoid_file_checks=false
memtable_prefix_bloom_size_ratio=0.000000
force_consistency_checks=false
max_write_buffer_number=2
ttl=0
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
level0_slowdown_writes_trigger=20
level_compaction_dynamic_level_bytes=false
compaction_options_fifo={allow_compaction=false;ttl=0;max_table_files_size=1073741824;}
inplace_update_num_locks=10000
level0_file_num_compaction_trigger=4
compression=kSnappyCompression
level0_stop_writes_trigger=36
num_levels=7
table_factory=BlockBasedTable
compression_per_level=
target_file_size_multiplier=1
min_write_buffer_number_to_merge=1
arena_block_size=524288
max_successive_merges=0
memtable_huge_page_size=0
compaction_pri=kByCompensatedSize
soft_pending_compaction_bytes_limit=68719476736
max_bytes_for_level_base=268435456
comparator=leveldb.BytewiseComparator
max_sequential_skip_in_iterations=8
bottommost_compression=kDisableCompressionOption
prefix_extractor=nullptr
memtable_insert_with_hint_prefix_extractor=nullptr
memtable_factory=SkipListFactory
compaction_filter=nullptr
compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;}
merge_operator=nullptr
compaction_style=kCompactionStyleLevel
[TableOptions/BlockBasedTable "default"]
format_version=2
whole_key_filtering=true
verify_compression=false
partition_filters=false
enable_index_compression=true
checksum=kCRC32c
index_block_restart_interval=1
pin_top_level_index_and_filter=true
block_align=false
block_size=4096
index_type=kBinarySearch
filter_policy=nullptr
metadata_block_size=4096
no_block_cache=false
block_size_deviation=10
data_block_index_type=kDataBlockBinarySearch
data_block_hash_table_util_ratio=0.750000
read_amp_bytes_per_bit=8589934592
cache_index_and_filter_blocks=false
block_restart_interval=16
pin_l0_filter_and_index_blocks_in_cache=false
hash_index_allow_collision=true
cache_index_and_filter_blocks_with_high_priority=false
flush_block_policy_factory=FlushBlockBySizePolicyFactory

1
src/Hydra/Core/Domain/DB.hs
View File

@ -12,6 +12,7 @@ data DBErrorType
= SystemError
| KeyNotFound
| InvalidType
| DecodingFailed
deriving (Generic, Ord, Eq, Enum, Bounded, Show, Read)
data DBError = DBError DBErrorType Text

94
src/Hydra/Core/Domain/KVDB.hs
View File

@ -11,66 +11,48 @@ import qualified Data.ByteString.Lazy as LBS
import Hydra.Core.Domain.DB
data KVDBOptions = KVDBOptions
{ _createIfMissing :: Bool
, _errorIfExists :: Bool
}
deriving (Show, Generic)
{ _createIfMissing :: Bool
, _errorIfExists :: Bool
}
deriving (Show, Generic)
data KVDBConfig db = KVDBConfig
{ _path :: FilePath
, _options :: KVDBOptions
}
deriving (Show, Generic)
{ _path :: FilePath
, _options :: KVDBOptions
}
deriving (Show, Generic)
type KVDBKey = ByteString
type KVDBValue = ByteString
-- class DB db where
-- getDbName :: FilePath
--
-- class DBEntity entity where
-- data DBKey entity :: *
-- data DBValue entity :: *
--
-- class DBEntity entity => ToDBKey entity src where
-- toDBKey :: src -> DBKey entity
--
-- class DBEntity entity => ToDBValue entity src where
-- toDBValue :: src -> DBValue entity
--
-- class (DB db, DBEntity entity) => DBModelEntity db entity
--
-- class DBModelEntity db entity => RawDBEntity db entity where
-- toRawDBKey :: DBKey entity -> DBKey
-- toRawDBValue :: DBValue entity -> DBValue
-- fromRawDBValue :: DBValue -> Maybe (DBValue entity)
--
-- -- TODO: this doesn't work by some strange reason.
-- default toRawDBValue :: ToJSON (DBValue entity) => DBValue entity -> DBValue
-- toRawDBValue = LBS.toStrict . A.encode
-- default fromRawDBValue :: FromJSON (DBValue entity) => DBValue -> Maybe (DBValue entity)
-- fromRawDBValue = A.decode . LBS.fromStrict
--
-- type DBE entity = (DBKey entity, DBValue entity)
--
-- defaultDbOptions :: DBOptions
-- defaultDbOptions = DBOptions
-- { _createIfMissing = False
-- , _errorIfExists = False
-- }
--
-- toDBEntity
-- :: (ToDBKey entity src, ToDBValue entity src)
-- => src
-- -> DBE entity
-- toDBEntity src = (toDBKey src, toDBValue src)
--
-- instance ToJSON (Storage db) where toJSON = genericToJSON noLensPrefix
-- instance FromJSON (Storage db) where parseJSON = genericParseJSON noLensPrefix
--
-- instance ToJSON DBOptions where toJSON = genericToJSON noLensPrefix
-- instance FromJSON DBOptions where parseJSON = genericParseJSON noLensPrefix
--
-- instance ToJSON (DBConfig db) where toJSON = genericToJSON noLensPrefix
-- instance FromJSON (DBConfig db) where parseJSON = genericParseJSON noLensPrefix
-- Domain data type /= DB data type.
-- DB data type can be very different.
-- Keys can be very different and can be obtained from different sources.
class DBEntity entity where
data KeyEntity entity :: *
data ValueEntity entity :: *
class AsKeyEntity entity src where
toKeyEntity :: src -> KeyEntity entity
class AsValueEntity entity src where
toValueEntity :: src -> ValueEntity entity
fromValueEntity :: ValueEntity entity -> src
class RawDBEntity entity where
toDBKey :: KeyEntity entity -> KVDBKey
toDBValue :: ValueEntity entity -> KVDBValue
fromDBValue :: KVDBValue -> Maybe (ValueEntity entity)
toDBKeyJSON :: ToJSON (KeyEntity entity) => KeyEntity entity -> KVDBKey
toDBKeyJSON = LBS.toStrict . A.encode
toDBValueJSON :: ToJSON (ValueEntity entity) => ValueEntity entity -> KVDBValue
toDBValueJSON = LBS.toStrict . A.encode
fromDBKeyJSON :: FromJSON (KeyEntity entity) => KVDBKey -> Maybe (KeyEntity entity)
fromDBKeyJSON = A.decode . LBS.fromStrict
fromDBValueJSON :: FromJSON (ValueEntity entity) => KVDBValue -> Maybe (ValueEntity entity)
fromDBValueJSON = A.decode . LBS.fromStrict

11
src/Hydra/Core/KVDB/Interpreter.hs
View File

@ -15,18 +15,17 @@ writeOpts = Rocks.defaultWriteOptions
}
interpretKVDBF :: Rocks.DB -> L.KVDBF a -> IO a
interpretKVDBF db (L.GetValue key) = do
interpretKVDBF db (L.Load key next) = do
mbVal <- Rocks.get db Rocks.defaultReadOptions key
pure $ case mbVal of
pure $ next $ case mbVal of
Nothing -> Left $ D.DBError D.KeyNotFound $ show key
Just val -> Right val
interpretKVDBF db (L.PutValue key val) =
Right <$> Rocks.put db writeOpts key val
interpretKVDBF db (L.Save key val next) =
next . Right <$> Rocks.put db writeOpts key val
runKVDBL :: R.RocksDBHandle -> L.KVDBL db a -> IO a
runKVDBL handle act = do
db <- takeMVar handle
-- res <- foldFree (interpretKVDBF db) act
res <- interpretKVDBF db act
res <- foldFree (interpretKVDBF db) act
putMVar handle db
pure res

74
src/Hydra/Core/KVDB/Language.hs
View File

@ -1,45 +1,63 @@
{-# LANGUAGE AllowAmbiguousTypes #-}
{-# LANGUAGE TemplateHaskell #-}
module Hydra.Core.KVDB.Language where
import Hydra.Prelude
-- import Data.Typeable (typeOf)
import qualified Hydra.Core.Domain.DB as D
import qualified Hydra.Core.Domain.KVDB as D
-- data KVDBF db a where
-- GetValue :: D.KVDBKey -> (D.DBResult D.KVDBValue -> next) -> KVDBF db next
-- PutValue :: D.KVDBKey -> D.KVDBValue -> (D.DBResult () -> next) -> KVDBF db next
-- deriving (Functor)
--
-- type KVDBL db = Free (KVDBF db)
import Language.Haskell.TH.MakeFunctor (makeFunctorInstance)
data KVDBF a where
GetValue :: D.KVDBKey -> KVDBF (D.DBResult D.KVDBValue)
PutValue :: D.KVDBKey -> D.KVDBValue -> KVDBF (D.DBResult ())
data KVDBF next where
Save :: D.KVDBKey -> D.KVDBValue -> (D.DBResult () -> next) -> KVDBF next
Load :: D.KVDBKey -> (D.DBResult D.KVDBValue -> next) -> KVDBF next
type KVDBL db = KVDBF
getValue :: D.KVDBKey -> KVDBL db (D.DBResult D.KVDBValue)
getValue = GetValue
makeFunctorInstance ''KVDBF
putValue :: D.KVDBKey -> D.KVDBValue -> KVDBL db (D.DBResult ())
putValue = PutValue
type KVDBL db = Free KVDBF
save'
:: forall src entity db
. D.DBEntity entity
=> D.AsKeyEntity entity src
=> D.AsValueEntity entity src
=> D.RawDBEntity entity
=> src
-> KVDBL db (D.DBResult ())
save' src = liftF $ Save dbkey dbval id
where
k :: D.KeyEntity entity
k = D.toKeyEntity src
v :: D.ValueEntity entity
v = D.toValueEntity src
dbkey = D.toDBKey k
dbval = D.toDBValue v
load'
:: forall entity dst db
. D.DBEntity entity
=> D.AsKeyEntity entity dst
=> D.AsValueEntity entity dst
=> D.RawDBEntity entity
=> Show (D.KeyEntity entity)
=> D.KeyEntity entity
-> KVDBL db (D.DBResult dst)
load' key = do
eRawVal <- liftF $ Load (D.toDBKey key) id
pure $ case eRawVal of
Left err -> Left err
Right val -> maybe (decodingErr val) (Right . D.fromValueEntity) $ mbE val
where
mbE :: D.KVDBValue -> Maybe (D.ValueEntity entity)
mbE = D.fromDBValue
decodingErr val = Left
$ D.DBError D.DecodingFailed
$ "Failed to decode entity, k: "
<> show key <> ", v: " <> show val
-- putEntity
-- :: forall entity db
-- . D.RawDBEntity db entity
-- => D.DBKey entity
-- -> D.DBValue entity
-- -> KVDBL db (D.DBResult ())
-- putEntity dbKey dbVal = let
-- rawKey = D.toRawDBKey @db dbKey
-- rawVal = D.toRawDBValue @db dbVal
-- in putValue rawKey rawVal
--
-- -- | Puts a typed entity to the corresponding DB.
-- putEntity'
-- :: forall entity db src
-- . D.RawDBEntity db entity