diff --git a/app/Astro/Astro/Catalogue.hs b/app/Astro/Astro/Catalogue.hs index a571734..8382c2a 100644 --- a/app/Astro/Astro/Catalogue.hs +++ b/app/Astro/Astro/Catalogue.hs @@ -12,25 +12,21 @@ import Astro.Types import Astro.KVDB.Model import Astro.Lens -withCatalogueDB - :: AppState - -> L.KVDBL db a - -> L.LangL a +withCatalogueDB :: AppState -> L.KVDBL db a -> L.LangL a withCatalogueDB st = L.withKVDB (st ^. catalogueDB) --- loadMeteorsCount :: L.KVDBL CatalogueDB Int loadMeteorsCount :: AppState -> L.LangL Int loadMeteorsCount st = do - eTest <- withCatalogueDB st $ L.getValue "test" - - pure 10 + eCount <- withCatalogueDB st $ L.getValue "meteors_count" + case eCount of + Left err -> L.logError ("Failed to get meteors count: " <> show err) + Right n -> pure 10 dynamicsMonitor :: AppState -> L.LangL () dynamicsMonitor st = do meteorsCount <- loadMeteorsCount st L.logInfo $ "Meteors count: " +|| meteorsCount ||+ "" - - pure () + L.delay $ 1000 initState :: AppConfig -> L.AppL AppState @@ -60,4 +56,5 @@ astroCatalogue cfg = do L.process $ dynamicsMonitor appSt - L.awaitAppForever + -- L.awaitAppForever + L.delay 10000000 diff --git a/app/Astro/Astro/Domain/Meteor.hs b/app/Astro/Astro/Domain/Meteor.hs new file mode 100644 index 0000000..0c4ef85 --- /dev/null +++ b/app/Astro/Astro/Domain/Meteor.hs @@ -0,0 +1,31 @@ +{-# LANGUAGE DeriveAnyClass #-} + +module Astro.Domain.Meteor where + +import Hydra.Prelude + +import qualified Data.Map as Map +import qualified Data.Set as Set +import Data.Time.Clock (UTCTime) + +type DateTime = UTCTime + +data Coords = Coords + { _azimuth :: Int + , _altitude :: Int + } + deriving (Show, Eq, Ord, Generic, ToJSON, FromJSON) + +data Meteor' k = Meteor' + { _id :: k + , _size :: Int + , _mass :: Int + , _coords :: Coords + , _timestamp :: DateTime + } + deriving (Show, Eq, Ord, Generic, ToJSON, FromJSON) + +type RawMeteor = Meteor' () +type Meteor = Meteor' Int + +-- type Meteors = D.StateVar (Set.Set Meteor) diff --git a/app/Astro/Astro/KVDB/Entities/Meteor.hs b/app/Astro/Astro/KVDB/Entities/Meteor.hs index 85a031f..316bca6 100644 --- a/app/Astro/Astro/KVDB/Entities/Meteor.hs +++ b/app/Astro/Astro/KVDB/Entities/Meteor.hs @@ -5,40 +5,61 @@ module Astro.KVDB.Entities.Meteor where import Hydra.Prelude -import qualified Data.Aeson as A -import qualified Data.ByteString.Lazy as LBS +import qualified Data.Aeson as A +import qualified Data.ByteString.Lazy as LBS +import Text.Printf +import qualified Hydra.Domain as D --- meteors (meteor_idx -> meteor_entity_json) --- ------------------------------------------------------------ --- 0000000 {} +import qualified Astro.Domain.Meteor as D + +-- catalogue +-- ( meteors_count_key -> int +-- , meteor_idx|0 -> meteor_entity_json +-- ) +-- ------------------------------------------------------------------ +-- meteors_count 1 +-- 0|0000000 {...} +-- 0|0000001 {...} data MeteorEntity --- --- instance KVDBModelEntity MeteorsTable MeteorEntity --- --- --- instance D.DBEntity KBlockPrevHashEntity where --- data DBKey KBlockPrevHashEntity = KBlockPrevHashKey D.BlockNumber --- deriving (Show, Eq, Ord) --- data DBValue KBlockPrevHashEntity = KBlockPrevHashValue D.StringHash --- deriving (Show, Eq, Ord, Generic, ToJSON, FromJSON) --- --- instance D.ToDBKey KBlockPrevHashEntity D.BlockNumber where --- toDBKey = KBlockPrevHashKey --- --- instance D.ToDBKey KBlockPrevHashEntity D.KBlock where --- toDBKey = KBlockPrevHashKey . D._number --- --- instance D.ToDBValue KBlockPrevHashEntity D.KBlock where --- toDBValue kBlock = KBlockPrevHashValue $ kBlock ^. Lens.prevHash --- --- -- TODO: this can be made by default --- instance D.RawDBEntity KBlocksDB KBlockPrevHashEntity where --- toRawDBKey (KBlockPrevHashKey kBlockIdx) = encodeUtf8 $ toKBlockPrevHashEntityKeyBase kBlockIdx --- toRawDBValue = LBS.toStrict . A.encode --- fromRawDBValue = A.decode . LBS.fromStrict --- +data MeteorsCountEntity + +-- MeteorEntity + +instance D.DBEntity MeteorEntity where + data KeyEntity MeteorEntity = MeteorKey Int + deriving (Show, Eq, Ord) + data ValueEntity MeteorEntity = MeteorValue D.Meteor + deriving (Show, Eq, Ord, Generic, ToJSON, FromJSON) + +instance D.AsKeyEntity MeteorEntity Int where + toKeyEntity = MeteorKey + +instance D.AsKeyEntity MeteorEntity D.Meteor where + toKeyEntity = MeteorKey . D._id + +instance D.AsValueEntity MeteorEntity D.Meteor where + toValueEntity = MeteorValue + fromValueEntity (MeteorValue v) = v + +instance D.RawDBEntity MeteorEntity where + toDBKey (MeteorKey idx) = show $ toMeteorEntityKey idx + toDBValue = D.toDBValueJSON + fromDBValue = D.fromDBValueJSON + +-- instance D.DBModelEntity CatalogueDB MeteorEntity + + + +-- instance D.DBModelEntity CatalogueDB MeteorsCountEntity + + +-- instance D.RawDBEntity CatalogueDB MeteorEntity where +-- toRawDBKey (MeteorKey idx) = A.encode kBlockIdx +-- toRawDBValue = LBS.toStrict . A.encode +-- fromRawDBValue = A.decode . LBS.fromStrict + -- -- KBlock entity -- -- instance D.DBEntity KBlockEntity where @@ -60,13 +81,10 @@ data MeteorEntity -- toRawDBKey (KBlockKey kBlockIdx) = encodeUtf8 $ toKBlockEntityKeyBase kBlockIdx -- toRawDBValue = LBS.toStrict . A.encode -- fromRawDBValue = A.decode . LBS.fromStrict --- --- --- toKBlockIdxBase :: KBlockIdx -> String --- toKBlockIdxBase = printf "%07d" --- --- toKBlockPrevHashEntityKeyBase :: KBlockIdx -> String --- toKBlockPrevHashEntityKeyBase = (<> "0") . toKBlockIdxBase --- --- toKBlockEntityKeyBase :: KBlockIdx -> String --- toKBlockEntityKeyBase = (<> "1") . toKBlockIdxBase + + +toIdxBase :: Int -> String +toIdxBase = printf "%07d" + +toMeteorEntityKey :: Int -> String +toMeteorEntityKey = ("0|" <>) . toIdxBase diff --git a/app/Astro/Main.hs b/app/Astro/Main.hs index 7832fd5..db52919 100644 --- a/app/Astro/Main.hs +++ b/app/Astro/Main.hs @@ -15,10 +15,19 @@ import qualified Hydra.Runtime as R import Astro.Types import Astro.Catalogue +loggerCfg :: D.LoggerConfig +loggerCfg = D.LoggerConfig + { D._format = "$prio $loggername: $msg" + , D._level = D.Debug + , D._logFilePath = "" + , D._logToConsole = True + , D._logToFile = False + } + main :: IO () main = do - loggerRt <- R.createVoidLoggerRuntime + loggerRt <- R.createLoggerRuntime loggerCfg coreRt <- R.createCoreRuntime loggerRt let cfg = AppConfig False 0 diff --git a/catalogue/000009.log b/catalogue/000009.log new file mode 100644 index 0000000..e69de29 diff --git a/catalogue/CURRENT b/catalogue/CURRENT new file mode 100644 index 0000000..d408ea0 --- /dev/null +++ b/catalogue/CURRENT @@ -0,0 +1 @@ +MANIFEST-000008 diff --git a/catalogue/IDENTITY b/catalogue/IDENTITY new file mode 100644 index 0000000..530c70e --- /dev/null +++ b/catalogue/IDENTITY @@ -0,0 +1 @@ +15b98768706bc3f9-120aec5d7215c312 \ No newline at end of file diff --git a/catalogue/LOCK b/catalogue/LOCK new file mode 100644 index 0000000..e69de29 diff --git a/catalogue/LOG b/catalogue/LOG new file mode 100644 index 0000000..371c11c --- /dev/null +++ b/catalogue/LOG @@ -0,0 +1,240 @@ +2019/08/10-17:02:41.640123 109c9b5c0 RocksDB version: 5.18.3 +2019/08/10-17:02:41.640589 109c9b5c0 Git sha rocksdb_build_git_sha: +2019/08/10-17:02:41.640592 109c9b5c0 Compile date Feb 21 2019 +2019/08/10-17:02:41.640593 109c9b5c0 DB SUMMARY +2019/08/10-17:02:41.640660 109c9b5c0 CURRENT file: CURRENT +2019/08/10-17:02:41.640662 109c9b5c0 IDENTITY file: IDENTITY +2019/08/10-17:02:41.640668 109c9b5c0 MANIFEST file: MANIFEST-000005 size: 59 Bytes +2019/08/10-17:02:41.640670 109c9b5c0 SST files in catalogue dir, Total Num: 0, files: +2019/08/10-17:02:41.640671 109c9b5c0 Write Ahead Log file in catalogue: 000006.log size: 0 ; +2019/08/10-17:02:41.640673 109c9b5c0 Options.error_if_exists: 0 +2019/08/10-17:02:41.640677 109c9b5c0 Options.create_if_missing: 1 +2019/08/10-17:02:41.640678 109c9b5c0 Options.paranoid_checks: 0 +2019/08/10-17:02:41.640679 109c9b5c0 Options.env: 0x1018392e8 +2019/08/10-17:02:41.640680 109c9b5c0 Options.info_log: 0x7fd2b1421400 +2019/08/10-17:02:41.640681 109c9b5c0 Options.max_file_opening_threads: 16 +2019/08/10-17:02:41.640682 109c9b5c0 Options.statistics: 0x0 +2019/08/10-17:02:41.640683 109c9b5c0 Options.use_fsync: 0 +2019/08/10-17:02:41.640684 109c9b5c0 Options.max_log_file_size: 0 +2019/08/10-17:02:41.640685 109c9b5c0 Options.max_manifest_file_size: 1073741824 +2019/08/10-17:02:41.640686 109c9b5c0 Options.log_file_time_to_roll: 0 +2019/08/10-17:02:41.640687 109c9b5c0 Options.keep_log_file_num: 1000 +2019/08/10-17:02:41.640688 109c9b5c0 Options.recycle_log_file_num: 0 +2019/08/10-17:02:41.640689 109c9b5c0 Options.allow_fallocate: 1 +2019/08/10-17:02:41.640689 109c9b5c0 Options.allow_mmap_reads: 0 +2019/08/10-17:02:41.640690 109c9b5c0 Options.allow_mmap_writes: 0 +2019/08/10-17:02:41.640691 109c9b5c0 Options.use_direct_reads: 0 +2019/08/10-17:02:41.640692 109c9b5c0 Options.use_direct_io_for_flush_and_compaction: 0 +2019/08/10-17:02:41.640693 109c9b5c0 Options.create_missing_column_families: 0 +2019/08/10-17:02:41.640694 109c9b5c0 Options.db_log_dir: +2019/08/10-17:02:41.640695 109c9b5c0 Options.wal_dir: catalogue +2019/08/10-17:02:41.640696 109c9b5c0 Options.table_cache_numshardbits: 6 +2019/08/10-17:02:41.640697 109c9b5c0 Options.max_subcompactions: 1 +2019/08/10-17:02:41.640698 109c9b5c0 Options.max_background_flushes: -1 +2019/08/10-17:02:41.640699 109c9b5c0 Options.WAL_ttl_seconds: 0 +2019/08/10-17:02:41.640699 109c9b5c0 Options.WAL_size_limit_MB: 0 +2019/08/10-17:02:41.640700 109c9b5c0 Options.manifest_preallocation_size: 4194304 +2019/08/10-17:02:41.640701 109c9b5c0 Options.is_fd_close_on_exec: 1 +2019/08/10-17:02:41.640702 109c9b5c0 Options.advise_random_on_open: 1 +2019/08/10-17:02:41.640703 109c9b5c0 Options.db_write_buffer_size: 0 +2019/08/10-17:02:41.640704 109c9b5c0 Options.write_buffer_manager: 0x7fd2b14210c0 +2019/08/10-17:02:41.640705 109c9b5c0 Options.access_hint_on_compaction_start: 1 +2019/08/10-17:02:41.640706 109c9b5c0 Options.new_table_reader_for_compaction_inputs: 0 +2019/08/10-17:02:41.640707 109c9b5c0 Options.random_access_max_buffer_size: 1048576 +2019/08/10-17:02:41.640708 109c9b5c0 Options.use_adaptive_mutex: 0 +2019/08/10-17:02:41.640708 109c9b5c0 Options.rate_limiter: 0x0 +2019/08/10-17:02:41.640710 109c9b5c0 Options.sst_file_manager.rate_bytes_per_sec: 0 +2019/08/10-17:02:41.640711 109c9b5c0 Options.wal_recovery_mode: 2 +2019/08/10-17:02:41.640711 109c9b5c0 Options.enable_thread_tracking: 0 +2019/08/10-17:02:41.640712 109c9b5c0 Options.enable_pipelined_write: 0 +2019/08/10-17:02:41.640740 109c9b5c0 Options.allow_concurrent_memtable_write: 1 +2019/08/10-17:02:41.640742 109c9b5c0 Options.enable_write_thread_adaptive_yield: 1 +2019/08/10-17:02:41.640743 109c9b5c0 Options.write_thread_max_yield_usec: 100 +2019/08/10-17:02:41.640744 109c9b5c0 Options.write_thread_slow_yield_usec: 3 +2019/08/10-17:02:41.640744 109c9b5c0 Options.row_cache: None +2019/08/10-17:02:41.640745 109c9b5c0 Options.wal_filter: None +2019/08/10-17:02:41.640746 109c9b5c0 Options.avoid_flush_during_recovery: 0 +2019/08/10-17:02:41.640747 109c9b5c0 Options.allow_ingest_behind: 0 +2019/08/10-17:02:41.640748 109c9b5c0 Options.preserve_deletes: 0 +2019/08/10-17:02:41.640749 109c9b5c0 Options.two_write_queues: 0 +2019/08/10-17:02:41.640750 109c9b5c0 Options.manual_wal_flush: 0 +2019/08/10-17:02:41.640751 109c9b5c0 Options.max_background_jobs: 2 +2019/08/10-17:02:41.640752 109c9b5c0 Options.max_background_compactions: -1 +2019/08/10-17:02:41.640753 109c9b5c0 Options.avoid_flush_during_shutdown: 0 +2019/08/10-17:02:41.640754 109c9b5c0 Options.writable_file_max_buffer_size: 1048576 +2019/08/10-17:02:41.640755 109c9b5c0 Options.delayed_write_rate : 16777216 +2019/08/10-17:02:41.640756 109c9b5c0 Options.max_total_wal_size: 0 +2019/08/10-17:02:41.640757 109c9b5c0 Options.delete_obsolete_files_period_micros: 21600000000 +2019/08/10-17:02:41.640758 109c9b5c0 Options.stats_dump_period_sec: 600 +2019/08/10-17:02:41.640759 109c9b5c0 Options.max_open_files: 256 +2019/08/10-17:02:41.640759 109c9b5c0 Options.bytes_per_sync: 0 +2019/08/10-17:02:41.640760 109c9b5c0 Options.wal_bytes_per_sync: 0 +2019/08/10-17:02:41.640761 109c9b5c0 Options.compaction_readahead_size: 0 +2019/08/10-17:02:41.640762 109c9b5c0 Compression algorithms supported: +2019/08/10-17:02:41.640763 109c9b5c0 kZSTD supported: 0 +2019/08/10-17:02:41.640764 109c9b5c0 kZlibCompression supported: 1 +2019/08/10-17:02:41.640765 109c9b5c0 kXpressCompression supported: 0 +2019/08/10-17:02:41.640766 109c9b5c0 kSnappyCompression supported: 1 +2019/08/10-17:02:41.640767 109c9b5c0 kZSTDNotFinalCompression supported: 0 +2019/08/10-17:02:41.640768 109c9b5c0 kLZ4HCCompression supported: 1 +2019/08/10-17:02:41.640769 109c9b5c0 kLZ4Compression supported: 1 +2019/08/10-17:02:41.640770 109c9b5c0 kBZip2Compression supported: 1 +2019/08/10-17:02:41.640774 109c9b5c0 Fast CRC32 supported: Supported on x86 +2019/08/10-17:02:41.640948 109c9b5c0 [/version_set.cc:3508] Recovering from manifest file: MANIFEST-000005 +2019/08/10-17:02:41.640999 109c9b5c0 [/column_family.cc:474] --------------- Options for column family [default]: +2019/08/10-17:02:41.641004 109c9b5c0 Options.comparator: leveldb.BytewiseComparator +2019/08/10-17:02:41.641005 109c9b5c0 Options.merge_operator: None +2019/08/10-17:02:41.641006 109c9b5c0 Options.compaction_filter: None +2019/08/10-17:02:41.641007 109c9b5c0 Options.compaction_filter_factory: None +2019/08/10-17:02:41.641008 109c9b5c0 Options.memtable_factory: SkipListFactory +2019/08/10-17:02:41.641009 109c9b5c0 Options.table_factory: BlockBasedTable +2019/08/10-17:02:41.641027 109c9b5c0 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7fd2b1420530) + cache_index_and_filter_blocks: 0 + cache_index_and_filter_blocks_with_high_priority: 0 + pin_l0_filter_and_index_blocks_in_cache: 0 + pin_top_level_index_and_filter: 1 + index_type: 0 + hash_index_allow_collision: 1 + checksum: 1 + no_block_cache: 0 + block_cache: 0x7fd2b1420578 + block_cache_name: LRUCache + block_cache_options: + capacity : 8388608 + num_shard_bits : 4 + strict_capacity_limit : 0 + memory_allocator : None + high_pri_pool_ratio: 0.000 + block_cache_compressed: 0x0 + persistent_cache: 0x0 + block_size: 4096 + block_size_deviation: 10 + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4096 + partition_filters: 0 + use_delta_encoding: 1 + filter_policy: nullptr + whole_key_filtering: 1 + verify_compression: 0 + read_amp_bytes_per_bit: 0 + format_version: 2 + enable_index_compression: 1 + block_align: 0 +2019/08/10-17:02:41.641053 109c9b5c0 Options.write_buffer_size: 4194304 +2019/08/10-17:02:41.641055 109c9b5c0 Options.max_write_buffer_number: 2 +2019/08/10-17:02:41.641058 109c9b5c0 Options.compression: Snappy +2019/08/10-17:02:41.641059 109c9b5c0 Options.bottommost_compression: Disabled +2019/08/10-17:02:41.641060 109c9b5c0 Options.prefix_extractor: nullptr +2019/08/10-17:02:41.641061 109c9b5c0 Options.memtable_insert_with_hint_prefix_extractor: nullptr +2019/08/10-17:02:41.641062 109c9b5c0 Options.num_levels: 7 +2019/08/10-17:02:41.641063 109c9b5c0 Options.min_write_buffer_number_to_merge: 1 +2019/08/10-17:02:41.641064 109c9b5c0 Options.max_write_buffer_number_to_maintain: 0 +2019/08/10-17:02:41.641065 109c9b5c0 Options.bottommost_compression_opts.window_bits: -14 +2019/08/10-17:02:41.641066 109c9b5c0 Options.bottommost_compression_opts.level: 32767 +2019/08/10-17:02:41.641067 109c9b5c0 Options.bottommost_compression_opts.strategy: 0 +2019/08/10-17:02:41.641068 109c9b5c0 Options.bottommost_compression_opts.max_dict_bytes: 0 +2019/08/10-17:02:41.641069 109c9b5c0 Options.bottommost_compression_opts.zstd_max_train_bytes: 0 +2019/08/10-17:02:41.641070 109c9b5c0 Options.bottommost_compression_opts.enabled: false +2019/08/10-17:02:41.641071 109c9b5c0 Options.compression_opts.window_bits: -14 +2019/08/10-17:02:41.641072 109c9b5c0 Options.compression_opts.level: 32767 +2019/08/10-17:02:41.641073 109c9b5c0 Options.compression_opts.strategy: 0 +2019/08/10-17:02:41.641074 109c9b5c0 Options.compression_opts.max_dict_bytes: 0 +2019/08/10-17:02:41.641075 109c9b5c0 Options.compression_opts.zstd_max_train_bytes: 0 +2019/08/10-17:02:41.641075 109c9b5c0 Options.compression_opts.enabled: false +2019/08/10-17:02:41.641076 109c9b5c0 Options.level0_file_num_compaction_trigger: 4 +2019/08/10-17:02:41.641077 109c9b5c0 Options.level0_slowdown_writes_trigger: 20 +2019/08/10-17:02:41.641078 109c9b5c0 Options.level0_stop_writes_trigger: 36 +2019/08/10-17:02:41.641079 109c9b5c0 Options.target_file_size_base: 67108864 +2019/08/10-17:02:41.641080 109c9b5c0 Options.target_file_size_multiplier: 1 +2019/08/10-17:02:41.641081 109c9b5c0 Options.max_bytes_for_level_base: 268435456 +2019/08/10-17:02:41.641082 109c9b5c0 Options.level_compaction_dynamic_level_bytes: 0 +2019/08/10-17:02:41.641083 109c9b5c0 Options.max_bytes_for_level_multiplier: 10.000000 +2019/08/10-17:02:41.641085 109c9b5c0 Options.max_bytes_for_level_multiplier_addtl[0]: 1 +2019/08/10-17:02:41.641086 109c9b5c0 Options.max_bytes_for_level_multiplier_addtl[1]: 1 +2019/08/10-17:02:41.641087 109c9b5c0 Options.max_bytes_for_level_multiplier_addtl[2]: 1 +2019/08/10-17:02:41.641088 109c9b5c0 Options.max_bytes_for_level_multiplier_addtl[3]: 1 +2019/08/10-17:02:41.641089 109c9b5c0 Options.max_bytes_for_level_multiplier_addtl[4]: 1 +2019/08/10-17:02:41.641090 109c9b5c0 Options.max_bytes_for_level_multiplier_addtl[5]: 1 +2019/08/10-17:02:41.641090 109c9b5c0 Options.max_bytes_for_level_multiplier_addtl[6]: 1 +2019/08/10-17:02:41.641091 109c9b5c0 Options.max_sequential_skip_in_iterations: 8 +2019/08/10-17:02:41.641092 109c9b5c0 Options.max_compaction_bytes: 1677721600 +2019/08/10-17:02:41.641093 109c9b5c0 Options.arena_block_size: 524288 +2019/08/10-17:02:41.641094 109c9b5c0 Options.soft_pending_compaction_bytes_limit: 68719476736 +2019/08/10-17:02:41.641095 109c9b5c0 Options.hard_pending_compaction_bytes_limit: 274877906944 +2019/08/10-17:02:41.641110 109c9b5c0 Options.rate_limit_delay_max_milliseconds: 100 +2019/08/10-17:02:41.641112 109c9b5c0 Options.disable_auto_compactions: 0 +2019/08/10-17:02:41.641113 109c9b5c0 Options.compaction_style: kCompactionStyleLevel +2019/08/10-17:02:41.641114 109c9b5c0 Options.compaction_pri: kByCompensatedSize +2019/08/10-17:02:41.641115 109c9b5c0 Options.compaction_options_universal.size_ratio: 1 +2019/08/10-17:02:41.641116 109c9b5c0 Options.compaction_options_universal.min_merge_width: 2 +2019/08/10-17:02:41.641117 109c9b5c0 Options.compaction_options_universal.max_merge_width: 4294967295 +2019/08/10-17:02:41.641118 109c9b5c0 Options.compaction_options_universal.max_size_amplification_percent: 200 +2019/08/10-17:02:41.641119 109c9b5c0 Options.compaction_options_universal.compression_size_percent: -1 +2019/08/10-17:02:41.641120 109c9b5c0 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize +2019/08/10-17:02:41.641121 109c9b5c0 Options.compaction_options_fifo.max_table_files_size: 1073741824 +2019/08/10-17:02:41.641122 109c9b5c0 Options.compaction_options_fifo.allow_compaction: 0 +2019/08/10-17:02:41.641123 109c9b5c0 Options.compaction_options_fifo.ttl: 0 +2019/08/10-17:02:41.641124 109c9b5c0 Options.table_properties_collectors: +2019/08/10-17:02:41.641125 109c9b5c0 Options.inplace_update_support: 0 +2019/08/10-17:02:41.641126 109c9b5c0 Options.inplace_update_num_locks: 10000 +2019/08/10-17:02:41.641127 109c9b5c0 Options.memtable_prefix_bloom_size_ratio: 0.000000 +2019/08/10-17:02:41.641128 109c9b5c0 Options.memtable_huge_page_size: 0 +2019/08/10-17:02:41.641129 109c9b5c0 Options.bloom_locality: 0 +2019/08/10-17:02:41.641130 109c9b5c0 Options.max_successive_merges: 0 +2019/08/10-17:02:41.641130 109c9b5c0 Options.optimize_filters_for_hits: 0 +2019/08/10-17:02:41.641131 109c9b5c0 Options.paranoid_file_checks: 0 +2019/08/10-17:02:41.641132 109c9b5c0 Options.force_consistency_checks: 0 +2019/08/10-17:02:41.641133 109c9b5c0 Options.report_bg_io_stats: 0 +2019/08/10-17:02:41.641134 109c9b5c0 Options.ttl: 0 +2019/08/10-17:02:41.641207 109c9b5c0 [/version_set.cc:3724] Recovered from manifest file:catalogue/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 +2019/08/10-17:02:41.641210 109c9b5c0 [/version_set.cc:3732] Column family [default] (ID 0), log number is 4 +2019/08/10-17:02:41.641298 109c9b5c0 EVENT_LOG_v1 {"time_micros": 1565431361641290, "job": 1, "event": "recovery_started", "log_files": [6]} +2019/08/10-17:02:41.641306 109c9b5c0 [/db_impl_open.cc:578] Recovering log #6 mode 2 +2019/08/10-17:02:41.641358 109c9b5c0 [/version_set.cc:3037] Creating manifest 8 +2019/08/10-17:02:41.642197 109c9b5c0 EVENT_LOG_v1 {"time_micros": 1565431361642193, "job": 1, "event": "recovery_finished"} +2019/08/10-17:02:41.645351 109c9b5c0 [/db_impl_open.cc:1314] DB pointer 0x7fd2b1904e00 +2019/08/10-17:02:41.645524 70000e3b9000 [WARN] [/db_impl.cc:669] ------- DUMPING STATS ------- +2019/08/10-17:02:41.645529 70000e3b9000 [WARN] [/db_impl.cc:670] +** DB Stats ** +Uptime(secs): 0.0 total, 0.0 interval +Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s +Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s +Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent +Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s +Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s +Interval stall: 00:00:0.000 H:M:S, 0.0 percent + +** Compaction Stats [default] ** +Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop +---------------------------------------------------------------------------------------------------------------------------------------------------------- + Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 + Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 +Uptime(secs): 0.0 total, 0.0 interval +Flush(GB): cumulative 0.000, interval 0.000 +AddFile(GB): cumulative 0.000, interval 0.000 +AddFile(Total Files): cumulative 0, interval 0 +AddFile(L0 Files): cumulative 0, interval 0 +AddFile(Keys): cumulative 0, interval 0 +Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count + +** File Read Latency Histogram By Level [default] ** + +** Compaction Stats [default] ** +Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop +---------------------------------------------------------------------------------------------------------------------------------------------------------- + Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 + Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 +Uptime(secs): 0.0 total, 0.0 interval +Flush(GB): cumulative 0.000, interval 0.000 +AddFile(GB): cumulative 0.000, interval 0.000 +AddFile(Total Files): cumulative 0, interval 0 +AddFile(L0 Files): cumulative 0, interval 0 +AddFile(Keys): cumulative 0, interval 0 +Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count + +** File Read Latency Histogram By Level [default] ** diff --git a/catalogue/LOG.old.1565431269860905 b/catalogue/LOG.old.1565431269860905 new file mode 100644 index 0000000..ca1eeb0 --- /dev/null +++ b/catalogue/LOG.old.1565431269860905 @@ -0,0 +1,234 @@ +2019/08/10-17:00:28.118940 10b3645c0 RocksDB version: 5.18.3 +2019/08/10-17:00:28.119471 10b3645c0 Git sha rocksdb_build_git_sha: +2019/08/10-17:00:28.119474 10b3645c0 Compile date Feb 21 2019 +2019/08/10-17:00:28.119476 10b3645c0 DB SUMMARY +2019/08/10-17:00:28.119518 10b3645c0 SST files in catalogue dir, Total Num: 0, files: +2019/08/10-17:00:28.119521 10b3645c0 Write Ahead Log file in catalogue: +2019/08/10-17:00:28.119522 10b3645c0 Options.error_if_exists: 0 +2019/08/10-17:00:28.119525 10b3645c0 Options.create_if_missing: 1 +2019/08/10-17:00:28.119526 10b3645c0 Options.paranoid_checks: 0 +2019/08/10-17:00:28.119527 10b3645c0 Options.env: 0x10913d2e8 +2019/08/10-17:00:28.119528 10b3645c0 Options.info_log: 0x7fb58fc089b0 +2019/08/10-17:00:28.119529 10b3645c0 Options.max_file_opening_threads: 16 +2019/08/10-17:00:28.119530 10b3645c0 Options.statistics: 0x0 +2019/08/10-17:00:28.119531 10b3645c0 Options.use_fsync: 0 +2019/08/10-17:00:28.119532 10b3645c0 Options.max_log_file_size: 0 +2019/08/10-17:00:28.119533 10b3645c0 Options.max_manifest_file_size: 1073741824 +2019/08/10-17:00:28.119534 10b3645c0 Options.log_file_time_to_roll: 0 +2019/08/10-17:00:28.119534 10b3645c0 Options.keep_log_file_num: 1000 +2019/08/10-17:00:28.119535 10b3645c0 Options.recycle_log_file_num: 0 +2019/08/10-17:00:28.119536 10b3645c0 Options.allow_fallocate: 1 +2019/08/10-17:00:28.119537 10b3645c0 Options.allow_mmap_reads: 0 +2019/08/10-17:00:28.119538 10b3645c0 Options.allow_mmap_writes: 0 +2019/08/10-17:00:28.119538 10b3645c0 Options.use_direct_reads: 0 +2019/08/10-17:00:28.119539 10b3645c0 Options.use_direct_io_for_flush_and_compaction: 0 +2019/08/10-17:00:28.119540 10b3645c0 Options.create_missing_column_families: 0 +2019/08/10-17:00:28.119541 10b3645c0 Options.db_log_dir: +2019/08/10-17:00:28.119542 10b3645c0 Options.wal_dir: catalogue +2019/08/10-17:00:28.119543 10b3645c0 Options.table_cache_numshardbits: 6 +2019/08/10-17:00:28.119543 10b3645c0 Options.max_subcompactions: 1 +2019/08/10-17:00:28.119544 10b3645c0 Options.max_background_flushes: -1 +2019/08/10-17:00:28.119545 10b3645c0 Options.WAL_ttl_seconds: 0 +2019/08/10-17:00:28.119546 10b3645c0 Options.WAL_size_limit_MB: 0 +2019/08/10-17:00:28.119547 10b3645c0 Options.manifest_preallocation_size: 4194304 +2019/08/10-17:00:28.119548 10b3645c0 Options.is_fd_close_on_exec: 1 +2019/08/10-17:00:28.119548 10b3645c0 Options.advise_random_on_open: 1 +2019/08/10-17:00:28.119549 10b3645c0 Options.db_write_buffer_size: 0 +2019/08/10-17:00:28.119550 10b3645c0 Options.write_buffer_manager: 0x7fb58fc08950 +2019/08/10-17:00:28.119551 10b3645c0 Options.access_hint_on_compaction_start: 1 +2019/08/10-17:00:28.119552 10b3645c0 Options.new_table_reader_for_compaction_inputs: 0 +2019/08/10-17:00:28.119553 10b3645c0 Options.random_access_max_buffer_size: 1048576 +2019/08/10-17:00:28.119553 10b3645c0 Options.use_adaptive_mutex: 0 +2019/08/10-17:00:28.119554 10b3645c0 Options.rate_limiter: 0x0 +2019/08/10-17:00:28.119555 10b3645c0 Options.sst_file_manager.rate_bytes_per_sec: 0 +2019/08/10-17:00:28.119556 10b3645c0 Options.wal_recovery_mode: 2 +2019/08/10-17:00:28.119557 10b3645c0 Options.enable_thread_tracking: 0 +2019/08/10-17:00:28.119558 10b3645c0 Options.enable_pipelined_write: 0 +2019/08/10-17:00:28.119559 10b3645c0 Options.allow_concurrent_memtable_write: 1 +2019/08/10-17:00:28.119559 10b3645c0 Options.enable_write_thread_adaptive_yield: 1 +2019/08/10-17:00:28.119580 10b3645c0 Options.write_thread_max_yield_usec: 100 +2019/08/10-17:00:28.119582 10b3645c0 Options.write_thread_slow_yield_usec: 3 +2019/08/10-17:00:28.119583 10b3645c0 Options.row_cache: None +2019/08/10-17:00:28.119583 10b3645c0 Options.wal_filter: None +2019/08/10-17:00:28.119584 10b3645c0 Options.avoid_flush_during_recovery: 0 +2019/08/10-17:00:28.119585 10b3645c0 Options.allow_ingest_behind: 0 +2019/08/10-17:00:28.119586 10b3645c0 Options.preserve_deletes: 0 +2019/08/10-17:00:28.119587 10b3645c0 Options.two_write_queues: 0 +2019/08/10-17:00:28.119588 10b3645c0 Options.manual_wal_flush: 0 +2019/08/10-17:00:28.119588 10b3645c0 Options.max_background_jobs: 2 +2019/08/10-17:00:28.119589 10b3645c0 Options.max_background_compactions: -1 +2019/08/10-17:00:28.119590 10b3645c0 Options.avoid_flush_during_shutdown: 0 +2019/08/10-17:00:28.119591 10b3645c0 Options.writable_file_max_buffer_size: 1048576 +2019/08/10-17:00:28.119592 10b3645c0 Options.delayed_write_rate : 16777216 +2019/08/10-17:00:28.119593 10b3645c0 Options.max_total_wal_size: 0 +2019/08/10-17:00:28.119594 10b3645c0 Options.delete_obsolete_files_period_micros: 21600000000 +2019/08/10-17:00:28.119594 10b3645c0 Options.stats_dump_period_sec: 600 +2019/08/10-17:00:28.119595 10b3645c0 Options.max_open_files: 256 +2019/08/10-17:00:28.119596 10b3645c0 Options.bytes_per_sync: 0 +2019/08/10-17:00:28.119597 10b3645c0 Options.wal_bytes_per_sync: 0 +2019/08/10-17:00:28.119598 10b3645c0 Options.compaction_readahead_size: 0 +2019/08/10-17:00:28.119599 10b3645c0 Compression algorithms supported: +2019/08/10-17:00:28.119600 10b3645c0 kZSTD supported: 0 +2019/08/10-17:00:28.119601 10b3645c0 kZlibCompression supported: 1 +2019/08/10-17:00:28.119602 10b3645c0 kXpressCompression supported: 0 +2019/08/10-17:00:28.119603 10b3645c0 kSnappyCompression supported: 1 +2019/08/10-17:00:28.119604 10b3645c0 kZSTDNotFinalCompression supported: 0 +2019/08/10-17:00:28.119604 10b3645c0 kLZ4HCCompression supported: 1 +2019/08/10-17:00:28.119605 10b3645c0 kLZ4Compression supported: 1 +2019/08/10-17:00:28.119606 10b3645c0 kBZip2Compression supported: 1 +2019/08/10-17:00:28.119607 10b3645c0 Fast CRC32 supported: Supported on x86 +2019/08/10-17:00:28.119723 10b3645c0 [/db_impl_open.cc:226] Creating manifest 1 +2019/08/10-17:00:28.121131 10b3645c0 [/version_set.cc:3508] Recovering from manifest file: MANIFEST-000001 +2019/08/10-17:00:28.121208 10b3645c0 [/column_family.cc:474] --------------- Options for column family [default]: +2019/08/10-17:00:28.121214 10b3645c0 Options.comparator: leveldb.BytewiseComparator +2019/08/10-17:00:28.121215 10b3645c0 Options.merge_operator: None +2019/08/10-17:00:28.121216 10b3645c0 Options.compaction_filter: None +2019/08/10-17:00:28.121217 10b3645c0 Options.compaction_filter_factory: None +2019/08/10-17:00:28.121218 10b3645c0 Options.memtable_factory: SkipListFactory +2019/08/10-17:00:28.121219 10b3645c0 Options.table_factory: BlockBasedTable +2019/08/10-17:00:28.121234 10b3645c0 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7fb58fc07070) + cache_index_and_filter_blocks: 0 + cache_index_and_filter_blocks_with_high_priority: 0 + pin_l0_filter_and_index_blocks_in_cache: 0 + pin_top_level_index_and_filter: 1 + index_type: 0 + hash_index_allow_collision: 1 + checksum: 1 + no_block_cache: 0 + block_cache: 0x7fb58fc07e78 + block_cache_name: LRUCache + block_cache_options: + capacity : 8388608 + num_shard_bits : 4 + strict_capacity_limit : 0 + memory_allocator : None + high_pri_pool_ratio: 0.000 + block_cache_compressed: 0x0 + persistent_cache: 0x0 + block_size: 4096 + block_size_deviation: 10 + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4096 + partition_filters: 0 + use_delta_encoding: 1 + filter_policy: nullptr + whole_key_filtering: 1 + verify_compression: 0 + read_amp_bytes_per_bit: 0 + format_version: 2 + enable_index_compression: 1 + block_align: 0 +2019/08/10-17:00:28.121260 10b3645c0 Options.write_buffer_size: 4194304 +2019/08/10-17:00:28.121262 10b3645c0 Options.max_write_buffer_number: 2 +2019/08/10-17:00:28.121265 10b3645c0 Options.compression: Snappy +2019/08/10-17:00:28.121266 10b3645c0 Options.bottommost_compression: Disabled +2019/08/10-17:00:28.121267 10b3645c0 Options.prefix_extractor: nullptr +2019/08/10-17:00:28.121268 10b3645c0 Options.memtable_insert_with_hint_prefix_extractor: nullptr +2019/08/10-17:00:28.121268 10b3645c0 Options.num_levels: 7 +2019/08/10-17:00:28.121269 10b3645c0 Options.min_write_buffer_number_to_merge: 1 +2019/08/10-17:00:28.121270 10b3645c0 Options.max_write_buffer_number_to_maintain: 0 +2019/08/10-17:00:28.121271 10b3645c0 Options.bottommost_compression_opts.window_bits: -14 +2019/08/10-17:00:28.121272 10b3645c0 Options.bottommost_compression_opts.level: 32767 +2019/08/10-17:00:28.121273 10b3645c0 Options.bottommost_compression_opts.strategy: 0 +2019/08/10-17:00:28.121273 10b3645c0 Options.bottommost_compression_opts.max_dict_bytes: 0 +2019/08/10-17:00:28.121274 10b3645c0 Options.bottommost_compression_opts.zstd_max_train_bytes: 0 +2019/08/10-17:00:28.121275 10b3645c0 Options.bottommost_compression_opts.enabled: false +2019/08/10-17:00:28.121276 10b3645c0 Options.compression_opts.window_bits: -14 +2019/08/10-17:00:28.121277 10b3645c0 Options.compression_opts.level: 32767 +2019/08/10-17:00:28.121277 10b3645c0 Options.compression_opts.strategy: 0 +2019/08/10-17:00:28.121278 10b3645c0 Options.compression_opts.max_dict_bytes: 0 +2019/08/10-17:00:28.121279 10b3645c0 Options.compression_opts.zstd_max_train_bytes: 0 +2019/08/10-17:00:28.121280 10b3645c0 Options.compression_opts.enabled: false +2019/08/10-17:00:28.121281 10b3645c0 Options.level0_file_num_compaction_trigger: 4 +2019/08/10-17:00:28.121281 10b3645c0 Options.level0_slowdown_writes_trigger: 20 +2019/08/10-17:00:28.121282 10b3645c0 Options.level0_stop_writes_trigger: 36 +2019/08/10-17:00:28.121283 10b3645c0 Options.target_file_size_base: 67108864 +2019/08/10-17:00:28.121284 10b3645c0 Options.target_file_size_multiplier: 1 +2019/08/10-17:00:28.121284 10b3645c0 Options.max_bytes_for_level_base: 268435456 +2019/08/10-17:00:28.121285 10b3645c0 Options.level_compaction_dynamic_level_bytes: 0 +2019/08/10-17:00:28.121286 10b3645c0 Options.max_bytes_for_level_multiplier: 10.000000 +2019/08/10-17:00:28.121292 10b3645c0 Options.max_bytes_for_level_multiplier_addtl[0]: 1 +2019/08/10-17:00:28.121293 10b3645c0 Options.max_bytes_for_level_multiplier_addtl[1]: 1 +2019/08/10-17:00:28.121294 10b3645c0 Options.max_bytes_for_level_multiplier_addtl[2]: 1 +2019/08/10-17:00:28.121295 10b3645c0 Options.max_bytes_for_level_multiplier_addtl[3]: 1 +2019/08/10-17:00:28.121296 10b3645c0 Options.max_bytes_for_level_multiplier_addtl[4]: 1 +2019/08/10-17:00:28.121296 10b3645c0 Options.max_bytes_for_level_multiplier_addtl[5]: 1 +2019/08/10-17:00:28.121297 10b3645c0 Options.max_bytes_for_level_multiplier_addtl[6]: 1 +2019/08/10-17:00:28.121298 10b3645c0 Options.max_sequential_skip_in_iterations: 8 +2019/08/10-17:00:28.121299 10b3645c0 Options.max_compaction_bytes: 1677721600 +2019/08/10-17:00:28.121300 10b3645c0 Options.arena_block_size: 524288 +2019/08/10-17:00:28.121300 10b3645c0 Options.soft_pending_compaction_bytes_limit: 68719476736 +2019/08/10-17:00:28.121301 10b3645c0 Options.hard_pending_compaction_bytes_limit: 274877906944 +2019/08/10-17:00:28.121302 10b3645c0 Options.rate_limit_delay_max_milliseconds: 100 +2019/08/10-17:00:28.121303 10b3645c0 Options.disable_auto_compactions: 0 +2019/08/10-17:00:28.121331 10b3645c0 Options.compaction_style: kCompactionStyleLevel +2019/08/10-17:00:28.121334 10b3645c0 Options.compaction_pri: kByCompensatedSize +2019/08/10-17:00:28.121336 10b3645c0 Options.compaction_options_universal.size_ratio: 1 +2019/08/10-17:00:28.121336 10b3645c0 Options.compaction_options_universal.min_merge_width: 2 +2019/08/10-17:00:28.121337 10b3645c0 Options.compaction_options_universal.max_merge_width: 4294967295 +2019/08/10-17:00:28.121338 10b3645c0 Options.compaction_options_universal.max_size_amplification_percent: 200 +2019/08/10-17:00:28.121339 10b3645c0 Options.compaction_options_universal.compression_size_percent: -1 +2019/08/10-17:00:28.121341 10b3645c0 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize +2019/08/10-17:00:28.121341 10b3645c0 Options.compaction_options_fifo.max_table_files_size: 1073741824 +2019/08/10-17:00:28.121342 10b3645c0 Options.compaction_options_fifo.allow_compaction: 0 +2019/08/10-17:00:28.121343 10b3645c0 Options.compaction_options_fifo.ttl: 0 +2019/08/10-17:00:28.121344 10b3645c0 Options.table_properties_collectors: +2019/08/10-17:00:28.121345 10b3645c0 Options.inplace_update_support: 0 +2019/08/10-17:00:28.121346 10b3645c0 Options.inplace_update_num_locks: 10000 +2019/08/10-17:00:28.121346 10b3645c0 Options.memtable_prefix_bloom_size_ratio: 0.000000 +2019/08/10-17:00:28.121348 10b3645c0 Options.memtable_huge_page_size: 0 +2019/08/10-17:00:28.121348 10b3645c0 Options.bloom_locality: 0 +2019/08/10-17:00:28.121349 10b3645c0 Options.max_successive_merges: 0 +2019/08/10-17:00:28.121350 10b3645c0 Options.optimize_filters_for_hits: 0 +2019/08/10-17:00:28.121351 10b3645c0 Options.paranoid_file_checks: 0 +2019/08/10-17:00:28.121352 10b3645c0 Options.force_consistency_checks: 0 +2019/08/10-17:00:28.121352 10b3645c0 Options.report_bg_io_stats: 0 +2019/08/10-17:00:28.121353 10b3645c0 Options.ttl: 0 +2019/08/10-17:00:28.121419 10b3645c0 [/version_set.cc:3724] Recovered from manifest file:catalogue/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 +2019/08/10-17:00:28.121422 10b3645c0 [/version_set.cc:3732] Column family [default] (ID 0), log number is 0 +2019/08/10-17:00:28.124295 10b3645c0 [/db_impl_open.cc:1314] DB pointer 0x7fb590005200 +2019/08/10-17:00:28.124449 700002b73000 [WARN] [/db_impl.cc:669] ------- DUMPING STATS ------- +2019/08/10-17:00:28.124454 700002b73000 [WARN] [/db_impl.cc:670] +** DB Stats ** +Uptime(secs): 0.0 total, 0.0 interval +Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s +Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s +Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent +Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s +Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s +Interval stall: 00:00:0.000 H:M:S, 0.0 percent + +** Compaction Stats [default] ** +Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop +---------------------------------------------------------------------------------------------------------------------------------------------------------- + Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 + Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 +Uptime(secs): 0.0 total, 0.0 interval +Flush(GB): cumulative 0.000, interval 0.000 +AddFile(GB): cumulative 0.000, interval 0.000 +AddFile(Total Files): cumulative 0, interval 0 +AddFile(L0 Files): cumulative 0, interval 0 +AddFile(Keys): cumulative 0, interval 0 +Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count + +** File Read Latency Histogram By Level [default] ** + +** Compaction Stats [default] ** +Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop +---------------------------------------------------------------------------------------------------------------------------------------------------------- + Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 + Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 +Uptime(secs): 0.0 total, 0.0 interval +Flush(GB): cumulative 0.000, interval 0.000 +AddFile(GB): cumulative 0.000, interval 0.000 +AddFile(Total Files): cumulative 0, interval 0 +AddFile(L0 Files): cumulative 0, interval 0 +AddFile(Keys): cumulative 0, interval 0 +Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count + +** File Read Latency Histogram By Level [default] ** diff --git a/catalogue/LOG.old.1565431361639591 b/catalogue/LOG.old.1565431361639591 new file mode 100644 index 0000000..30db911 --- /dev/null +++ b/catalogue/LOG.old.1565431361639591 @@ -0,0 +1,240 @@ +2019/08/10-17:01:09.861388 11765d5c0 RocksDB version: 5.18.3 +2019/08/10-17:01:09.861825 11765d5c0 Git sha rocksdb_build_git_sha: +2019/08/10-17:01:09.861828 11765d5c0 Compile date Feb 21 2019 +2019/08/10-17:01:09.861829 11765d5c0 DB SUMMARY +2019/08/10-17:01:09.861895 11765d5c0 CURRENT file: CURRENT +2019/08/10-17:01:09.861897 11765d5c0 IDENTITY file: IDENTITY +2019/08/10-17:01:09.861903 11765d5c0 MANIFEST file: MANIFEST-000001 size: 13 Bytes +2019/08/10-17:01:09.861905 11765d5c0 SST files in catalogue dir, Total Num: 0, files: +2019/08/10-17:01:09.861906 11765d5c0 Write Ahead Log file in catalogue: 000003.log size: 0 ; +2019/08/10-17:01:09.861908 11765d5c0 Options.error_if_exists: 0 +2019/08/10-17:01:09.861911 11765d5c0 Options.create_if_missing: 1 +2019/08/10-17:01:09.861912 11765d5c0 Options.paranoid_checks: 0 +2019/08/10-17:01:09.861913 11765d5c0 Options.env: 0x10c9f22e8 +2019/08/10-17:01:09.861914 11765d5c0 Options.info_log: 0x7f9d42d02030 +2019/08/10-17:01:09.861915 11765d5c0 Options.max_file_opening_threads: 16 +2019/08/10-17:01:09.861916 11765d5c0 Options.statistics: 0x0 +2019/08/10-17:01:09.861917 11765d5c0 Options.use_fsync: 0 +2019/08/10-17:01:09.861918 11765d5c0 Options.max_log_file_size: 0 +2019/08/10-17:01:09.861919 11765d5c0 Options.max_manifest_file_size: 1073741824 +2019/08/10-17:01:09.861920 11765d5c0 Options.log_file_time_to_roll: 0 +2019/08/10-17:01:09.861921 11765d5c0 Options.keep_log_file_num: 1000 +2019/08/10-17:01:09.861922 11765d5c0 Options.recycle_log_file_num: 0 +2019/08/10-17:01:09.861923 11765d5c0 Options.allow_fallocate: 1 +2019/08/10-17:01:09.861924 11765d5c0 Options.allow_mmap_reads: 0 +2019/08/10-17:01:09.861925 11765d5c0 Options.allow_mmap_writes: 0 +2019/08/10-17:01:09.861926 11765d5c0 Options.use_direct_reads: 0 +2019/08/10-17:01:09.861927 11765d5c0 Options.use_direct_io_for_flush_and_compaction: 0 +2019/08/10-17:01:09.861927 11765d5c0 Options.create_missing_column_families: 0 +2019/08/10-17:01:09.861928 11765d5c0 Options.db_log_dir: +2019/08/10-17:01:09.861929 11765d5c0 Options.wal_dir: catalogue +2019/08/10-17:01:09.861930 11765d5c0 Options.table_cache_numshardbits: 6 +2019/08/10-17:01:09.861931 11765d5c0 Options.max_subcompactions: 1 +2019/08/10-17:01:09.861932 11765d5c0 Options.max_background_flushes: -1 +2019/08/10-17:01:09.861933 11765d5c0 Options.WAL_ttl_seconds: 0 +2019/08/10-17:01:09.861934 11765d5c0 Options.WAL_size_limit_MB: 0 +2019/08/10-17:01:09.861935 11765d5c0 Options.manifest_preallocation_size: 4194304 +2019/08/10-17:01:09.861936 11765d5c0 Options.is_fd_close_on_exec: 1 +2019/08/10-17:01:09.861937 11765d5c0 Options.advise_random_on_open: 1 +2019/08/10-17:01:09.861938 11765d5c0 Options.db_write_buffer_size: 0 +2019/08/10-17:01:09.861939 11765d5c0 Options.write_buffer_manager: 0x7f9d42d01cf0 +2019/08/10-17:01:09.861940 11765d5c0 Options.access_hint_on_compaction_start: 1 +2019/08/10-17:01:09.861941 11765d5c0 Options.new_table_reader_for_compaction_inputs: 0 +2019/08/10-17:01:09.861942 11765d5c0 Options.random_access_max_buffer_size: 1048576 +2019/08/10-17:01:09.861942 11765d5c0 Options.use_adaptive_mutex: 0 +2019/08/10-17:01:09.861943 11765d5c0 Options.rate_limiter: 0x0 +2019/08/10-17:01:09.861945 11765d5c0 Options.sst_file_manager.rate_bytes_per_sec: 0 +2019/08/10-17:01:09.861946 11765d5c0 Options.wal_recovery_mode: 2 +2019/08/10-17:01:09.861947 11765d5c0 Options.enable_thread_tracking: 0 +2019/08/10-17:01:09.861948 11765d5c0 Options.enable_pipelined_write: 0 +2019/08/10-17:01:09.861970 11765d5c0 Options.allow_concurrent_memtable_write: 1 +2019/08/10-17:01:09.861972 11765d5c0 Options.enable_write_thread_adaptive_yield: 1 +2019/08/10-17:01:09.861973 11765d5c0 Options.write_thread_max_yield_usec: 100 +2019/08/10-17:01:09.861974 11765d5c0 Options.write_thread_slow_yield_usec: 3 +2019/08/10-17:01:09.861975 11765d5c0 Options.row_cache: None +2019/08/10-17:01:09.861976 11765d5c0 Options.wal_filter: None +2019/08/10-17:01:09.861977 11765d5c0 Options.avoid_flush_during_recovery: 0 +2019/08/10-17:01:09.861978 11765d5c0 Options.allow_ingest_behind: 0 +2019/08/10-17:01:09.861978 11765d5c0 Options.preserve_deletes: 0 +2019/08/10-17:01:09.861979 11765d5c0 Options.two_write_queues: 0 +2019/08/10-17:01:09.861980 11765d5c0 Options.manual_wal_flush: 0 +2019/08/10-17:01:09.861981 11765d5c0 Options.max_background_jobs: 2 +2019/08/10-17:01:09.861982 11765d5c0 Options.max_background_compactions: -1 +2019/08/10-17:01:09.861983 11765d5c0 Options.avoid_flush_during_shutdown: 0 +2019/08/10-17:01:09.861984 11765d5c0 Options.writable_file_max_buffer_size: 1048576 +2019/08/10-17:01:09.861985 11765d5c0 Options.delayed_write_rate : 16777216 +2019/08/10-17:01:09.861986 11765d5c0 Options.max_total_wal_size: 0 +2019/08/10-17:01:09.861987 11765d5c0 Options.delete_obsolete_files_period_micros: 21600000000 +2019/08/10-17:01:09.861988 11765d5c0 Options.stats_dump_period_sec: 600 +2019/08/10-17:01:09.861989 11765d5c0 Options.max_open_files: 256 +2019/08/10-17:01:09.861990 11765d5c0 Options.bytes_per_sync: 0 +2019/08/10-17:01:09.861991 11765d5c0 Options.wal_bytes_per_sync: 0 +2019/08/10-17:01:09.861991 11765d5c0 Options.compaction_readahead_size: 0 +2019/08/10-17:01:09.861992 11765d5c0 Compression algorithms supported: +2019/08/10-17:01:09.861993 11765d5c0 kZSTD supported: 0 +2019/08/10-17:01:09.861994 11765d5c0 kZlibCompression supported: 1 +2019/08/10-17:01:09.861996 11765d5c0 kXpressCompression supported: 0 +2019/08/10-17:01:09.861997 11765d5c0 kSnappyCompression supported: 1 +2019/08/10-17:01:09.861997 11765d5c0 kZSTDNotFinalCompression supported: 0 +2019/08/10-17:01:09.861998 11765d5c0 kLZ4HCCompression supported: 1 +2019/08/10-17:01:09.861999 11765d5c0 kLZ4Compression supported: 1 +2019/08/10-17:01:09.862000 11765d5c0 kBZip2Compression supported: 1 +2019/08/10-17:01:09.862004 11765d5c0 Fast CRC32 supported: Supported on x86 +2019/08/10-17:01:09.862180 11765d5c0 [/version_set.cc:3508] Recovering from manifest file: MANIFEST-000001 +2019/08/10-17:01:09.862231 11765d5c0 [/column_family.cc:474] --------------- Options for column family [default]: +2019/08/10-17:01:09.862236 11765d5c0 Options.comparator: leveldb.BytewiseComparator +2019/08/10-17:01:09.862237 11765d5c0 Options.merge_operator: None +2019/08/10-17:01:09.862238 11765d5c0 Options.compaction_filter: None +2019/08/10-17:01:09.862239 11765d5c0 Options.compaction_filter_factory: None +2019/08/10-17:01:09.862240 11765d5c0 Options.memtable_factory: SkipListFactory +2019/08/10-17:01:09.862241 11765d5c0 Options.table_factory: BlockBasedTable +2019/08/10-17:01:09.862263 11765d5c0 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7f9d42d01160) + cache_index_and_filter_blocks: 0 + cache_index_and_filter_blocks_with_high_priority: 0 + pin_l0_filter_and_index_blocks_in_cache: 0 + pin_top_level_index_and_filter: 1 + index_type: 0 + hash_index_allow_collision: 1 + checksum: 1 + no_block_cache: 0 + block_cache: 0x7f9d42d011a8 + block_cache_name: LRUCache + block_cache_options: + capacity : 8388608 + num_shard_bits : 4 + strict_capacity_limit : 0 + memory_allocator : None + high_pri_pool_ratio: 0.000 + block_cache_compressed: 0x0 + persistent_cache: 0x0 + block_size: 4096 + block_size_deviation: 10 + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4096 + partition_filters: 0 + use_delta_encoding: 1 + filter_policy: nullptr + whole_key_filtering: 1 + verify_compression: 0 + read_amp_bytes_per_bit: 0 + format_version: 2 + enable_index_compression: 1 + block_align: 0 +2019/08/10-17:01:09.862291 11765d5c0 Options.write_buffer_size: 4194304 +2019/08/10-17:01:09.862293 11765d5c0 Options.max_write_buffer_number: 2 +2019/08/10-17:01:09.862296 11765d5c0 Options.compression: Snappy +2019/08/10-17:01:09.862297 11765d5c0 Options.bottommost_compression: Disabled +2019/08/10-17:01:09.862298 11765d5c0 Options.prefix_extractor: nullptr +2019/08/10-17:01:09.862299 11765d5c0 Options.memtable_insert_with_hint_prefix_extractor: nullptr +2019/08/10-17:01:09.862300 11765d5c0 Options.num_levels: 7 +2019/08/10-17:01:09.862301 11765d5c0 Options.min_write_buffer_number_to_merge: 1 +2019/08/10-17:01:09.862302 11765d5c0 Options.max_write_buffer_number_to_maintain: 0 +2019/08/10-17:01:09.862303 11765d5c0 Options.bottommost_compression_opts.window_bits: -14 +2019/08/10-17:01:09.862304 11765d5c0 Options.bottommost_compression_opts.level: 32767 +2019/08/10-17:01:09.862305 11765d5c0 Options.bottommost_compression_opts.strategy: 0 +2019/08/10-17:01:09.862306 11765d5c0 Options.bottommost_compression_opts.max_dict_bytes: 0 +2019/08/10-17:01:09.862307 11765d5c0 Options.bottommost_compression_opts.zstd_max_train_bytes: 0 +2019/08/10-17:01:09.862308 11765d5c0 Options.bottommost_compression_opts.enabled: false +2019/08/10-17:01:09.862309 11765d5c0 Options.compression_opts.window_bits: -14 +2019/08/10-17:01:09.862310 11765d5c0 Options.compression_opts.level: 32767 +2019/08/10-17:01:09.862311 11765d5c0 Options.compression_opts.strategy: 0 +2019/08/10-17:01:09.862311 11765d5c0 Options.compression_opts.max_dict_bytes: 0 +2019/08/10-17:01:09.862312 11765d5c0 Options.compression_opts.zstd_max_train_bytes: 0 +2019/08/10-17:01:09.862313 11765d5c0 Options.compression_opts.enabled: false +2019/08/10-17:01:09.862314 11765d5c0 Options.level0_file_num_compaction_trigger: 4 +2019/08/10-17:01:09.862315 11765d5c0 Options.level0_slowdown_writes_trigger: 20 +2019/08/10-17:01:09.862316 11765d5c0 Options.level0_stop_writes_trigger: 36 +2019/08/10-17:01:09.862317 11765d5c0 Options.target_file_size_base: 67108864 +2019/08/10-17:01:09.862318 11765d5c0 Options.target_file_size_multiplier: 1 +2019/08/10-17:01:09.862319 11765d5c0 Options.max_bytes_for_level_base: 268435456 +2019/08/10-17:01:09.862320 11765d5c0 Options.level_compaction_dynamic_level_bytes: 0 +2019/08/10-17:01:09.862321 11765d5c0 Options.max_bytes_for_level_multiplier: 10.000000 +2019/08/10-17:01:09.862323 11765d5c0 Options.max_bytes_for_level_multiplier_addtl[0]: 1 +2019/08/10-17:01:09.862324 11765d5c0 Options.max_bytes_for_level_multiplier_addtl[1]: 1 +2019/08/10-17:01:09.862325 11765d5c0 Options.max_bytes_for_level_multiplier_addtl[2]: 1 +2019/08/10-17:01:09.862326 11765d5c0 Options.max_bytes_for_level_multiplier_addtl[3]: 1 +2019/08/10-17:01:09.862327 11765d5c0 Options.max_bytes_for_level_multiplier_addtl[4]: 1 +2019/08/10-17:01:09.862327 11765d5c0 Options.max_bytes_for_level_multiplier_addtl[5]: 1 +2019/08/10-17:01:09.862328 11765d5c0 Options.max_bytes_for_level_multiplier_addtl[6]: 1 +2019/08/10-17:01:09.862329 11765d5c0 Options.max_sequential_skip_in_iterations: 8 +2019/08/10-17:01:09.862330 11765d5c0 Options.max_compaction_bytes: 1677721600 +2019/08/10-17:01:09.862331 11765d5c0 Options.arena_block_size: 524288 +2019/08/10-17:01:09.862332 11765d5c0 Options.soft_pending_compaction_bytes_limit: 68719476736 +2019/08/10-17:01:09.862333 11765d5c0 Options.hard_pending_compaction_bytes_limit: 274877906944 +2019/08/10-17:01:09.862349 11765d5c0 Options.rate_limit_delay_max_milliseconds: 100 +2019/08/10-17:01:09.862351 11765d5c0 Options.disable_auto_compactions: 0 +2019/08/10-17:01:09.862352 11765d5c0 Options.compaction_style: kCompactionStyleLevel +2019/08/10-17:01:09.862353 11765d5c0 Options.compaction_pri: kByCompensatedSize +2019/08/10-17:01:09.862354 11765d5c0 Options.compaction_options_universal.size_ratio: 1 +2019/08/10-17:01:09.862355 11765d5c0 Options.compaction_options_universal.min_merge_width: 2 +2019/08/10-17:01:09.862356 11765d5c0 Options.compaction_options_universal.max_merge_width: 4294967295 +2019/08/10-17:01:09.862357 11765d5c0 Options.compaction_options_universal.max_size_amplification_percent: 200 +2019/08/10-17:01:09.862358 11765d5c0 Options.compaction_options_universal.compression_size_percent: -1 +2019/08/10-17:01:09.862359 11765d5c0 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize +2019/08/10-17:01:09.862360 11765d5c0 Options.compaction_options_fifo.max_table_files_size: 1073741824 +2019/08/10-17:01:09.862361 11765d5c0 Options.compaction_options_fifo.allow_compaction: 0 +2019/08/10-17:01:09.862362 11765d5c0 Options.compaction_options_fifo.ttl: 0 +2019/08/10-17:01:09.862363 11765d5c0 Options.table_properties_collectors: +2019/08/10-17:01:09.862364 11765d5c0 Options.inplace_update_support: 0 +2019/08/10-17:01:09.862365 11765d5c0 Options.inplace_update_num_locks: 10000 +2019/08/10-17:01:09.862366 11765d5c0 Options.memtable_prefix_bloom_size_ratio: 0.000000 +2019/08/10-17:01:09.862367 11765d5c0 Options.memtable_huge_page_size: 0 +2019/08/10-17:01:09.862368 11765d5c0 Options.bloom_locality: 0 +2019/08/10-17:01:09.862369 11765d5c0 Options.max_successive_merges: 0 +2019/08/10-17:01:09.862369 11765d5c0 Options.optimize_filters_for_hits: 0 +2019/08/10-17:01:09.862370 11765d5c0 Options.paranoid_file_checks: 0 +2019/08/10-17:01:09.862371 11765d5c0 Options.force_consistency_checks: 0 +2019/08/10-17:01:09.862372 11765d5c0 Options.report_bg_io_stats: 0 +2019/08/10-17:01:09.862373 11765d5c0 Options.ttl: 0 +2019/08/10-17:01:09.862443 11765d5c0 [/version_set.cc:3724] Recovered from manifest file:catalogue/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 +2019/08/10-17:01:09.862446 11765d5c0 [/version_set.cc:3732] Column family [default] (ID 0), log number is 0 +2019/08/10-17:01:09.862535 11765d5c0 EVENT_LOG_v1 {"time_micros": 1565431269862527, "job": 1, "event": "recovery_started", "log_files": [3]} +2019/08/10-17:01:09.862544 11765d5c0 [/db_impl_open.cc:578] Recovering log #3 mode 2 +2019/08/10-17:01:09.862597 11765d5c0 [/version_set.cc:3037] Creating manifest 5 +2019/08/10-17:01:09.863284 11765d5c0 EVENT_LOG_v1 {"time_micros": 1565431269863280, "job": 1, "event": "recovery_finished"} +2019/08/10-17:01:09.866266 11765d5c0 [/db_impl_open.cc:1314] DB pointer 0x7f9d44002a00 +2019/08/10-17:01:09.866451 70000bd0e000 [WARN] [/db_impl.cc:669] ------- DUMPING STATS ------- +2019/08/10-17:01:09.866460 70000bd0e000 [WARN] [/db_impl.cc:670] +** DB Stats ** +Uptime(secs): 0.0 total, 0.0 interval +Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s +Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s +Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent +Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s +Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s +Interval stall: 00:00:0.000 H:M:S, 0.0 percent + +** Compaction Stats [default] ** +Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop +---------------------------------------------------------------------------------------------------------------------------------------------------------- + Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 + Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 +Uptime(secs): 0.0 total, 0.0 interval +Flush(GB): cumulative 0.000, interval 0.000 +AddFile(GB): cumulative 0.000, interval 0.000 +AddFile(Total Files): cumulative 0, interval 0 +AddFile(L0 Files): cumulative 0, interval 0 +AddFile(Keys): cumulative 0, interval 0 +Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count + +** File Read Latency Histogram By Level [default] ** + +** Compaction Stats [default] ** +Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop +---------------------------------------------------------------------------------------------------------------------------------------------------------- + Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 + Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 +Uptime(secs): 0.0 total, 0.0 interval +Flush(GB): cumulative 0.000, interval 0.000 +AddFile(GB): cumulative 0.000, interval 0.000 +AddFile(Total Files): cumulative 0, interval 0 +AddFile(L0 Files): cumulative 0, interval 0 +AddFile(Keys): cumulative 0, interval 0 +Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count + +** File Read Latency Histogram By Level [default] ** diff --git a/catalogue/MANIFEST-000008 b/catalogue/MANIFEST-000008 new file mode 100644 index 0000000..d65bcd6 Binary files /dev/null and b/catalogue/MANIFEST-000008 differ diff --git a/catalogue/OPTIONS-000008 b/catalogue/OPTIONS-000008 new file mode 100644 index 0000000..43ac078 --- /dev/null +++ b/catalogue/OPTIONS-000008 @@ -0,0 +1,150 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=5.18.3 + options_file_version=1.1 + +[DBOptions] + allow_mmap_writes=false + base_background_compactions=-1 + new_table_reader_for_compaction_inputs=false + db_log_dir= + wal_recovery_mode=kPointInTimeRecovery + use_direct_reads=false + write_thread_max_yield_usec=100 + max_manifest_file_size=1073741824 + allow_2pc=false + allow_fallocate=true + fail_if_options_file_error=false + allow_ingest_behind=false + allow_mmap_reads=false + skip_log_error_on_recovery=false + recycle_log_file_num=0 + delete_obsolete_files_period_micros=21600000000 + compaction_readahead_size=0 + use_direct_io_for_flush_and_compaction=false + log_file_time_to_roll=0 + create_missing_column_families=false + advise_random_on_open=true + max_log_file_size=0 + stats_dump_period_sec=600 + enable_thread_tracking=false + use_adaptive_mutex=false + create_if_missing=true + is_fd_close_on_exec=true + max_background_flushes=-1 + manifest_preallocation_size=4194304 + error_if_exists=false + skip_stats_update_on_db_open=false + max_open_files=256 + random_access_max_buffer_size=1048576 + use_fsync=false + max_background_jobs=2 + two_write_queues=false + max_background_compactions=-1 + max_file_opening_threads=16 + table_cache_numshardbits=6 + keep_log_file_num=1000 + avoid_flush_during_shutdown=false + db_write_buffer_size=0 + max_total_wal_size=0 + wal_dir=catalogue + max_subcompactions=1 + atomic_flush=false + WAL_size_limit_MB=0 + paranoid_checks=false + allow_concurrent_memtable_write=true + writable_file_max_buffer_size=1048576 + WAL_ttl_seconds=0 + delayed_write_rate=16777216 + bytes_per_sync=0 + wal_bytes_per_sync=0 + enable_pipelined_write=false + enable_write_thread_adaptive_yield=true + write_thread_slow_yield_usec=3 + access_hint_on_compaction_start=NORMAL + info_log_level=INFO_LEVEL + dump_malloc_stats=false + avoid_flush_during_recovery=false + preserve_deletes=false + manual_wal_flush=false + + +[CFOptions "default"] + report_bg_io_stats=false + inplace_update_support=false + max_compaction_bytes=1677721600 + disable_auto_compactions=false + write_buffer_size=4194304 + bloom_locality=0 + max_bytes_for_level_multiplier=10.000000 + compaction_filter_factory=nullptr + optimize_filters_for_hits=false + target_file_size_base=67108864 + max_write_buffer_number_to_maintain=0 + hard_pending_compaction_bytes_limit=274877906944 + paranoid_file_checks=false + memtable_prefix_bloom_size_ratio=0.000000 + force_consistency_checks=false + max_write_buffer_number=2 + ttl=0 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + level0_slowdown_writes_trigger=20 + level_compaction_dynamic_level_bytes=false + compaction_options_fifo={allow_compaction=false;ttl=0;max_table_files_size=1073741824;} + inplace_update_num_locks=10000 + level0_file_num_compaction_trigger=4 + compression=kSnappyCompression + level0_stop_writes_trigger=36 + num_levels=7 + table_factory=BlockBasedTable + compression_per_level= + target_file_size_multiplier=1 + min_write_buffer_number_to_merge=1 + arena_block_size=524288 + max_successive_merges=0 + memtable_huge_page_size=0 + compaction_pri=kByCompensatedSize + soft_pending_compaction_bytes_limit=68719476736 + max_bytes_for_level_base=268435456 + comparator=leveldb.BytewiseComparator + max_sequential_skip_in_iterations=8 + bottommost_compression=kDisableCompressionOption + prefix_extractor=nullptr + memtable_insert_with_hint_prefix_extractor=nullptr + memtable_factory=SkipListFactory + compaction_filter=nullptr + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;} + merge_operator=nullptr + compaction_style=kCompactionStyleLevel + +[TableOptions/BlockBasedTable "default"] + format_version=2 + whole_key_filtering=true + verify_compression=false + partition_filters=false + enable_index_compression=true + checksum=kCRC32c + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + read_amp_bytes_per_bit=8589934592 + cache_index_and_filter_blocks=false + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=false + hash_index_allow_collision=true + cache_index_and_filter_blocks_with_high_priority=false + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/catalogue/OPTIONS-000011 b/catalogue/OPTIONS-000011 new file mode 100644 index 0000000..43ac078 --- /dev/null +++ b/catalogue/OPTIONS-000011 @@ -0,0 +1,150 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=5.18.3 + options_file_version=1.1 + +[DBOptions] + allow_mmap_writes=false + base_background_compactions=-1 + new_table_reader_for_compaction_inputs=false + db_log_dir= + wal_recovery_mode=kPointInTimeRecovery + use_direct_reads=false + write_thread_max_yield_usec=100 + max_manifest_file_size=1073741824 + allow_2pc=false + allow_fallocate=true + fail_if_options_file_error=false + allow_ingest_behind=false + allow_mmap_reads=false + skip_log_error_on_recovery=false + recycle_log_file_num=0 + delete_obsolete_files_period_micros=21600000000 + compaction_readahead_size=0 + use_direct_io_for_flush_and_compaction=false + log_file_time_to_roll=0 + create_missing_column_families=false + advise_random_on_open=true + max_log_file_size=0 + stats_dump_period_sec=600 + enable_thread_tracking=false + use_adaptive_mutex=false + create_if_missing=true + is_fd_close_on_exec=true + max_background_flushes=-1 + manifest_preallocation_size=4194304 + error_if_exists=false + skip_stats_update_on_db_open=false + max_open_files=256 + random_access_max_buffer_size=1048576 + use_fsync=false + max_background_jobs=2 + two_write_queues=false + max_background_compactions=-1 + max_file_opening_threads=16 + table_cache_numshardbits=6 + keep_log_file_num=1000 + avoid_flush_during_shutdown=false + db_write_buffer_size=0 + max_total_wal_size=0 + wal_dir=catalogue + max_subcompactions=1 + atomic_flush=false + WAL_size_limit_MB=0 + paranoid_checks=false + allow_concurrent_memtable_write=true + writable_file_max_buffer_size=1048576 + WAL_ttl_seconds=0 + delayed_write_rate=16777216 + bytes_per_sync=0 + wal_bytes_per_sync=0 + enable_pipelined_write=false + enable_write_thread_adaptive_yield=true + write_thread_slow_yield_usec=3 + access_hint_on_compaction_start=NORMAL + info_log_level=INFO_LEVEL + dump_malloc_stats=false + avoid_flush_during_recovery=false + preserve_deletes=false + manual_wal_flush=false + + +[CFOptions "default"] + report_bg_io_stats=false + inplace_update_support=false + max_compaction_bytes=1677721600 + disable_auto_compactions=false + write_buffer_size=4194304 + bloom_locality=0 + max_bytes_for_level_multiplier=10.000000 + compaction_filter_factory=nullptr + optimize_filters_for_hits=false + target_file_size_base=67108864 + max_write_buffer_number_to_maintain=0 + hard_pending_compaction_bytes_limit=274877906944 + paranoid_file_checks=false + memtable_prefix_bloom_size_ratio=0.000000 + force_consistency_checks=false + max_write_buffer_number=2 + ttl=0 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + level0_slowdown_writes_trigger=20 + level_compaction_dynamic_level_bytes=false + compaction_options_fifo={allow_compaction=false;ttl=0;max_table_files_size=1073741824;} + inplace_update_num_locks=10000 + level0_file_num_compaction_trigger=4 + compression=kSnappyCompression + level0_stop_writes_trigger=36 + num_levels=7 + table_factory=BlockBasedTable + compression_per_level= + target_file_size_multiplier=1 + min_write_buffer_number_to_merge=1 + arena_block_size=524288 + max_successive_merges=0 + memtable_huge_page_size=0 + compaction_pri=kByCompensatedSize + soft_pending_compaction_bytes_limit=68719476736 + max_bytes_for_level_base=268435456 + comparator=leveldb.BytewiseComparator + max_sequential_skip_in_iterations=8 + bottommost_compression=kDisableCompressionOption + prefix_extractor=nullptr + memtable_insert_with_hint_prefix_extractor=nullptr + memtable_factory=SkipListFactory + compaction_filter=nullptr + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;} + merge_operator=nullptr + compaction_style=kCompactionStyleLevel + +[TableOptions/BlockBasedTable "default"] + format_version=2 + whole_key_filtering=true + verify_compression=false + partition_filters=false + enable_index_compression=true + checksum=kCRC32c + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + read_amp_bytes_per_bit=8589934592 + cache_index_and_filter_blocks=false + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=false + hash_index_allow_collision=true + cache_index_and_filter_blocks_with_high_priority=false + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/src/Hydra/Core/Domain/DB.hs b/src/Hydra/Core/Domain/DB.hs index 3f56f4d..86ef84a 100644 --- a/src/Hydra/Core/Domain/DB.hs +++ b/src/Hydra/Core/Domain/DB.hs @@ -12,6 +12,7 @@ data DBErrorType = SystemError | KeyNotFound | InvalidType + | DecodingFailed deriving (Generic, Ord, Eq, Enum, Bounded, Show, Read) data DBError = DBError DBErrorType Text diff --git a/src/Hydra/Core/Domain/KVDB.hs b/src/Hydra/Core/Domain/KVDB.hs index 551bf7a..8c1ea3e 100644 --- a/src/Hydra/Core/Domain/KVDB.hs +++ b/src/Hydra/Core/Domain/KVDB.hs @@ -11,66 +11,48 @@ import qualified Data.ByteString.Lazy as LBS import Hydra.Core.Domain.DB data KVDBOptions = KVDBOptions - { _createIfMissing :: Bool - , _errorIfExists :: Bool - } - deriving (Show, Generic) + { _createIfMissing :: Bool + , _errorIfExists :: Bool + } + deriving (Show, Generic) data KVDBConfig db = KVDBConfig - { _path :: FilePath - , _options :: KVDBOptions - } - deriving (Show, Generic) + { _path :: FilePath + , _options :: KVDBOptions + } + deriving (Show, Generic) type KVDBKey = ByteString type KVDBValue = ByteString --- class DB db where --- getDbName :: FilePath --- --- class DBEntity entity where --- data DBKey entity :: * --- data DBValue entity :: * --- --- class DBEntity entity => ToDBKey entity src where --- toDBKey :: src -> DBKey entity --- --- class DBEntity entity => ToDBValue entity src where --- toDBValue :: src -> DBValue entity --- --- class (DB db, DBEntity entity) => DBModelEntity db entity --- --- class DBModelEntity db entity => RawDBEntity db entity where --- toRawDBKey :: DBKey entity -> DBKey --- toRawDBValue :: DBValue entity -> DBValue --- fromRawDBValue :: DBValue -> Maybe (DBValue entity) --- --- -- TODO: this doesn't work by some strange reason. --- default toRawDBValue :: ToJSON (DBValue entity) => DBValue entity -> DBValue --- toRawDBValue = LBS.toStrict . A.encode --- default fromRawDBValue :: FromJSON (DBValue entity) => DBValue -> Maybe (DBValue entity) --- fromRawDBValue = A.decode . LBS.fromStrict --- --- type DBE entity = (DBKey entity, DBValue entity) --- --- defaultDbOptions :: DBOptions --- defaultDbOptions = DBOptions --- { _createIfMissing = False --- , _errorIfExists = False --- } --- --- toDBEntity --- :: (ToDBKey entity src, ToDBValue entity src) --- => src --- -> DBE entity --- toDBEntity src = (toDBKey src, toDBValue src) --- --- instance ToJSON (Storage db) where toJSON = genericToJSON noLensPrefix --- instance FromJSON (Storage db) where parseJSON = genericParseJSON noLensPrefix --- --- instance ToJSON DBOptions where toJSON = genericToJSON noLensPrefix --- instance FromJSON DBOptions where parseJSON = genericParseJSON noLensPrefix --- --- instance ToJSON (DBConfig db) where toJSON = genericToJSON noLensPrefix --- instance FromJSON (DBConfig db) where parseJSON = genericParseJSON noLensPrefix +-- Domain data type /= DB data type. +-- DB data type can be very different. +-- Keys can be very different and can be obtained from different sources. +class DBEntity entity where + data KeyEntity entity :: * + data ValueEntity entity :: * + +class AsKeyEntity entity src where + toKeyEntity :: src -> KeyEntity entity + +class AsValueEntity entity src where + toValueEntity :: src -> ValueEntity entity + fromValueEntity :: ValueEntity entity -> src + +class RawDBEntity entity where + toDBKey :: KeyEntity entity -> KVDBKey + toDBValue :: ValueEntity entity -> KVDBValue + fromDBValue :: KVDBValue -> Maybe (ValueEntity entity) + +toDBKeyJSON :: ToJSON (KeyEntity entity) => KeyEntity entity -> KVDBKey +toDBKeyJSON = LBS.toStrict . A.encode + +toDBValueJSON :: ToJSON (ValueEntity entity) => ValueEntity entity -> KVDBValue +toDBValueJSON = LBS.toStrict . A.encode + +fromDBKeyJSON :: FromJSON (KeyEntity entity) => KVDBKey -> Maybe (KeyEntity entity) +fromDBKeyJSON = A.decode . LBS.fromStrict + +fromDBValueJSON :: FromJSON (ValueEntity entity) => KVDBValue -> Maybe (ValueEntity entity) +fromDBValueJSON = A.decode . LBS.fromStrict diff --git a/src/Hydra/Core/KVDB/Interpreter.hs b/src/Hydra/Core/KVDB/Interpreter.hs index 0e20914..b9cada0 100644 --- a/src/Hydra/Core/KVDB/Interpreter.hs +++ b/src/Hydra/Core/KVDB/Interpreter.hs @@ -15,18 +15,17 @@ writeOpts = Rocks.defaultWriteOptions } interpretKVDBF :: Rocks.DB -> L.KVDBF a -> IO a -interpretKVDBF db (L.GetValue key) = do +interpretKVDBF db (L.Load key next) = do mbVal <- Rocks.get db Rocks.defaultReadOptions key - pure $ case mbVal of + pure $ next $ case mbVal of Nothing -> Left $ D.DBError D.KeyNotFound $ show key Just val -> Right val -interpretKVDBF db (L.PutValue key val) = - Right <$> Rocks.put db writeOpts key val +interpretKVDBF db (L.Save key val next) = + next . Right <$> Rocks.put db writeOpts key val runKVDBL :: R.RocksDBHandle -> L.KVDBL db a -> IO a runKVDBL handle act = do db <- takeMVar handle - -- res <- foldFree (interpretKVDBF db) act - res <- interpretKVDBF db act + res <- foldFree (interpretKVDBF db) act putMVar handle db pure res diff --git a/src/Hydra/Core/KVDB/Language.hs b/src/Hydra/Core/KVDB/Language.hs index c369791..3047af5 100644 --- a/src/Hydra/Core/KVDB/Language.hs +++ b/src/Hydra/Core/KVDB/Language.hs @@ -1,45 +1,63 @@ {-# LANGUAGE AllowAmbiguousTypes #-} +{-# LANGUAGE TemplateHaskell #-} module Hydra.Core.KVDB.Language where import Hydra.Prelude --- import Data.Typeable (typeOf) - import qualified Hydra.Core.Domain.DB as D import qualified Hydra.Core.Domain.KVDB as D --- data KVDBF db a where --- GetValue :: D.KVDBKey -> (D.DBResult D.KVDBValue -> next) -> KVDBF db next --- PutValue :: D.KVDBKey -> D.KVDBValue -> (D.DBResult () -> next) -> KVDBF db next --- deriving (Functor) --- --- type KVDBL db = Free (KVDBF db) +import Language.Haskell.TH.MakeFunctor (makeFunctorInstance) -data KVDBF a where - GetValue :: D.KVDBKey -> KVDBF (D.DBResult D.KVDBValue) - PutValue :: D.KVDBKey -> D.KVDBValue -> KVDBF (D.DBResult ()) +data KVDBF next where + Save :: D.KVDBKey -> D.KVDBValue -> (D.DBResult () -> next) -> KVDBF next + Load :: D.KVDBKey -> (D.DBResult D.KVDBValue -> next) -> KVDBF next -type KVDBL db = KVDBF -getValue :: D.KVDBKey -> KVDBL db (D.DBResult D.KVDBValue) -getValue = GetValue +makeFunctorInstance ''KVDBF -putValue :: D.KVDBKey -> D.KVDBValue -> KVDBL db (D.DBResult ()) -putValue = PutValue +type KVDBL db = Free KVDBF + +save' + :: forall src entity db + . D.DBEntity entity + => D.AsKeyEntity entity src + => D.AsValueEntity entity src + => D.RawDBEntity entity + => src + -> KVDBL db (D.DBResult ()) +save' src = liftF $ Save dbkey dbval id + where + k :: D.KeyEntity entity + k = D.toKeyEntity src + v :: D.ValueEntity entity + v = D.toValueEntity src + dbkey = D.toDBKey k + dbval = D.toDBValue v + +load' + :: forall entity dst db + . D.DBEntity entity + => D.AsKeyEntity entity dst + => D.AsValueEntity entity dst + => D.RawDBEntity entity + => Show (D.KeyEntity entity) + => D.KeyEntity entity + -> KVDBL db (D.DBResult dst) +load' key = do + eRawVal <- liftF $ Load (D.toDBKey key) id + pure $ case eRawVal of + Left err -> Left err + Right val -> maybe (decodingErr val) (Right . D.fromValueEntity) $ mbE val + where + mbE :: D.KVDBValue -> Maybe (D.ValueEntity entity) + mbE = D.fromDBValue + decodingErr val = Left + $ D.DBError D.DecodingFailed + $ "Failed to decode entity, k: " + <> show key <> ", v: " <> show val --- putEntity --- :: forall entity db --- . D.RawDBEntity db entity --- => D.DBKey entity --- -> D.DBValue entity --- -> KVDBL db (D.DBResult ()) --- putEntity dbKey dbVal = let --- rawKey = D.toRawDBKey @db dbKey --- rawVal = D.toRawDBValue @db dbVal --- in putValue rawKey rawVal --- --- -- | Puts a typed entity to the corresponding DB. -- putEntity' -- :: forall entity db src -- . D.RawDBEntity db entity