mirror of
https://github.com/wader/fq.git
synced 2024-08-16 23:40:43 +03:00
Merge pull request #824 from mikez/leveldb
leveldb: Add LevelDB support
This commit is contained in:
commit
b05c7ec54f
@ -102,6 +102,9 @@ ipv6_packet,
|
||||
jpeg,
|
||||
json,
|
||||
jsonl,
|
||||
[leveldb_descriptor](doc/formats.md#leveldb_descriptor),
|
||||
[leveldb_log](doc/formats.md#leveldb_log),
|
||||
[leveldb_table](doc/formats.md#leveldb_table),
|
||||
[luajit](doc/formats.md#luajit),
|
||||
[macho](doc/formats.md#macho),
|
||||
macho_fat,
|
||||
|
316
doc/formats.md
316
doc/formats.md
@ -2,138 +2,141 @@
|
||||
|
||||
[fq -rn -L . 'include "formats"; formats_table']: sh-start
|
||||
|
||||
|Name |Description |Dependencies|
|
||||
|- |- |-|
|
||||
|[`aac_frame`](#aac_frame) |Advanced Audio Coding frame |<sub></sub>|
|
||||
|`adts` |Audio Data Transport Stream |<sub>`adts_frame`</sub>|
|
||||
|`adts_frame` |Audio Data Transport Stream frame |<sub>`aac_frame`</sub>|
|
||||
|`aiff` |Audio Interchange File Format |<sub></sub>|
|
||||
|`amf0` |Action Message Format 0 |<sub></sub>|
|
||||
|`apev2` |APEv2 metadata tag |<sub>`image`</sub>|
|
||||
|[`apple_bookmark`](#apple_bookmark) |Apple BookmarkData |<sub></sub>|
|
||||
|`ar` |Unix archive |<sub>`probe`</sub>|
|
||||
|[`asn1_ber`](#asn1_ber) |ASN1 BER (basic encoding rules, also CER and DER) |<sub></sub>|
|
||||
|`av1_ccr` |AV1 Codec Configuration Record |<sub></sub>|
|
||||
|`av1_frame` |AV1 frame |<sub>`av1_obu`</sub>|
|
||||
|`av1_obu` |AV1 Open Bitstream Unit |<sub></sub>|
|
||||
|`avc_annexb` |H.264/AVC Annex B |<sub>`avc_nalu`</sub>|
|
||||
|[`avc_au`](#avc_au) |H.264/AVC Access Unit |<sub>`avc_nalu`</sub>|
|
||||
|`avc_dcr` |H.264/AVC Decoder Configuration Record |<sub>`avc_nalu`</sub>|
|
||||
|`avc_nalu` |H.264/AVC Network Access Layer Unit |<sub>`avc_sps` `avc_pps` `avc_sei`</sub>|
|
||||
|`avc_pps` |H.264/AVC Picture Parameter Set |<sub></sub>|
|
||||
|`avc_sei` |H.264/AVC Supplemental Enhancement Information |<sub></sub>|
|
||||
|`avc_sps` |H.264/AVC Sequence Parameter Set |<sub></sub>|
|
||||
|[`avi`](#avi) |Audio Video Interleaved |<sub>`avc_au` `hevc_au` `mp3_frame` `flac_frame`</sub>|
|
||||
|[`avro_ocf`](#avro_ocf) |Avro object container file |<sub></sub>|
|
||||
|[`bencode`](#bencode) |BitTorrent bencoding |<sub></sub>|
|
||||
|`bitcoin_blkdat` |Bitcoin blk.dat |<sub>`bitcoin_block`</sub>|
|
||||
|[`bitcoin_block`](#bitcoin_block) |Bitcoin block |<sub>`bitcoin_transaction`</sub>|
|
||||
|`bitcoin_script` |Bitcoin script |<sub></sub>|
|
||||
|`bitcoin_transaction` |Bitcoin transaction |<sub>`bitcoin_script`</sub>|
|
||||
|[`bits`](#bits) |Raw bits |<sub></sub>|
|
||||
|[`bplist`](#bplist) |Apple Binary Property List |<sub></sub>|
|
||||
|`bsd_loopback_frame` |BSD loopback frame |<sub>`inet_packet`</sub>|
|
||||
|[`bson`](#bson) |Binary JSON |<sub></sub>|
|
||||
|[`bytes`](#bytes) |Raw bytes |<sub></sub>|
|
||||
|`bzip2` |bzip2 compression |<sub>`probe`</sub>|
|
||||
|[`caff`](#caff) |Live2D Cubism archive |<sub>`probe`</sub>|
|
||||
|[`cbor`](#cbor) |Concise Binary Object Representation |<sub></sub>|
|
||||
|[`csv`](#csv) |Comma separated values |<sub></sub>|
|
||||
|`dns` |DNS packet |<sub></sub>|
|
||||
|`dns_tcp` |DNS packet (TCP) |<sub></sub>|
|
||||
|`elf` |Executable and Linkable Format |<sub></sub>|
|
||||
|`ether8023_frame` |Ethernet 802.3 frame |<sub>`inet_packet`</sub>|
|
||||
|`exif` |Exchangeable Image File Format |<sub></sub>|
|
||||
|`fairplay_spc` |FairPlay Server Playback Context |<sub></sub>|
|
||||
|`flac` |Free Lossless Audio Codec file |<sub>`flac_metadatablocks` `flac_frame`</sub>|
|
||||
|[`flac_frame`](#flac_frame) |FLAC frame |<sub></sub>|
|
||||
|`flac_metadatablock` |FLAC metadatablock |<sub>`flac_streaminfo` `flac_picture` `vorbis_comment`</sub>|
|
||||
|`flac_metadatablocks` |FLAC metadatablocks |<sub>`flac_metadatablock`</sub>|
|
||||
|`flac_picture` |FLAC metadatablock picture |<sub>`image`</sub>|
|
||||
|`flac_streaminfo` |FLAC streaminfo |<sub></sub>|
|
||||
|`gif` |Graphics Interchange Format |<sub></sub>|
|
||||
|`gzip` |gzip compression |<sub>`probe`</sub>|
|
||||
|`hevc_annexb` |H.265/HEVC Annex B |<sub>`hevc_nalu`</sub>|
|
||||
|[`hevc_au`](#hevc_au) |H.265/HEVC Access Unit |<sub>`hevc_nalu`</sub>|
|
||||
|`hevc_dcr` |H.265/HEVC Decoder Configuration Record |<sub>`hevc_nalu`</sub>|
|
||||
|`hevc_nalu` |H.265/HEVC Network Access Layer Unit |<sub>`hevc_vps` `hevc_pps` `hevc_sps`</sub>|
|
||||
|`hevc_pps` |H.265/HEVC Picture Parameter Set |<sub></sub>|
|
||||
|`hevc_sps` |H.265/HEVC Sequence Parameter Set |<sub></sub>|
|
||||
|`hevc_vps` |H.265/HEVC Video Parameter Set |<sub></sub>|
|
||||
|[`html`](#html) |HyperText Markup Language |<sub></sub>|
|
||||
|`icc_profile` |International Color Consortium profile |<sub></sub>|
|
||||
|`icmp` |Internet Control Message Protocol |<sub></sub>|
|
||||
|`icmpv6` |Internet Control Message Protocol v6 |<sub></sub>|
|
||||
|`id3v1` |ID3v1 metadata |<sub></sub>|
|
||||
|`id3v11` |ID3v1.1 metadata |<sub></sub>|
|
||||
|`id3v2` |ID3v2 metadata |<sub>`image`</sub>|
|
||||
|`ipv4_packet` |Internet protocol v4 packet |<sub>`ip_packet`</sub>|
|
||||
|`ipv6_packet` |Internet protocol v6 packet |<sub>`ip_packet`</sub>|
|
||||
|`jpeg` |Joint Photographic Experts Group file |<sub>`exif` `icc_profile`</sub>|
|
||||
|`json` |JavaScript Object Notation |<sub></sub>|
|
||||
|`jsonl` |JavaScript Object Notation Lines |<sub></sub>|
|
||||
|[`luajit`](#luajit) |LuaJIT 2.0 bytecode |<sub></sub>|
|
||||
|[`macho`](#macho) |Mach-O macOS executable |<sub></sub>|
|
||||
|`macho_fat` |Fat Mach-O macOS executable (multi-architecture) |<sub>`macho`</sub>|
|
||||
|[`markdown`](#markdown) |Markdown |<sub></sub>|
|
||||
|[`matroska`](#matroska) |Matroska file |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `image` `mp3_frame` `mpeg_asc` `mpeg_pes_packet` `mpeg_spu` `opus_packet` `vorbis_packet` `vp8_frame` `vp9_cfm` `vp9_frame`</sub>|
|
||||
|[`moc3`](#moc3) |MOC3 file |<sub></sub>|
|
||||
|[`mp3`](#mp3) |MP3 file |<sub>`id3v2` `id3v1` `id3v11` `apev2` `mp3_frame`</sub>|
|
||||
|`mp3_frame` |MPEG audio layer 3 frame |<sub>`mp3_frame_tags`</sub>|
|
||||
|`mp3_frame_vbri` |MP3 frame Fraunhofer encoder variable bitrate tag |<sub></sub>|
|
||||
|`mp3_frame_xing` |MP3 frame Xing/Info tag |<sub></sub>|
|
||||
|[`mp4`](#mp4) |ISOBMFF, QuickTime and similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `icc_profile` `id3v2` `image` `jpeg` `mp3_frame` `mpeg_es` `mpeg_pes_packet` `opus_packet` `png` `prores_frame` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr`</sub>|
|
||||
|`mpeg_asc` |MPEG-4 Audio Specific Config |<sub></sub>|
|
||||
|`mpeg_es` |MPEG Elementary Stream |<sub>`mpeg_asc` `vorbis_packet`</sub>|
|
||||
|`mpeg_pes` |MPEG Packetized elementary stream |<sub>`mpeg_pes_packet` `mpeg_spu`</sub>|
|
||||
|`mpeg_pes_packet` |MPEG Packetized elementary stream packet |<sub></sub>|
|
||||
|`mpeg_spu` |Sub Picture Unit (DVD subtitle) |<sub></sub>|
|
||||
|`mpeg_ts` |MPEG Transport Stream |<sub></sub>|
|
||||
|[`msgpack`](#msgpack) |MessagePack |<sub></sub>|
|
||||
|`ogg` |OGG file |<sub>`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`</sub>|
|
||||
|`ogg_page` |OGG page |<sub></sub>|
|
||||
|[`opentimestamps`](#opentimestamps) |OpenTimestamps file |<sub></sub>|
|
||||
|`opus_packet` |Opus packet |<sub>`vorbis_comment`</sub>|
|
||||
|[`pcap`](#pcap) |PCAP packet capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|
||||
|`pcapng` |PCAPNG packet capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|
||||
|[`pg_btree`](#pg_btree) |PostgreSQL btree index file |<sub></sub>|
|
||||
|[`pg_control`](#pg_control) |PostgreSQL control file |<sub></sub>|
|
||||
|[`pg_heap`](#pg_heap) |PostgreSQL heap file |<sub></sub>|
|
||||
|`png` |Portable Network Graphics file |<sub>`icc_profile` `exif`</sub>|
|
||||
|`prores_frame` |Apple ProRes frame |<sub></sub>|
|
||||
|[`protobuf`](#protobuf) |Protobuf |<sub></sub>|
|
||||
|`protobuf_widevine` |Widevine protobuf |<sub>`protobuf`</sub>|
|
||||
|`pssh_playready` |PlayReady PSSH |<sub></sub>|
|
||||
|[`rtmp`](#rtmp) |Real-Time Messaging Protocol |<sub>`amf0` `mpeg_asc`</sub>|
|
||||
|`sll2_packet` |Linux cooked capture encapsulation v2 |<sub>`inet_packet`</sub>|
|
||||
|`sll_packet` |Linux cooked capture encapsulation |<sub>`inet_packet`</sub>|
|
||||
|`tar` |Tar archive |<sub>`probe`</sub>|
|
||||
|`tcp_segment` |Transmission control protocol segment |<sub></sub>|
|
||||
|`tiff` |Tag Image File Format |<sub>`icc_profile`</sub>|
|
||||
|[`tls`](#tls) |Transport layer security |<sub>`asn1_ber`</sub>|
|
||||
|`toml` |Tom's Obvious, Minimal Language |<sub></sub>|
|
||||
|[`tzif`](#tzif) |Time Zone Information Format |<sub></sub>|
|
||||
|`udp_datagram` |User datagram protocol |<sub>`udp_payload`</sub>|
|
||||
|`vorbis_comment` |Vorbis comment |<sub>`flac_picture`</sub>|
|
||||
|`vorbis_packet` |Vorbis packet |<sub>`vorbis_comment`</sub>|
|
||||
|`vp8_frame` |VP8 frame |<sub></sub>|
|
||||
|`vp9_cfm` |VP9 Codec Feature Metadata |<sub></sub>|
|
||||
|`vp9_frame` |VP9 frame |<sub></sub>|
|
||||
|`vpx_ccr` |VPX Codec Configuration Record |<sub></sub>|
|
||||
|[`wasm`](#wasm) |WebAssembly Binary Format |<sub></sub>|
|
||||
|`wav` |WAV file |<sub>`id3v2` `id3v1` `id3v11`</sub>|
|
||||
|`webp` |WebP image |<sub>`exif` `vp8_frame` `icc_profile` `xml`</sub>|
|
||||
|[`xml`](#xml) |Extensible Markup Language |<sub></sub>|
|
||||
|`yaml` |YAML Ain't Markup Language |<sub></sub>|
|
||||
|[`zip`](#zip) |ZIP archive |<sub>`probe`</sub>|
|
||||
|`image` |Group |<sub>`gif` `jpeg` `mp4` `png` `tiff` `webp`</sub>|
|
||||
|`inet_packet` |Group |<sub>`ipv4_packet` `ipv6_packet`</sub>|
|
||||
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|
||||
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `ipv4_packet` `ipv6_packet` `sll2_packet` `sll_packet`</sub>|
|
||||
|`mp3_frame_tags` |Group |<sub>`mp3_frame_vbri` `mp3_frame_xing`</sub>|
|
||||
|`probe` |Group |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `flac` `gif` `gzip` `html` `jpeg` `json` `jsonl` `luajit` `macho` `macho_fat` `matroska` `moc3` `mp3` `mp4` `mpeg_ts` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|
||||
|`tcp_stream` |Group |<sub>`dns_tcp` `rtmp` `tls`</sub>|
|
||||
|`udp_payload` |Group |<sub>`dns`</sub>|
|
||||
|Name |Description |Dependencies|
|
||||
|- |- |-|
|
||||
|[`aac_frame`](#aac_frame) |Advanced Audio Coding frame |<sub></sub>|
|
||||
|`adts` |Audio Data Transport Stream |<sub>`adts_frame`</sub>|
|
||||
|`adts_frame` |Audio Data Transport Stream frame |<sub>`aac_frame`</sub>|
|
||||
|`aiff` |Audio Interchange File Format |<sub></sub>|
|
||||
|`amf0` |Action Message Format 0 |<sub></sub>|
|
||||
|`apev2` |APEv2 metadata tag |<sub>`image`</sub>|
|
||||
|[`apple_bookmark`](#apple_bookmark) |Apple BookmarkData |<sub></sub>|
|
||||
|`ar` |Unix archive |<sub>`probe`</sub>|
|
||||
|[`asn1_ber`](#asn1_ber) |ASN1 BER (basic encoding rules, also CER and DER) |<sub></sub>|
|
||||
|`av1_ccr` |AV1 Codec Configuration Record |<sub></sub>|
|
||||
|`av1_frame` |AV1 frame |<sub>`av1_obu`</sub>|
|
||||
|`av1_obu` |AV1 Open Bitstream Unit |<sub></sub>|
|
||||
|`avc_annexb` |H.264/AVC Annex B |<sub>`avc_nalu`</sub>|
|
||||
|[`avc_au`](#avc_au) |H.264/AVC Access Unit |<sub>`avc_nalu`</sub>|
|
||||
|`avc_dcr` |H.264/AVC Decoder Configuration Record |<sub>`avc_nalu`</sub>|
|
||||
|`avc_nalu` |H.264/AVC Network Access Layer Unit |<sub>`avc_sps` `avc_pps` `avc_sei`</sub>|
|
||||
|`avc_pps` |H.264/AVC Picture Parameter Set |<sub></sub>|
|
||||
|`avc_sei` |H.264/AVC Supplemental Enhancement Information |<sub></sub>|
|
||||
|`avc_sps` |H.264/AVC Sequence Parameter Set |<sub></sub>|
|
||||
|[`avi`](#avi) |Audio Video Interleaved |<sub>`avc_au` `hevc_au` `mp3_frame` `flac_frame`</sub>|
|
||||
|[`avro_ocf`](#avro_ocf) |Avro object container file |<sub></sub>|
|
||||
|[`bencode`](#bencode) |BitTorrent bencoding |<sub></sub>|
|
||||
|`bitcoin_blkdat` |Bitcoin blk.dat |<sub>`bitcoin_block`</sub>|
|
||||
|[`bitcoin_block`](#bitcoin_block) |Bitcoin block |<sub>`bitcoin_transaction`</sub>|
|
||||
|`bitcoin_script` |Bitcoin script |<sub></sub>|
|
||||
|`bitcoin_transaction` |Bitcoin transaction |<sub>`bitcoin_script`</sub>|
|
||||
|[`bits`](#bits) |Raw bits |<sub></sub>|
|
||||
|[`bplist`](#bplist) |Apple Binary Property List |<sub></sub>|
|
||||
|`bsd_loopback_frame` |BSD loopback frame |<sub>`inet_packet`</sub>|
|
||||
|[`bson`](#bson) |Binary JSON |<sub></sub>|
|
||||
|[`bytes`](#bytes) |Raw bytes |<sub></sub>|
|
||||
|`bzip2` |bzip2 compression |<sub>`probe`</sub>|
|
||||
|[`caff`](#caff) |Live2D Cubism archive |<sub>`probe`</sub>|
|
||||
|[`cbor`](#cbor) |Concise Binary Object Representation |<sub></sub>|
|
||||
|[`csv`](#csv) |Comma separated values |<sub></sub>|
|
||||
|`dns` |DNS packet |<sub></sub>|
|
||||
|`dns_tcp` |DNS packet (TCP) |<sub></sub>|
|
||||
|`elf` |Executable and Linkable Format |<sub></sub>|
|
||||
|`ether8023_frame` |Ethernet 802.3 frame |<sub>`inet_packet`</sub>|
|
||||
|`exif` |Exchangeable Image File Format |<sub></sub>|
|
||||
|`fairplay_spc` |FairPlay Server Playback Context |<sub></sub>|
|
||||
|`flac` |Free Lossless Audio Codec file |<sub>`flac_metadatablocks` `flac_frame`</sub>|
|
||||
|[`flac_frame`](#flac_frame) |FLAC frame |<sub></sub>|
|
||||
|`flac_metadatablock` |FLAC metadatablock |<sub>`flac_streaminfo` `flac_picture` `vorbis_comment`</sub>|
|
||||
|`flac_metadatablocks` |FLAC metadatablocks |<sub>`flac_metadatablock`</sub>|
|
||||
|`flac_picture` |FLAC metadatablock picture |<sub>`image`</sub>|
|
||||
|`flac_streaminfo` |FLAC streaminfo |<sub></sub>|
|
||||
|`gif` |Graphics Interchange Format |<sub></sub>|
|
||||
|`gzip` |gzip compression |<sub>`probe`</sub>|
|
||||
|`hevc_annexb` |H.265/HEVC Annex B |<sub>`hevc_nalu`</sub>|
|
||||
|[`hevc_au`](#hevc_au) |H.265/HEVC Access Unit |<sub>`hevc_nalu`</sub>|
|
||||
|`hevc_dcr` |H.265/HEVC Decoder Configuration Record |<sub>`hevc_nalu`</sub>|
|
||||
|`hevc_nalu` |H.265/HEVC Network Access Layer Unit |<sub>`hevc_vps` `hevc_pps` `hevc_sps`</sub>|
|
||||
|`hevc_pps` |H.265/HEVC Picture Parameter Set |<sub></sub>|
|
||||
|`hevc_sps` |H.265/HEVC Sequence Parameter Set |<sub></sub>|
|
||||
|`hevc_vps` |H.265/HEVC Video Parameter Set |<sub></sub>|
|
||||
|[`html`](#html) |HyperText Markup Language |<sub></sub>|
|
||||
|`icc_profile` |International Color Consortium profile |<sub></sub>|
|
||||
|`icmp` |Internet Control Message Protocol |<sub></sub>|
|
||||
|`icmpv6` |Internet Control Message Protocol v6 |<sub></sub>|
|
||||
|`id3v1` |ID3v1 metadata |<sub></sub>|
|
||||
|`id3v11` |ID3v1.1 metadata |<sub></sub>|
|
||||
|`id3v2` |ID3v2 metadata |<sub>`image`</sub>|
|
||||
|`ipv4_packet` |Internet protocol v4 packet |<sub>`ip_packet`</sub>|
|
||||
|`ipv6_packet` |Internet protocol v6 packet |<sub>`ip_packet`</sub>|
|
||||
|`jpeg` |Joint Photographic Experts Group file |<sub>`exif` `icc_profile`</sub>|
|
||||
|`json` |JavaScript Object Notation |<sub></sub>|
|
||||
|`jsonl` |JavaScript Object Notation Lines |<sub></sub>|
|
||||
|[`leveldb_descriptor`](#leveldb_descriptor) |LevelDB Descriptor |<sub></sub>|
|
||||
|[`leveldb_log`](#leveldb_log) |LevelDB Log |<sub></sub>|
|
||||
|[`leveldb_table`](#leveldb_table) |LevelDB Table |<sub></sub>|
|
||||
|[`luajit`](#luajit) |LuaJIT 2.0 bytecode |<sub></sub>|
|
||||
|[`macho`](#macho) |Mach-O macOS executable |<sub></sub>|
|
||||
|`macho_fat` |Fat Mach-O macOS executable (multi-architecture) |<sub>`macho`</sub>|
|
||||
|[`markdown`](#markdown) |Markdown |<sub></sub>|
|
||||
|[`matroska`](#matroska) |Matroska file |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `image` `mp3_frame` `mpeg_asc` `mpeg_pes_packet` `mpeg_spu` `opus_packet` `vorbis_packet` `vp8_frame` `vp9_cfm` `vp9_frame`</sub>|
|
||||
|[`moc3`](#moc3) |MOC3 file |<sub></sub>|
|
||||
|[`mp3`](#mp3) |MP3 file |<sub>`id3v2` `id3v1` `id3v11` `apev2` `mp3_frame`</sub>|
|
||||
|`mp3_frame` |MPEG audio layer 3 frame |<sub>`mp3_frame_tags`</sub>|
|
||||
|`mp3_frame_vbri` |MP3 frame Fraunhofer encoder variable bitrate tag |<sub></sub>|
|
||||
|`mp3_frame_xing` |MP3 frame Xing/Info tag |<sub></sub>|
|
||||
|[`mp4`](#mp4) |ISOBMFF, QuickTime and similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `icc_profile` `id3v2` `image` `jpeg` `mp3_frame` `mpeg_es` `mpeg_pes_packet` `opus_packet` `png` `prores_frame` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr`</sub>|
|
||||
|`mpeg_asc` |MPEG-4 Audio Specific Config |<sub></sub>|
|
||||
|`mpeg_es` |MPEG Elementary Stream |<sub>`mpeg_asc` `vorbis_packet`</sub>|
|
||||
|`mpeg_pes` |MPEG Packetized elementary stream |<sub>`mpeg_pes_packet` `mpeg_spu`</sub>|
|
||||
|`mpeg_pes_packet` |MPEG Packetized elementary stream packet |<sub></sub>|
|
||||
|`mpeg_spu` |Sub Picture Unit (DVD subtitle) |<sub></sub>|
|
||||
|`mpeg_ts` |MPEG Transport Stream |<sub></sub>|
|
||||
|[`msgpack`](#msgpack) |MessagePack |<sub></sub>|
|
||||
|`ogg` |OGG file |<sub>`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`</sub>|
|
||||
|`ogg_page` |OGG page |<sub></sub>|
|
||||
|[`opentimestamps`](#opentimestamps) |OpenTimestamps file |<sub></sub>|
|
||||
|`opus_packet` |Opus packet |<sub>`vorbis_comment`</sub>|
|
||||
|[`pcap`](#pcap) |PCAP packet capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|
||||
|`pcapng` |PCAPNG packet capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|
||||
|[`pg_btree`](#pg_btree) |PostgreSQL btree index file |<sub></sub>|
|
||||
|[`pg_control`](#pg_control) |PostgreSQL control file |<sub></sub>|
|
||||
|[`pg_heap`](#pg_heap) |PostgreSQL heap file |<sub></sub>|
|
||||
|`png` |Portable Network Graphics file |<sub>`icc_profile` `exif`</sub>|
|
||||
|`prores_frame` |Apple ProRes frame |<sub></sub>|
|
||||
|[`protobuf`](#protobuf) |Protobuf |<sub></sub>|
|
||||
|`protobuf_widevine` |Widevine protobuf |<sub>`protobuf`</sub>|
|
||||
|`pssh_playready` |PlayReady PSSH |<sub></sub>|
|
||||
|[`rtmp`](#rtmp) |Real-Time Messaging Protocol |<sub>`amf0` `mpeg_asc`</sub>|
|
||||
|`sll2_packet` |Linux cooked capture encapsulation v2 |<sub>`inet_packet`</sub>|
|
||||
|`sll_packet` |Linux cooked capture encapsulation |<sub>`inet_packet`</sub>|
|
||||
|`tar` |Tar archive |<sub>`probe`</sub>|
|
||||
|`tcp_segment` |Transmission control protocol segment |<sub></sub>|
|
||||
|`tiff` |Tag Image File Format |<sub>`icc_profile`</sub>|
|
||||
|[`tls`](#tls) |Transport layer security |<sub>`asn1_ber`</sub>|
|
||||
|`toml` |Tom's Obvious, Minimal Language |<sub></sub>|
|
||||
|[`tzif`](#tzif) |Time Zone Information Format |<sub></sub>|
|
||||
|`udp_datagram` |User datagram protocol |<sub>`udp_payload`</sub>|
|
||||
|`vorbis_comment` |Vorbis comment |<sub>`flac_picture`</sub>|
|
||||
|`vorbis_packet` |Vorbis packet |<sub>`vorbis_comment`</sub>|
|
||||
|`vp8_frame` |VP8 frame |<sub></sub>|
|
||||
|`vp9_cfm` |VP9 Codec Feature Metadata |<sub></sub>|
|
||||
|`vp9_frame` |VP9 frame |<sub></sub>|
|
||||
|`vpx_ccr` |VPX Codec Configuration Record |<sub></sub>|
|
||||
|[`wasm`](#wasm) |WebAssembly Binary Format |<sub></sub>|
|
||||
|`wav` |WAV file |<sub>`id3v2` `id3v1` `id3v11`</sub>|
|
||||
|`webp` |WebP image |<sub>`exif` `vp8_frame` `icc_profile` `xml`</sub>|
|
||||
|[`xml`](#xml) |Extensible Markup Language |<sub></sub>|
|
||||
|`yaml` |YAML Ain't Markup Language |<sub></sub>|
|
||||
|[`zip`](#zip) |ZIP archive |<sub>`probe`</sub>|
|
||||
|`image` |Group |<sub>`gif` `jpeg` `mp4` `png` `tiff` `webp`</sub>|
|
||||
|`inet_packet` |Group |<sub>`ipv4_packet` `ipv6_packet`</sub>|
|
||||
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|
||||
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `ipv4_packet` `ipv6_packet` `sll2_packet` `sll_packet`</sub>|
|
||||
|`mp3_frame_tags` |Group |<sub>`mp3_frame_vbri` `mp3_frame_xing`</sub>|
|
||||
|`probe` |Group |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `flac` `gif` `gzip` `html` `jpeg` `json` `jsonl` `leveldb_table` `luajit` `macho` `macho_fat` `matroska` `moc3` `mp3` `mp4` `mpeg_ts` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|
||||
|`tcp_stream` |Group |<sub>`dns_tcp` `rtmp` `tls`</sub>|
|
||||
|`udp_payload` |Group |<sub>`dns`</sub>|
|
||||
|
||||
[#]: sh-end
|
||||
|
||||
@ -690,6 +693,55 @@ $ fq -n -d html '[inputs | {key: input_filename, value: .html.head.title?}] | fr
|
||||
$ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href' file.html
|
||||
```
|
||||
|
||||
## leveldb_descriptor
|
||||
|
||||
### Limitations
|
||||
|
||||
- fragmented non-"full" records are not merged and decoded further.
|
||||
|
||||
### Authors
|
||||
|
||||
- [@mikez](https://github.com/mikez), original author
|
||||
|
||||
### References
|
||||
|
||||
- https://github.com/google/leveldb/blob/main/doc/impl.md#manifest
|
||||
- https://github.com/google/leveldb/blob/main/doc/log_format.md
|
||||
- https://github.com/google/leveldb/blob/main/db/version_edit.cc
|
||||
|
||||
## leveldb_log
|
||||
|
||||
### Limitations
|
||||
|
||||
- fragmented non-"full" records are not merged and decoded further.
|
||||
|
||||
### Authors
|
||||
|
||||
- [@mikez](https://github.com/mikez), original author
|
||||
|
||||
### References
|
||||
|
||||
- https://github.com/google/leveldb/blob/main/doc/impl.md#log-files
|
||||
- https://github.com/google/leveldb/blob/main/doc/log_format.md
|
||||
- https://github.com/google/leveldb/blob/main/db/write_batch.cc
|
||||
|
||||
## leveldb_table
|
||||
|
||||
### Limitations
|
||||
|
||||
- no Meta Blocks (like "filter") are decoded yet.
|
||||
- Zstandard uncompression is not implemented yet.
|
||||
|
||||
### Authors
|
||||
|
||||
- [@mikez](https://github.com/mikez), original author
|
||||
|
||||
### References
|
||||
|
||||
- https://github.com/google/leveldb/blob/main/doc/table_format.md
|
||||
- https://github.com/google/leveldb/blob/main/doc/impl.md
|
||||
- https://github.com/google/leveldb/blob/main/doc/index.md
|
||||
|
||||
## luajit
|
||||
|
||||
### Authors
|
||||
|
@ -14,6 +14,7 @@ $ fq -n _registry.groups.probe
|
||||
"gif",
|
||||
"gzip",
|
||||
"jpeg",
|
||||
"leveldb_table",
|
||||
"luajit",
|
||||
"macho",
|
||||
"macho_fat",
|
||||
@ -111,6 +112,9 @@ ipv6_packet Internet protocol v6 packet
|
||||
jpeg Joint Photographic Experts Group file
|
||||
json JavaScript Object Notation
|
||||
jsonl JavaScript Object Notation Lines
|
||||
leveldb_descriptor LevelDB Descriptor
|
||||
leveldb_log LevelDB Log
|
||||
leveldb_table LevelDB Table
|
||||
luajit LuaJIT 2.0 bytecode
|
||||
macho Mach-O macOS executable
|
||||
macho_fat Fat Mach-O macOS executable (multi-architecture)
|
||||
|
@ -30,6 +30,7 @@ import (
|
||||
_ "github.com/wader/fq/format/inet"
|
||||
_ "github.com/wader/fq/format/jpeg"
|
||||
_ "github.com/wader/fq/format/json"
|
||||
_ "github.com/wader/fq/format/leveldb"
|
||||
_ "github.com/wader/fq/format/luajit"
|
||||
_ "github.com/wader/fq/format/markdown"
|
||||
_ "github.com/wader/fq/format/math"
|
||||
|
@ -125,6 +125,9 @@ var (
|
||||
JPEG = &decode.Group{Name: "jpeg"}
|
||||
JSON = &decode.Group{Name: "json"}
|
||||
JSONL = &decode.Group{Name: "jsonl"}
|
||||
LevelDB_Descriptor = &decode.Group{Name: "leveldb_descriptor"}
|
||||
LevelDB_LDB = &decode.Group{Name: "leveldb_table"}
|
||||
LevelDB_LOG = &decode.Group{Name: "leveldb_log"}
|
||||
LuaJIT = &decode.Group{Name: "luajit"}
|
||||
MachO = &decode.Group{Name: "macho"}
|
||||
MachO_Fat = &decode.Group{Name: "macho_fat"}
|
||||
|
121
format/leveldb/leveldb_descriptor.go
Normal file
121
format/leveldb/leveldb_descriptor.go
Normal file
@ -0,0 +1,121 @@
|
||||
package leveldb
|
||||
|
||||
// https://github.com/google/leveldb/blob/main/doc/impl.md#manifest
|
||||
// https://github.com/google/leveldb/blob/main/db/version_edit.cc
|
||||
//
|
||||
// Files in LevelDB using this format include:
|
||||
// - MANIFEST-*
|
||||
|
||||
import (
|
||||
"embed"
|
||||
|
||||
"github.com/wader/fq/format"
|
||||
"github.com/wader/fq/pkg/decode"
|
||||
"github.com/wader/fq/pkg/interp"
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
|
||||
//go:embed leveldb_descriptor.jq
|
||||
//go:embed leveldb_descriptor.md
|
||||
var leveldbDescriptorFS embed.FS
|
||||
|
||||
func init() {
|
||||
interp.RegisterFormat(
|
||||
format.LevelDB_Descriptor,
|
||||
&decode.Format{
|
||||
Description: "LevelDB Descriptor",
|
||||
DecodeFn: ldbDescriptorDecode,
|
||||
Functions: []string{"torepr"},
|
||||
})
|
||||
interp.RegisterFS(leveldbDescriptorFS)
|
||||
}
|
||||
|
||||
const (
|
||||
tagTypeComparator = 1
|
||||
tagTypeLogNumber = 2
|
||||
tagTypeNextFileNumber = 3
|
||||
tagTypeLastSequence = 4
|
||||
tagTypeCompactPointer = 5
|
||||
tagTypeDeletedFile = 6
|
||||
tagTypeNewFile = 7
|
||||
// 8 not used anymore
|
||||
tagTypePrevLogNumber = 9
|
||||
)
|
||||
|
||||
var tagTypes = scalar.UintMapSymStr{
|
||||
tagTypeComparator: "comparator",
|
||||
tagTypeLogNumber: "log_number",
|
||||
tagTypeNextFileNumber: "next_file_number",
|
||||
tagTypeLastSequence: "last_sequence",
|
||||
tagTypeCompactPointer: "compact_pointer",
|
||||
tagTypeDeletedFile: "deleted_file",
|
||||
tagTypeNewFile: "new_file",
|
||||
tagTypePrevLogNumber: "previous_log_number",
|
||||
}
|
||||
|
||||
func ldbDescriptorDecode(d *decode.D) any {
|
||||
rro := recordReadOptions{readDataFn: func(size int64, recordType int, d *decode.D) {
|
||||
if recordType == recordTypeFull {
|
||||
d.FieldStruct("data", func(d *decode.D) {
|
||||
d.LimitedFn(size, readManifest)
|
||||
})
|
||||
} else {
|
||||
d.FieldRawLen("data", size)
|
||||
}
|
||||
}}
|
||||
readBlockSequence(rro, d)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// List of sorted tables for each level involving key ranges and other metadata.
|
||||
func readManifest(d *decode.D) {
|
||||
d.FieldArray("tags", func(d *decode.D) {
|
||||
for !d.End() {
|
||||
d.FieldStruct("tag", func(d *decode.D) {
|
||||
tag := d.FieldULEB128("key", tagTypes)
|
||||
switch tag {
|
||||
case tagTypeComparator:
|
||||
readLengthPrefixedString("value", d)
|
||||
case tagTypeLogNumber,
|
||||
tagTypePrevLogNumber,
|
||||
tagTypeNextFileNumber,
|
||||
tagTypeLastSequence:
|
||||
d.FieldULEB128("value")
|
||||
case tagTypeCompactPointer:
|
||||
d.FieldStruct("value", func(d *decode.D) {
|
||||
d.FieldULEB128("level")
|
||||
readTagInternalKey("internal_key", d)
|
||||
})
|
||||
case tagTypeDeletedFile:
|
||||
d.FieldStruct("value", func(d *decode.D) {
|
||||
d.FieldULEB128("level")
|
||||
d.FieldULEB128("file_number")
|
||||
})
|
||||
case tagTypeNewFile:
|
||||
d.FieldStruct("value", func(d *decode.D) {
|
||||
d.FieldULEB128("level")
|
||||
d.FieldULEB128("file_number")
|
||||
d.FieldULEB128("file_size")
|
||||
readTagInternalKey("smallest_internal_key", d)
|
||||
readTagInternalKey("largest_internal_key", d)
|
||||
})
|
||||
default:
|
||||
d.Fatalf("unknown tag: %d", tag)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func readTagInternalKey(name string, d *decode.D) {
|
||||
d.FieldStruct(name, func(d *decode.D) {
|
||||
length := d.FieldULEB128("length")
|
||||
d.FieldStruct("data", func(d *decode.D) {
|
||||
err := readInternalKey(nil, int(length), d)
|
||||
if err != nil {
|
||||
d.Errorf("%v", err)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
37
format/leveldb/leveldb_descriptor.jq
Normal file
37
format/leveldb/leveldb_descriptor.jq
Normal file
@ -0,0 +1,37 @@
|
||||
# TK(2023-12-07): in the output, object-properties are
|
||||
# sorted alphabetically... how can one prevent this
|
||||
# and keep the original order?
|
||||
def _leveldb_descriptor_torepr:
|
||||
def _f:
|
||||
if .type == "root" then
|
||||
[ .value.blocks[].records[]
|
||||
| {type: "record", value: .}
|
||||
| _f
|
||||
]
|
||||
elif .type == "record" then
|
||||
if .value.header.record_type != "full" then
|
||||
empty
|
||||
else
|
||||
[ .value.data.tags[]
|
||||
| {type: "tag", value: {(.key): .value}}
|
||||
| _f
|
||||
]
|
||||
end
|
||||
elif .type == "tag" then
|
||||
( .value
|
||||
| if .comparator then .comparator |= .data else . end
|
||||
| if .new_file then
|
||||
( .new_file.largest_internal_key |= .data
|
||||
| .new_file.smallest_internal_key |= .data
|
||||
)
|
||||
else .
|
||||
end
|
||||
| if .compact_pointer then
|
||||
.compact_pointer.internal_key |= .data
|
||||
else .
|
||||
end
|
||||
)
|
||||
end;
|
||||
( {type: "root", value: .}
|
||||
| _f
|
||||
);
|
13
format/leveldb/leveldb_descriptor.md
Normal file
13
format/leveldb/leveldb_descriptor.md
Normal file
@ -0,0 +1,13 @@
|
||||
### Limitations
|
||||
|
||||
- fragmented non-"full" records are not merged and decoded further.
|
||||
|
||||
### Authors
|
||||
|
||||
- [@mikez](https://github.com/mikez), original author
|
||||
|
||||
### References
|
||||
|
||||
- https://github.com/google/leveldb/blob/main/doc/impl.md#manifest
|
||||
- https://github.com/google/leveldb/blob/main/doc/log_format.md
|
||||
- https://github.com/google/leveldb/blob/main/db/version_edit.cc
|
86
format/leveldb/leveldb_log.go
Normal file
86
format/leveldb/leveldb_log.go
Normal file
@ -0,0 +1,86 @@
|
||||
package leveldb
|
||||
|
||||
// https://github.com/google/leveldb/blob/main/doc/impl.md#log-files
|
||||
// https://github.com/google/leveldb/blob/main/db/write_batch.cc
|
||||
//
|
||||
// Files in LevelDB using this format include:
|
||||
// - *.log
|
||||
|
||||
import (
|
||||
"embed"
|
||||
|
||||
"github.com/wader/fq/format"
|
||||
"github.com/wader/fq/pkg/decode"
|
||||
"github.com/wader/fq/pkg/interp"
|
||||
)
|
||||
|
||||
//go:embed leveldb_log.md
|
||||
var leveldbLogFS embed.FS
|
||||
|
||||
func init() {
|
||||
interp.RegisterFormat(
|
||||
format.LevelDB_LOG,
|
||||
&decode.Format{
|
||||
Description: "LevelDB Log",
|
||||
DecodeFn: ldbLogDecode,
|
||||
})
|
||||
interp.RegisterFS(leveldbLogFS)
|
||||
}
|
||||
|
||||
func ldbLogDecode(d *decode.D) any {
|
||||
rro := recordReadOptions{readDataFn: func(size int64, recordType int, d *decode.D) {
|
||||
if recordType == recordTypeFull {
|
||||
d.FieldStruct("data", func(d *decode.D) {
|
||||
d.LimitedFn(size, readBatch)
|
||||
})
|
||||
} else {
|
||||
d.FieldRawLen("data", size)
|
||||
}
|
||||
}}
|
||||
readBlockSequence(rro, d)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// https://github.com/google/leveldb/blob/main/db/write_batch.cc#L5-L14
|
||||
//
|
||||
// WriteBatch::rep_ :=
|
||||
//
|
||||
// sequence: fixed64
|
||||
// count: fixed32
|
||||
// data: record[count]
|
||||
//
|
||||
// record :=
|
||||
//
|
||||
// kTypeValue varstring varstring
|
||||
// kTypeDeletion varstring
|
||||
//
|
||||
// varstring :=
|
||||
//
|
||||
// len: varint32
|
||||
// data: uint8[len]
|
||||
func readBatch(d *decode.D) {
|
||||
d.FieldU64("sequence")
|
||||
expectedCount := d.FieldU32("count")
|
||||
actualCount := uint64(0)
|
||||
d.FieldArray("records", func(d *decode.D) {
|
||||
for !d.End() {
|
||||
d.FieldStruct("record", func(d *decode.D) {
|
||||
valueType := d.FieldULEB128("type", valueTypes)
|
||||
switch valueType {
|
||||
case valueTypeDeletion:
|
||||
readLengthPrefixedString("key", d)
|
||||
case valueTypeValue:
|
||||
readLengthPrefixedString("key", d)
|
||||
readLengthPrefixedString("value", d)
|
||||
default:
|
||||
d.Fatalf("unknown value type: %d", valueType)
|
||||
}
|
||||
})
|
||||
actualCount++
|
||||
}
|
||||
})
|
||||
if actualCount != expectedCount {
|
||||
d.Errorf("actual record count (%d) does not equal expected count (%d)", actualCount, expectedCount)
|
||||
}
|
||||
}
|
13
format/leveldb/leveldb_log.md
Normal file
13
format/leveldb/leveldb_log.md
Normal file
@ -0,0 +1,13 @@
|
||||
### Limitations
|
||||
|
||||
- fragmented non-"full" records are not merged and decoded further.
|
||||
|
||||
### Authors
|
||||
|
||||
- [@mikez](https://github.com/mikez), original author
|
||||
|
||||
### References
|
||||
|
||||
- https://github.com/google/leveldb/blob/main/doc/impl.md#log-files
|
||||
- https://github.com/google/leveldb/blob/main/doc/log_format.md
|
||||
- https://github.com/google/leveldb/blob/main/db/write_batch.cc
|
133
format/leveldb/leveldb_log_blocks.go
Normal file
133
format/leveldb/leveldb_log_blocks.go
Normal file
@ -0,0 +1,133 @@
|
||||
package leveldb
|
||||
|
||||
// https://github.com/google/leveldb/blob/main/doc/log_format.md
|
||||
//
|
||||
// Files in LevelDB using the "log-format" of block sequences include:
|
||||
// - *.log
|
||||
// - MANIFEST-*
|
||||
|
||||
import (
|
||||
"github.com/wader/fq/internal/mathex"
|
||||
"github.com/wader/fq/pkg/decode"
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
|
||||
type recordReadOptions struct {
|
||||
// Both .log- and MANIFEST-files use the Log-format,
|
||||
// i.e., a sequence of records split into 32KB blocks.
|
||||
// However, the format of the data within the records differ.
|
||||
// This function specifies how to read said data.
|
||||
readDataFn func(size int64, recordType int, d *decode.D)
|
||||
}
|
||||
|
||||
// https://github.com/google/leveldb/blob/main/db/log_format.h
|
||||
const (
|
||||
// checksum (4 bytes) + length (2 bytes) + record type (1 byte)
|
||||
headerSize = (4 + 2 + 1) * 8
|
||||
|
||||
blockSize = (32 * 1024) * 8 // 32KB
|
||||
|
||||
recordTypeZero = 0 // preallocated file regions
|
||||
recordTypeFull = 1
|
||||
recordTypeFirst = 2 // fragments
|
||||
recordTypeMiddle = 3
|
||||
recordTypeLast = 4
|
||||
)
|
||||
|
||||
var recordTypes = scalar.UintMapSymStr{
|
||||
recordTypeZero: "zero",
|
||||
recordTypeFull: "full",
|
||||
recordTypeFirst: "first",
|
||||
recordTypeMiddle: "middle",
|
||||
recordTypeLast: "last",
|
||||
}
|
||||
|
||||
// Read a sequence of 32KB-blocks (the last one may be less).
|
||||
// https://github.com/google/leveldb/blob/main/db/log_reader.cc#L189
|
||||
func readBlockSequence(rro recordReadOptions, d *decode.D) {
|
||||
d.Endian = decode.LittleEndian
|
||||
|
||||
d.FieldArray("blocks", func(d *decode.D) {
|
||||
for d.BitsLeft() >= headerSize {
|
||||
d.LimitedFn(mathex.Min(blockSize, d.BitsLeft()), func(d *decode.D) {
|
||||
d.FieldStruct("block", bind(readLogBlock, rro))
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
if d.BitsLeft() > 0 {
|
||||
// The reference implementation says:
|
||||
// "[...] if buffer_ is non-empty, we have a truncated header at the
|
||||
// end of the file, which can be caused by the writer crashing in the
|
||||
// middle of writing the header. Instead of considering this an error,
|
||||
// just report EOF."
|
||||
d.FieldRawLen("truncated_block", d.BitsLeft())
|
||||
}
|
||||
}
|
||||
|
||||
// Read a Log-block, consisting of up to 32KB of records and an optional trailer.
|
||||
//
|
||||
// block := record* trailer?
|
||||
func readLogBlock(rro recordReadOptions, d *decode.D) {
|
||||
if d.BitsLeft() > blockSize {
|
||||
d.Fatalf("Bits left greater than maximum log-block size of 32KB.")
|
||||
}
|
||||
// record*
|
||||
d.FieldArray("records", func(d *decode.D) {
|
||||
for d.BitsLeft() >= headerSize {
|
||||
d.FieldStruct("record", bind(readLogRecord, rro))
|
||||
}
|
||||
})
|
||||
// trailer?
|
||||
if d.BitsLeft() > 0 {
|
||||
d.FieldRawLen("trailer", d.BitsLeft())
|
||||
}
|
||||
}
|
||||
|
||||
// Read a Log-record.
|
||||
//
|
||||
// checksum: uint32 // crc32c of type and data[] ; little-endian
|
||||
// length: uint16 // little-endian
|
||||
// type: uint8 // One of FULL, FIRST, MIDDLE, LAST
|
||||
// data: uint8[length]
|
||||
//
|
||||
// via https://github.com/google/leveldb/blob/main/doc/log_format.md
|
||||
func readLogRecord(rro recordReadOptions, d *decode.D) {
|
||||
// header
|
||||
var checksumValue *decode.Value
|
||||
var length int64
|
||||
var recordType int
|
||||
d.LimitedFn(headerSize, func(d *decode.D) {
|
||||
d.FieldStruct("header", func(d *decode.D) {
|
||||
d.FieldU32("checksum", scalar.UintHex)
|
||||
checksumValue = d.FieldGet("checksum")
|
||||
length = int64(d.FieldU16("length"))
|
||||
recordType = int(d.FieldU8("record_type", recordTypes))
|
||||
})
|
||||
})
|
||||
|
||||
// verify checksum: record type (1 byte) + data (`length` bytes)
|
||||
d.RangeFn(d.Pos()-8, (1+length)*8, func(d *decode.D) {
|
||||
bytesToCheck := d.Bits(int(d.BitsLeft()))
|
||||
actualChecksum := computeChecksum(bytesToCheck)
|
||||
_ = checksumValue.TryUintScalarFn(d.UintAssert(uint64(actualChecksum)))
|
||||
})
|
||||
|
||||
// data
|
||||
dataSize := length * 8
|
||||
rro.readDataFn(dataSize, recordType, d)
|
||||
}
|
||||
|
||||
func readLengthPrefixedString(name string, d *decode.D) {
|
||||
d.FieldStruct(name, func(d *decode.D) {
|
||||
length := d.FieldULEB128("length")
|
||||
d.FieldUTF8("data", int(length))
|
||||
})
|
||||
}
|
||||
|
||||
// simplified `functools.partial` (Python) or `Function.prototype.bind` (JavaScript)
|
||||
func bind(f func(recordReadOptions, *decode.D), rro recordReadOptions) func(*decode.D) {
|
||||
return func(d *decode.D) {
|
||||
f(rro, d)
|
||||
}
|
||||
}
|
473
format/leveldb/leveldb_table.go
Normal file
473
format/leveldb/leveldb_table.go
Normal file
@ -0,0 +1,473 @@
|
||||
package leveldb
|
||||
|
||||
// https://github.com/google/leveldb/blob/main/doc/table_format.md
|
||||
// https://github.com/google/leveldb/blob/main/doc/impl.md
|
||||
// https://github.com/google/leveldb/blob/main/doc/index.md
|
||||
//
|
||||
// Files in LevelDB using this format include:
|
||||
// - *.ldb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"embed"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
|
||||
"github.com/golang/snappy"
|
||||
"github.com/wader/fq/format"
|
||||
"github.com/wader/fq/pkg/bitio"
|
||||
"github.com/wader/fq/pkg/decode"
|
||||
"github.com/wader/fq/pkg/interp"
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
|
||||
//go:embed leveldb_table.md
|
||||
var leveldbTableFS embed.FS
|
||||
|
||||
func init() {
|
||||
interp.RegisterFormat(
|
||||
format.LevelDB_LDB,
|
||||
&decode.Format{
|
||||
Description: "LevelDB Table",
|
||||
Groups: []*decode.Group{format.Probe},
|
||||
DecodeFn: ldbTableDecode,
|
||||
})
|
||||
interp.RegisterFS(leveldbTableFS)
|
||||
}
|
||||
|
||||
const (
|
||||
// four varints (each max 10 bytes) + magic number (8 bytes)
|
||||
// https://github.com/google/leveldb/blob/main/table/format.h#L53
|
||||
footerEncodedLength = (4*10 + 8) * 8
|
||||
magicNumberLength = 8 * 8
|
||||
// leading 64 bits of
|
||||
// echo http://code.google.com/p/leveldb/ | sha1sum
|
||||
// https://github.com/google/leveldb/blob/main/table/format.h#L76
|
||||
tableMagicNumber = 0xdb4775248b80fb57
|
||||
uint32BitSize = 32
|
||||
)
|
||||
|
||||
// https://github.com/google/leveldb/blob/main/include/leveldb/options.h#L25
|
||||
const (
|
||||
compressionTypeNone = 0x0
|
||||
compressionTypeSnappy = 0x1
|
||||
compressionTypeZstandard = 0x2
|
||||
)
|
||||
|
||||
var compressionTypes = scalar.UintMapSymStr{
|
||||
compressionTypeNone: "none",
|
||||
compressionTypeSnappy: "snappy",
|
||||
compressionTypeZstandard: "zstd",
|
||||
}
|
||||
|
||||
// https://github.com/google/leveldb/blob/main/db/dbformat.h#L54
|
||||
const (
|
||||
valueTypeDeletion = 0x0
|
||||
valueTypeValue = 0x1
|
||||
)
|
||||
|
||||
var valueTypes = scalar.UintMapSymStr{
|
||||
valueTypeDeletion: "deletion",
|
||||
valueTypeValue: "value",
|
||||
}
|
||||
|
||||
type blockHandle struct {
|
||||
offset uint64
|
||||
size uint64
|
||||
}
|
||||
|
||||
func ldbTableDecode(d *decode.D) any {
|
||||
d.Endian = decode.LittleEndian
|
||||
|
||||
// footer
|
||||
|
||||
var indexOffset int64
|
||||
var indexSize int64
|
||||
var metaIndexOffset int64
|
||||
var metaIndexSize int64
|
||||
|
||||
d.FieldStruct("footer", func(d *decode.D) {
|
||||
// check for magic number and fail fast if it isn't there
|
||||
d.SeekAbs(d.Len() - magicNumberLength)
|
||||
d.FieldU64("magic_number", d.UintAssert(tableMagicNumber), scalar.UintHex)
|
||||
|
||||
d.SeekAbs(d.Len() - footerEncodedLength)
|
||||
d.LimitedFn(footerEncodedLength-magicNumberLength, func(d *decode.D) {
|
||||
d.FieldStruct("metaindex_handle", func(d *decode.D) {
|
||||
metaIndexOffset = int64(d.FieldULEB128("offset"))
|
||||
metaIndexSize = int64(d.FieldULEB128("size"))
|
||||
})
|
||||
d.FieldStruct("index_handle", func(d *decode.D) {
|
||||
indexOffset = int64(d.FieldULEB128("offset"))
|
||||
indexSize = int64(d.FieldULEB128("size"))
|
||||
})
|
||||
d.FieldRawLen("padding", d.BitsLeft())
|
||||
})
|
||||
})
|
||||
|
||||
// metaindex
|
||||
|
||||
d.SeekAbs(metaIndexOffset * 8)
|
||||
var metaHandles []blockHandle
|
||||
readTableBlock(
|
||||
"metaindex",
|
||||
metaIndexSize,
|
||||
keyValueContentsReader(
|
||||
nil,
|
||||
func(d *decode.D) {
|
||||
handle := readBlockHandle(d)
|
||||
metaHandles = append(metaHandles, handle)
|
||||
},
|
||||
),
|
||||
d,
|
||||
)
|
||||
|
||||
// index
|
||||
|
||||
d.SeekAbs(indexOffset * 8)
|
||||
var dataHandles []blockHandle
|
||||
readTableBlock(
|
||||
"index",
|
||||
indexSize,
|
||||
keyValueContentsReader(
|
||||
readInternalKey,
|
||||
func(d *decode.D) {
|
||||
handle := readBlockHandle(d)
|
||||
dataHandles = append(dataHandles, handle)
|
||||
},
|
||||
),
|
||||
d,
|
||||
)
|
||||
|
||||
// meta
|
||||
|
||||
if len(metaHandles) > 0 {
|
||||
d.FieldArray("meta", func(d *decode.D) {
|
||||
for _, handle := range metaHandles {
|
||||
d.SeekAbs(int64(handle.offset) * 8)
|
||||
readTableBlock("meta_block", int64(handle.size), readMetaContent, d)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// data
|
||||
|
||||
if len(dataHandles) > 0 {
|
||||
d.FieldArray("data", func(d *decode.D) {
|
||||
for _, handle := range dataHandles {
|
||||
d.SeekAbs(int64(handle.offset) * 8)
|
||||
readTableBlock(
|
||||
"data_block",
|
||||
int64(handle.size),
|
||||
keyValueContentsReader(readInternalKey, nil),
|
||||
d,
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Readers
|
||||
|
||||
// Read block contents as well as compression + checksum bytes following it.
|
||||
// The function `readTableBlockContents` gets the _uncompressed_ bytebuffer.
|
||||
// https://github.com/google/leveldb/blob/main/table/format.cc#L69
|
||||
func readTableBlock(
|
||||
name string,
|
||||
size int64,
|
||||
readTableBlockContents func(size int64, d *decode.D),
|
||||
d *decode.D,
|
||||
) {
|
||||
d.FieldStruct(name, func(d *decode.D) {
|
||||
start := d.Pos()
|
||||
br := d.RawLen(size * 8)
|
||||
// compression (1 byte)
|
||||
compressionType := d.FieldU8("compression", compressionTypes, scalar.UintHex)
|
||||
// checksum (4 bytes)
|
||||
data := d.ReadAllBits(br)
|
||||
bytesToCheck := append(data, uint8(compressionType))
|
||||
checksum := computeChecksum(bytesToCheck)
|
||||
d.FieldU32("checksum", d.UintAssert(uint64(checksum)), scalar.UintHex)
|
||||
// decompress if needed
|
||||
d.SeekAbs(start)
|
||||
if compressionType == compressionTypeNone {
|
||||
d.FieldStruct("uncompressed", func(d *decode.D) {
|
||||
readTableBlockContents(size, d)
|
||||
})
|
||||
} else {
|
||||
compressedSize := size
|
||||
compressed := data
|
||||
bb := &bytes.Buffer{}
|
||||
switch compressionType {
|
||||
case compressionTypeSnappy:
|
||||
decompressed, err := snappy.Decode(nil, compressed)
|
||||
if err != nil {
|
||||
d.Errorf("failed decompressing data: %v", err)
|
||||
}
|
||||
d.Copy(bb, bytes.NewReader(decompressed))
|
||||
default:
|
||||
d.Errorf("Unsupported compression type: %x", compressionType)
|
||||
}
|
||||
if bb.Len() > 0 {
|
||||
d.FieldStructRootBitBufFn(
|
||||
"uncompressed",
|
||||
bitio.NewBitReader(bb.Bytes(), -1),
|
||||
func(d *decode.D) {
|
||||
readTableBlockContents(int64(bb.Len()), d)
|
||||
},
|
||||
)
|
||||
}
|
||||
d.FieldRawLen("compressed", compressedSize*8)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
// Read content encoded as a sequence of key/value-entries and a trailer of restarts.
|
||||
// https://github.com/google/leveldb/blob/main/table/block_builder.cc#L16
|
||||
// https://github.com/google/leveldb/blob/main/table/block.cc#L48
|
||||
func readKeyValueContents(
|
||||
keyCallbackFn func(sharedBytes []byte, unsharedSize int, d *decode.D) error,
|
||||
valueCallbackFn func(d *decode.D),
|
||||
size int64,
|
||||
d *decode.D,
|
||||
) {
|
||||
start := d.Pos()
|
||||
end := start + size*8
|
||||
|
||||
var restartOffset int64
|
||||
d.SeekAbs(end - uint32BitSize)
|
||||
d.FieldStruct("trailer", func(d *decode.D) {
|
||||
numRestarts := int64(d.FieldU32("num_restarts"))
|
||||
restartOffset = size*8 - (1+numRestarts)*uint32BitSize
|
||||
d.SeekAbs(start + restartOffset)
|
||||
d.FieldArray("restarts", func(d *decode.D) {
|
||||
for i := 0; i < int(numRestarts); i++ {
|
||||
d.FieldU32("restart")
|
||||
}
|
||||
})
|
||||
})
|
||||
// TK: how do you make an empty entries-array appear _above_ the trailer?
|
||||
// Right now, its omitted if empty.
|
||||
if restartOffset <= 0 {
|
||||
return
|
||||
}
|
||||
d.SeekAbs(start)
|
||||
d.FieldArray("entries", func(d *decode.D) {
|
||||
var lastKey []byte
|
||||
for d.Pos() < start+restartOffset {
|
||||
d.FieldStruct("entry", func(d *decode.D) {
|
||||
// https://github.com/google/leveldb/blob/main/table/block.cc#L48-L75
|
||||
shared := int64(d.FieldULEB128("shared_bytes"))
|
||||
unshared := int64(d.FieldULEB128("unshared_bytes"))
|
||||
valueLength := int64(d.FieldULEB128("value_length"))
|
||||
|
||||
// read key
|
||||
// https://github.com/google/leveldb/blob/main/table/block.cc#L261
|
||||
if int(shared) > len(lastKey) {
|
||||
d.Fatalf("`shared` size is larger than `lastKey` length")
|
||||
}
|
||||
keyPrefix := lastKey[:shared]
|
||||
keySuffix := readBytesWithoutChangingPosition(unshared, d)
|
||||
lastKey = append(keyPrefix, keySuffix...)
|
||||
|
||||
if keyCallbackFn == nil && shared == 0 {
|
||||
d.FieldUTF8("key", int(unshared))
|
||||
} else {
|
||||
d.FieldStruct("key", func(d *decode.D) {
|
||||
if keyCallbackFn == nil {
|
||||
keyCallbackFn = readPrefixedBytes
|
||||
}
|
||||
err := keyCallbackFn(keyPrefix, int(unshared), d)
|
||||
if err != nil {
|
||||
d.Errorf("%v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// read value
|
||||
if valueCallbackFn == nil {
|
||||
d.FieldUTF8("value", int(valueLength))
|
||||
} else {
|
||||
d.FieldStruct("value", valueCallbackFn)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func readBytesWithoutChangingPosition(nBytes int64, d *decode.D) []byte {
|
||||
var result []byte
|
||||
d.RangeFn(d.Pos(), nBytes*8, func(d *decode.D) {
|
||||
br := d.RawLen(d.BitsLeft())
|
||||
result = d.ReadAllBits(br)
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
// https://github.com/google/leveldb/blob/main/table/format.cc#L24
|
||||
func readBlockHandle(d *decode.D) blockHandle {
|
||||
return blockHandle{
|
||||
offset: d.FieldULEB128("offset"),
|
||||
size: d.FieldULEB128("size"),
|
||||
}
|
||||
}
|
||||
|
||||
// Read bytes and prefix with given bytes;
|
||||
// name read bytes "suffix" and the merged bytes "full" (synthetic field).
|
||||
func readPrefixedBytes(prefixBytes []byte, nBytes int, d *decode.D) error {
|
||||
br, err := d.TryFieldRawLen("suffix", int64(nBytes)*8)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
full := append(prefixBytes, d.ReadAllBits(br)...)
|
||||
d.FieldValueStr("full", string(full), strInferred)
|
||||
return nil
|
||||
}
|
||||
|
||||
// An "internal key" consists of the triple (user_key, type, sequence_number).
|
||||
// https://github.com/google/leveldb/blob/main/db/dbformat.h#L171
|
||||
func readInternalKey(sharedBytes []byte, unsharedSize int, d *decode.D) error {
|
||||
// In the LevelDB encoding, the internal key can be cut at any byte:
|
||||
// including the user_key, type, or sequence_number:
|
||||
// https://github.com/google/leveldb/blob/main/table/block_builder.cc#L79-L83
|
||||
//
|
||||
// The resulting prefix is then shared among subsequent keys and not
|
||||
// specified explicitly by them. Here, we handle each cutoff case.
|
||||
//
|
||||
// All sizes are in bytes unless mentioned otherwise.
|
||||
keySize := len(sharedBytes) + unsharedSize
|
||||
typeAndSequenceNumberSize := 8
|
||||
// key
|
||||
// +-----------------------------------------------+
|
||||
// user_key
|
||||
// +---------------------------------+
|
||||
// ⁞ user_key_suffix type sequence_number
|
||||
// [AAAAAAAAAAAA]⁞[BBBBBBBBBBBBBBBBBB] [T] [SSSSSSS]
|
||||
// ⁞ 1 7 bytes
|
||||
// +------------+⁞+--------------------------------+
|
||||
// shared ⁞ unshared
|
||||
// ⁞
|
||||
// cutoff
|
||||
if keySize < typeAndSequenceNumberSize || int64(unsharedSize) > d.BitsLeft()/8 {
|
||||
return fmt.Errorf("key size %d or unshared size %d invalid", keySize, unsharedSize)
|
||||
}
|
||||
|
||||
// case 1: user_key, type, and sequence_number fit fully in unshared.
|
||||
if len(sharedBytes) == 0 {
|
||||
d.FieldUTF8("user_key", keySize-typeAndSequenceNumberSize)
|
||||
d.FieldU8("type", valueTypes, scalar.UintHex)
|
||||
d.FieldU56("sequence_number")
|
||||
return nil
|
||||
}
|
||||
|
||||
// case 2: type and sequence_number fit fully in unshared: simulate user_key value.
|
||||
if unsharedSize >= typeAndSequenceNumberSize {
|
||||
suffix := fieldUTF8ReturnBytes("user_key_suffix", unsharedSize-typeAndSequenceNumberSize, d)
|
||||
d.FieldValueStr("user_key", stringify(sharedBytes, suffix), strInferred)
|
||||
d.FieldU8("type", valueTypes, scalar.UintHex)
|
||||
d.FieldU56("sequence_number")
|
||||
return nil
|
||||
}
|
||||
|
||||
// case 3: sequence_number fits fully in unshared: simulate user_key and type value,
|
||||
sequenceNumberSize := typeAndSequenceNumberSize - 1
|
||||
if unsharedSize == sequenceNumberSize {
|
||||
lastIndex := len(sharedBytes) - 1
|
||||
d.FieldValueStr("user_key", string(sharedBytes[:lastIndex]), strInferred)
|
||||
d.FieldValueUint("type", uint64(sharedBytes[lastIndex]), valueTypes, scalar.UintHex, uintInferred)
|
||||
d.FieldU56("sequence_number")
|
||||
return nil
|
||||
}
|
||||
|
||||
// case 4: sequence_number cut: simulate user_key, type, and sequence_number value.
|
||||
typeByteIndex := keySize - typeAndSequenceNumberSize
|
||||
d.FieldValueStr("user_key", string(sharedBytes[:typeByteIndex]), strInferred)
|
||||
d.FieldValueUint("type", uint64(sharedBytes[typeByteIndex]), valueTypes, scalar.UintHex, uintInferred)
|
||||
var suffixBytes []byte
|
||||
if unsharedSize > 0 {
|
||||
br := d.FieldRawLen("sequence_number_suffix", int64(unsharedSize)*8)
|
||||
suffixBytes = d.ReadAllBits(br)
|
||||
}
|
||||
sequenceNumberBytes := append(
|
||||
sharedBytes[typeByteIndex+1:keySize-unsharedSize],
|
||||
suffixBytes...,
|
||||
)
|
||||
sequenceNumberBE := bitio.Read64(sequenceNumberBytes[:], 0, int64(sequenceNumberSize*8))
|
||||
sequenceNumberLE := bitio.ReverseBytes64(56, sequenceNumberBE)
|
||||
d.FieldValueUint("sequence_number", sequenceNumberLE, uintInferred)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read content encoded in the "filter" or "stats" Meta Block format.
|
||||
// https://github.com/google/leveldb/blob/main/doc/table_format.md#filter-meta-block
|
||||
// https://github.com/google/leveldb/blob/main/table/filter_block.cc
|
||||
func readMetaContent(nBytes int64, d *decode.D) {
|
||||
// TK(2023-12-04)
|
||||
d.FieldRawLen("raw", nBytes*8)
|
||||
}
|
||||
|
||||
// Helpers
|
||||
|
||||
var strInferred = scalar.StrFn(func(s scalar.Str) (scalar.Str, error) {
|
||||
s.Description = "inferred"
|
||||
return s, nil
|
||||
})
|
||||
|
||||
var uintInferred = scalar.UintFn(func(s scalar.Uint) (scalar.Uint, error) {
|
||||
s.Description = "inferred"
|
||||
return s, nil
|
||||
})
|
||||
|
||||
func keyValueContentsReader(
|
||||
keyCallbackFn func(sharedPrefix []byte, unsharedSize int, d *decode.D) error,
|
||||
valueCallbackFn func(d *decode.D),
|
||||
) func(size int64, d *decode.D) {
|
||||
return func(size int64, d *decode.D) {
|
||||
readKeyValueContents(keyCallbackFn, valueCallbackFn, size, d)
|
||||
}
|
||||
}
|
||||
|
||||
// Compute the checksum: a CRC32 as in RFC3720 + custom mask.
|
||||
// https://datatracker.ietf.org/doc/html/rfc3720#appendix-B.4
|
||||
func computeChecksum(bytes []uint8) uint32 {
|
||||
crc32C := crc32.New(crc32.MakeTable(crc32.Castagnoli))
|
||||
crc32C.Write(bytes)
|
||||
return mask(crc32C.Sum32())
|
||||
}
|
||||
|
||||
// Return a masked representation of the CRC.
|
||||
// https://github.com/google/leveldb/blob/main/util/crc32c.h#L29
|
||||
func mask(crc uint32) uint32 {
|
||||
const kMaskDelta = 0xa282ead8
|
||||
// Rotate right by 15 bits and add a constant.
|
||||
return ((crc >> 15) | (crc << 17)) + kMaskDelta
|
||||
}
|
||||
|
||||
// Concatinate byteslices and convert into a string.
|
||||
func stringify(byteSlices ...[]byte) string {
|
||||
totalSize := 0
|
||||
for _, b := range byteSlices {
|
||||
totalSize += len(b)
|
||||
}
|
||||
|
||||
result := make([]byte, 0, totalSize)
|
||||
|
||||
for _, b := range byteSlices {
|
||||
result = append(result, b...)
|
||||
}
|
||||
|
||||
return string(result)
|
||||
}
|
||||
|
||||
func fieldUTF8ReturnBytes(name string, nBytes int, d *decode.D) []byte {
|
||||
var result []byte
|
||||
d.RangeFn(d.Pos(), int64(nBytes)*8, func(d *decode.D) {
|
||||
result = d.BytesLen(nBytes)
|
||||
})
|
||||
d.FieldUTF8(name, nBytes)
|
||||
return result
|
||||
}
|
14
format/leveldb/leveldb_table.md
Normal file
14
format/leveldb/leveldb_table.md
Normal file
@ -0,0 +1,14 @@
|
||||
### Limitations
|
||||
|
||||
- no Meta Blocks (like "filter") are decoded yet.
|
||||
- Zstandard uncompression is not implemented yet.
|
||||
|
||||
### Authors
|
||||
|
||||
- [@mikez](https://github.com/mikez), original author
|
||||
|
||||
### References
|
||||
|
||||
- https://github.com/google/leveldb/blob/main/doc/table_format.md
|
||||
- https://github.com/google/leveldb/blob/main/doc/impl.md
|
||||
- https://github.com/google/leveldb/blob/main/doc/index.md
|
24
format/leveldb/testdata/help_leveldb_descriptor.fqtest
vendored
Normal file
24
format/leveldb/testdata/help_leveldb_descriptor.fqtest
vendored
Normal file
@ -0,0 +1,24 @@
|
||||
$ fq -h leveldb_descriptor
|
||||
leveldb_descriptor: LevelDB Descriptor decoder
|
||||
|
||||
Decode examples
|
||||
===============
|
||||
|
||||
# Decode file as leveldb_descriptor
|
||||
$ fq -d leveldb_descriptor . file
|
||||
# Decode value as leveldb_descriptor
|
||||
... | leveldb_descriptor
|
||||
|
||||
Limitations
|
||||
===========
|
||||
- fragmented non-"full" records are not merged and decoded further.
|
||||
|
||||
Authors
|
||||
=======
|
||||
- @mikez (https://github.com/mikez), original author
|
||||
|
||||
References
|
||||
==========
|
||||
- https://github.com/google/leveldb/blob/main/doc/impl.md#manifest
|
||||
- https://github.com/google/leveldb/blob/main/doc/log_format.md
|
||||
- https://github.com/google/leveldb/blob/main/db/version_edit.cc
|
24
format/leveldb/testdata/help_leveldb_log.fqtest
vendored
Normal file
24
format/leveldb/testdata/help_leveldb_log.fqtest
vendored
Normal file
@ -0,0 +1,24 @@
|
||||
$ fq -h leveldb_log
|
||||
leveldb_log: LevelDB Log decoder
|
||||
|
||||
Decode examples
|
||||
===============
|
||||
|
||||
# Decode file as leveldb_log
|
||||
$ fq -d leveldb_log . file
|
||||
# Decode value as leveldb_log
|
||||
... | leveldb_log
|
||||
|
||||
Limitations
|
||||
===========
|
||||
- fragmented non-"full" records are not merged and decoded further.
|
||||
|
||||
Authors
|
||||
=======
|
||||
- @mikez (https://github.com/mikez), original author
|
||||
|
||||
References
|
||||
==========
|
||||
- https://github.com/google/leveldb/blob/main/doc/impl.md#log-files
|
||||
- https://github.com/google/leveldb/blob/main/doc/log_format.md
|
||||
- https://github.com/google/leveldb/blob/main/db/write_batch.cc
|
25
format/leveldb/testdata/help_leveldb_table.fqtest
vendored
Normal file
25
format/leveldb/testdata/help_leveldb_table.fqtest
vendored
Normal file
@ -0,0 +1,25 @@
|
||||
$ fq -h leveldb_table
|
||||
leveldb_table: LevelDB Table decoder
|
||||
|
||||
Decode examples
|
||||
===============
|
||||
|
||||
# Decode file as leveldb_table
|
||||
$ fq -d leveldb_table . file
|
||||
# Decode value as leveldb_table
|
||||
... | leveldb_table
|
||||
|
||||
Limitations
|
||||
===========
|
||||
- no Meta Blocks (like "filter") are decoded yet.
|
||||
- Zstandard uncompression is not implemented yet.
|
||||
|
||||
Authors
|
||||
=======
|
||||
- @mikez (https://github.com/mikez), original author
|
||||
|
||||
References
|
||||
==========
|
||||
- https://github.com/google/leveldb/blob/main/doc/table_format.md
|
||||
- https://github.com/google/leveldb/blob/main/doc/impl.md
|
||||
- https://github.com/google/leveldb/blob/main/doc/index.md
|
87
format/leveldb/testdata/leveldb_log.fqtest
vendored
Normal file
87
format/leveldb/testdata/leveldb_log.fqtest
vendored
Normal file
@ -0,0 +1,87 @@
|
||||
$ fq -d leveldb_log dv log_only.ldb/000003.log
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: log_only.ldb/000003.log (leveldb_log) 0x0-0x62c (1580)
|
||||
| | | blocks[0:1]: 0x0-0x62c (1580)
|
||||
| | | [0]{}: block 0x0-0x62c (1580)
|
||||
| | | records[0:4]: 0x0-0x62c (1580)
|
||||
| | | [0]{}: record 0x0-0x1df (479)
|
||||
| | | header{}: 0x0-0x7 (7)
|
||||
0x000|18 93 40 61 |..@a | checksum: 0x61409318 (valid) 0x0-0x4 (4)
|
||||
0x000| d8 01 | .. | length: 472 0x4-0x6 (2)
|
||||
0x000| 01 | . | record_type: "full" (1) 0x6-0x7 (1)
|
||||
| | | data{}: 0x7-0x1df (472)
|
||||
0x000| 01 00 00 00 00 00 00 00 | ........ | sequence: 1 0x7-0xf (8)
|
||||
0x000| 01| .| count: 1 0xf-0x13 (4)
|
||||
0x010|00 00 00 |... |
|
||||
| | | records[0:1]: 0x13-0x1df (460)
|
||||
| | | [0]{}: record 0x13-0x1df (460)
|
||||
0x010| 01 | . | type: "value" (1) 0x13-0x14 (1)
|
||||
| | | key{}: 0x14-0x20 (12)
|
||||
0x010| 0b | . | length: 11 0x14-0x15 (1)
|
||||
0x010| 6c 6f 72 65 6d 2e 6c 6f 72 65 6d| lorem.lorem| data: "lorem.lorem" 0x15-0x20 (11)
|
||||
| | | value{}: 0x20-0x1df (447)
|
||||
0x020|bd 03 |.. | length: 445 0x20-0x22 (2)
|
||||
0x020| 4c 6f 72 65 6d 20 69 70 73 75 6d 20 64 6f| Lorem ipsum do| data: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x22-0x1df (445)
|
||||
0x030|6c 6f 72 20 73 69 74 20 61 6d 65 74 2c 20 63 6f|lor sit amet, co|
|
||||
* |until 0x1de.7 (445) | |
|
||||
| | | [1]{}: record 0x1df-0x3be (479)
|
||||
| | | header{}: 0x1df-0x1e6 (7)
|
||||
0x1d0| 5a| Z| checksum: 0x12ba655a (valid) 0x1df-0x1e3 (4)
|
||||
0x1e0|65 ba 12 |e.. |
|
||||
0x1e0| d8 01 | .. | length: 472 0x1e3-0x1e5 (2)
|
||||
0x1e0| 01 | . | record_type: "full" (1) 0x1e5-0x1e6 (1)
|
||||
| | | data{}: 0x1e6-0x3be (472)
|
||||
0x1e0| 02 00 00 00 00 00 00 00 | ........ | sequence: 2 0x1e6-0x1ee (8)
|
||||
0x1e0| 01 00| ..| count: 1 0x1ee-0x1f2 (4)
|
||||
0x1f0|00 00 |.. |
|
||||
| | | records[0:1]: 0x1f2-0x3be (460)
|
||||
| | | [0]{}: record 0x1f2-0x3be (460)
|
||||
0x1f0| 01 | . | type: "value" (1) 0x1f2-0x1f3 (1)
|
||||
| | | key{}: 0x1f3-0x1ff (12)
|
||||
0x1f0| 0b | . | length: 11 0x1f3-0x1f4 (1)
|
||||
0x1f0| 6c 6f 72 65 6d 2e 69 70 73 75 6d | lorem.ipsum | data: "lorem.ipsum" 0x1f4-0x1ff (11)
|
||||
| | | value{}: 0x1ff-0x3be (447)
|
||||
0x1f0| bd| .| length: 445 0x1ff-0x201 (2)
|
||||
0x200|03 |. |
|
||||
0x200| 4c 6f 72 65 6d 20 69 70 73 75 6d 20 64 6f 6c| Lorem ipsum dol| data: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x201-0x3be (445)
|
||||
0x210|6f 72 20 73 69 74 20 61 6d 65 74 2c 20 63 6f 6e|or sit amet, con|
|
||||
* |until 0x3bd.7 (445) | |
|
||||
| | | [2]{}: record 0x3be-0x59d (479)
|
||||
| | | header{}: 0x3be-0x3c5 (7)
|
||||
0x3b0| 09 81| ..| checksum: 0x890d8109 (valid) 0x3be-0x3c2 (4)
|
||||
0x3c0|0d 89 |.. |
|
||||
0x3c0| d8 01 | .. | length: 472 0x3c2-0x3c4 (2)
|
||||
0x3c0| 01 | . | record_type: "full" (1) 0x3c4-0x3c5 (1)
|
||||
| | | data{}: 0x3c5-0x59d (472)
|
||||
0x3c0| 03 00 00 00 00 00 00 00 | ........ | sequence: 3 0x3c5-0x3cd (8)
|
||||
0x3c0| 01 00 00| ...| count: 1 0x3cd-0x3d1 (4)
|
||||
0x3d0|00 |. |
|
||||
| | | records[0:1]: 0x3d1-0x59d (460)
|
||||
| | | [0]{}: record 0x3d1-0x59d (460)
|
||||
0x3d0| 01 | . | type: "value" (1) 0x3d1-0x3d2 (1)
|
||||
| | | key{}: 0x3d2-0x3de (12)
|
||||
0x3d0| 0b | . | length: 11 0x3d2-0x3d3 (1)
|
||||
0x3d0| 6c 6f 72 65 6d 2e 64 6f 6c 6f 72 | lorem.dolor | data: "lorem.dolor" 0x3d3-0x3de (11)
|
||||
| | | value{}: 0x3de-0x59d (447)
|
||||
0x3d0| bd 03| ..| length: 445 0x3de-0x3e0 (2)
|
||||
0x3e0|4c 6f 72 65 6d 20 69 70 73 75 6d 20 64 6f 6c 6f|Lorem ipsum dolo| data: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x3e0-0x59d (445)
|
||||
* |until 0x59c.7 (445) | |
|
||||
| | | [3]{}: record 0x59d-0x62c (143)
|
||||
| | | header{}: 0x59d-0x5a4 (7)
|
||||
0x590| a0 86 3e| ..>| checksum: 0xb3e86a0 (valid) 0x59d-0x5a1 (4)
|
||||
0x5a0|0b |. |
|
||||
0x5a0| 88 00 | .. | length: 136 0x5a1-0x5a3 (2)
|
||||
0x5a0| 01 | . | record_type: "full" (1) 0x5a3-0x5a4 (1)
|
||||
| | | data{}: 0x5a4-0x62c (136)
|
||||
0x5a0| 04 00 00 00 00 00 00 00 | ........ | sequence: 4 0x5a4-0x5ac (8)
|
||||
0x5a0| 01 00 00 00| ....| count: 1 0x5ac-0x5b0 (4)
|
||||
| | | records[0:1]: 0x5b0-0x62c (124)
|
||||
| | | [0]{}: record 0x5b0-0x62c (124)
|
||||
0x5b0|01 |. | type: "value" (1) 0x5b0-0x5b1 (1)
|
||||
| | | key{}: 0x5b1-0x5b5 (4)
|
||||
0x5b0| 03 | . | length: 3 0x5b1-0x5b2 (1)
|
||||
0x5b0| 72 6f 77 | row | data: "row" 0x5b2-0x5b5 (3)
|
||||
| | | value{}: 0x5b5-0x62c (119)
|
||||
0x5b0| 76 | v | length: 118 0x5b5-0x5b6 (1)
|
||||
0x5b0| 52 6f 77 2c 20 72 6f 77 2c 20| Row, row, | data: "Row, row, row your boat\nGently down the stream...." 0x5b6-0x62c (118)
|
||||
0x5c0|72 6f 77 20 79 6f 75 72 20 62 6f 61 74 0a 47 65|row your boat.Ge|
|
||||
* |until 0x62b.7 (end) (118) | |
|
57
format/leveldb/testdata/leveldb_manifest.fqtest
vendored
Normal file
57
format/leveldb/testdata/leveldb_manifest.fqtest
vendored
Normal file
@ -0,0 +1,57 @@
|
||||
$ fq -d leveldb_descriptor dv uncompressed.ldb/MANIFEST-000004
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: uncompressed.ldb/MANIFEST-000004 (leveldb_descriptor) 0x0-0x57 (87)
|
||||
| | | blocks[0:1]: 0x0-0x57 (87)
|
||||
| | | [0]{}: block 0x0-0x57 (87)
|
||||
| | | records[0:2]: 0x0-0x57 (87)
|
||||
| | | [0]{}: record 0x0-0x23 (35)
|
||||
| | | header{}: 0x0-0x7 (7)
|
||||
0x00|56 f9 b8 f8 |V... | checksum: 0xf8b8f956 (valid) 0x0-0x4 (4)
|
||||
0x00| 1c 00 | .. | length: 28 0x4-0x6 (2)
|
||||
0x00| 01 | . | record_type: "full" (1) 0x6-0x7 (1)
|
||||
| | | data{}: 0x7-0x23 (28)
|
||||
| | | tags[0:1]: 0x7-0x23 (28)
|
||||
| | | [0]{}: tag 0x7-0x23 (28)
|
||||
0x00| 01 | . | key: "comparator" (1) 0x7-0x8 (1)
|
||||
| | | value{}: 0x8-0x23 (27)
|
||||
0x00| 1a | . | length: 26 0x8-0x9 (1)
|
||||
0x00| 6c 65 76 65 6c 64 62| leveldb| data: "leveldb.BytewiseComparator" 0x9-0x23 (26)
|
||||
0x10|2e 42 79 74 65 77 69 73 65 43 6f 6d 70 61 72 61|.BytewiseCompara|
|
||||
0x20|74 6f 72 |tor |
|
||||
| | | [1]{}: record 0x23-0x57 (52)
|
||||
| | | header{}: 0x23-0x2a (7)
|
||||
0x20| d2 53 ab 33 | .S.3 | checksum: 0x33ab53d2 (valid) 0x23-0x27 (4)
|
||||
0x20| 2d 00 | -. | length: 45 0x27-0x29 (2)
|
||||
0x20| 01 | . | record_type: "full" (1) 0x29-0x2a (1)
|
||||
| | | data{}: 0x2a-0x57 (45)
|
||||
| | | tags[0:5]: 0x2a-0x57 (45)
|
||||
| | | [0]{}: tag 0x2a-0x2c (2)
|
||||
0x20| 02 | . | key: "log_number" (2) 0x2a-0x2b (1)
|
||||
0x20| 06 | . | value: 6 0x2b-0x2c (1)
|
||||
| | | [1]{}: tag 0x2c-0x2e (2)
|
||||
0x20| 09 | . | key: "previous_log_number" (9) 0x2c-0x2d (1)
|
||||
0x20| 00 | . | value: 0 0x2d-0x2e (1)
|
||||
| | | [2]{}: tag 0x2e-0x30 (2)
|
||||
0x20| 03 | . | key: "next_file_number" (3) 0x2e-0x2f (1)
|
||||
0x20| 07| .| value: 7 0x2f-0x30 (1)
|
||||
| | | [3]{}: tag 0x30-0x32 (2)
|
||||
0x30|04 |. | key: "last_sequence" (4) 0x30-0x31 (1)
|
||||
0x30| 04 | . | value: 4 0x31-0x32 (1)
|
||||
| | | [4]{}: tag 0x32-0x57 (37)
|
||||
0x30| 07 | . | key: "new_file" (7) 0x32-0x33 (1)
|
||||
| | | value{}: 0x33-0x57 (36)
|
||||
0x30| 00 | . | level: 0 0x33-0x34 (1)
|
||||
0x30| 05 | . | file_number: 5 0x34-0x35 (1)
|
||||
0x30| da 0c | .. | file_size: 1626 0x35-0x37 (2)
|
||||
| | | smallest_internal_key{}: 0x37-0x4b (20)
|
||||
0x30| 13 | . | length: 19 0x37-0x38 (1)
|
||||
| | | data{}: 0x38-0x4b (19)
|
||||
0x30| 6c 6f 72 65 6d 2e 64 6f| lorem.do| user_key: "lorem.dolor" 0x38-0x43 (11)
|
||||
0x40|6c 6f 72 |lor |
|
||||
0x40| 01 | . | type: "value" (0x1) 0x43-0x44 (1)
|
||||
0x40| 03 00 00 00 00 00 00 | ....... | sequence_number: 3 0x44-0x4b (7)
|
||||
| | | largest_internal_key{}: 0x4b-0x57 (12)
|
||||
0x40| 0b | . | length: 11 0x4b-0x4c (1)
|
||||
| | | data{}: 0x4c-0x57 (11)
|
||||
0x40| 72 6f 77 | row | user_key: "row" 0x4c-0x4f (3)
|
||||
0x40| 01| .| type: "value" (0x1) 0x4f-0x50 (1)
|
||||
0x50|04 00 00 00 00 00 00| |.......| | sequence_number: 4 0x50-0x57 (7)
|
39
format/leveldb/testdata/leveldb_manifest_torepr.fqtest
vendored
Normal file
39
format/leveldb/testdata/leveldb_manifest_torepr.fqtest
vendored
Normal file
@ -0,0 +1,39 @@
|
||||
$ fq -d leveldb_descriptor torepr uncompressed.ldb/MANIFEST-000004
|
||||
[
|
||||
[
|
||||
{
|
||||
"comparator": "leveldb.BytewiseComparator"
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"log_number": 6
|
||||
},
|
||||
{
|
||||
"previous_log_number": 0
|
||||
},
|
||||
{
|
||||
"next_file_number": 7
|
||||
},
|
||||
{
|
||||
"last_sequence": 4
|
||||
},
|
||||
{
|
||||
"new_file": {
|
||||
"file_number": 5,
|
||||
"file_size": 1626,
|
||||
"largest_internal_key": {
|
||||
"sequence_number": 4,
|
||||
"type": "value",
|
||||
"user_key": "row"
|
||||
},
|
||||
"level": 0,
|
||||
"smallest_internal_key": {
|
||||
"sequence_number": 3,
|
||||
"type": "value",
|
||||
"user_key": "lorem.dolor"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
]
|
5539
format/leveldb/testdata/leveldb_table_repeats.fqtest
vendored
Normal file
5539
format/leveldb/testdata/leveldb_table_repeats.fqtest
vendored
Normal file
File diff suppressed because it is too large
Load Diff
104
format/leveldb/testdata/leveldb_table_snappy.fqtest
vendored
Normal file
104
format/leveldb/testdata/leveldb_table_snappy.fqtest
vendored
Normal file
@ -0,0 +1,104 @@
|
||||
$ fq -d leveldb_table dv snappy.ldb/000005.ldb
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: snappy.ldb/000005.ldb (leveldb_table) 0x0-0x2c4 (708)
|
||||
| | | data[0:1]: 0x0-0x26b (619)
|
||||
| | | [0]{}: data_block 0x0-0x26b (619)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| uncompressed{}: 0x0-0x5fc (1532)
|
||||
| | | entries[0:4]: 0x0-0x5f4 (1524)
|
||||
| | | [0]{}: entry 0x0-0x1d4 (468)
|
||||
0x000|00 |. | shared_bytes: 0 0x0-0x1 (1)
|
||||
0x000| 13 | . | unshared_bytes: 19 0x1-0x2 (1)
|
||||
0x000| bd 03 | .. | value_length: 445 0x2-0x4 (2)
|
||||
| | | key{}: 0x4-0x17 (19)
|
||||
0x000| 6c 6f 72 65 6d 2e 64 6f 6c 6f 72 | lorem.dolor | user_key: "lorem.dolor" 0x4-0xf (11)
|
||||
0x000| 01| .| type: "value" (0x1) 0xf-0x10 (1)
|
||||
0x001|03 00 00 00 00 00 00 |....... | sequence_number: 3 0x10-0x17 (7)
|
||||
0x001| 4c 6f 72 65 6d 20 69 70 73| Lorem ips| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x17-0x1d4 (445)
|
||||
0x002|75 6d 20 64 6f 6c 6f 72 20 73 69 74 20 61 6d 65|um dolor sit ame|
|
||||
* |until 0x1d3.7 (445) | |
|
||||
| | | [1]{}: entry 0x1d4-0x3a2 (462)
|
||||
0x01d| 06 | . | shared_bytes: 6 0x1d4-0x1d5 (1)
|
||||
0x01d| 0d | . | unshared_bytes: 13 0x1d5-0x1d6 (1)
|
||||
0x01d| bd 03 | .. | value_length: 445 0x1d6-0x1d8 (2)
|
||||
| | | key{}: 0x1d8-0x1e5 (13)
|
||||
0x01d| 69 70 73 75 6d | ipsum | user_key_suffix: "ipsum" 0x1d8-0x1dd (5)
|
||||
| | | user_key: "lorem.ipsum" (inferred)
|
||||
0x01d| 01 | . | type: "value" (0x1) 0x1dd-0x1de (1)
|
||||
0x01d| 02 00| ..| sequence_number: 2 0x1de-0x1e5 (7)
|
||||
0x01e|00 00 00 00 00 |..... |
|
||||
0x01e| 4c 6f 72 65 6d 20 69 70 73 75 6d| Lorem ipsum| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x1e5-0x3a2 (445)
|
||||
0x01f|20 64 6f 6c 6f 72 20 73 69 74 20 61 6d 65 74 2c| dolor sit amet,|
|
||||
* |until 0x3a1.7 (445) | |
|
||||
| | | [2]{}: entry 0x3a2-0x570 (462)
|
||||
0x03a| 06 | . | shared_bytes: 6 0x3a2-0x3a3 (1)
|
||||
0x03a| 0d | . | unshared_bytes: 13 0x3a3-0x3a4 (1)
|
||||
0x03a| bd 03 | .. | value_length: 445 0x3a4-0x3a6 (2)
|
||||
| | | key{}: 0x3a6-0x3b3 (13)
|
||||
0x03a| 6c 6f 72 65 6d | lorem | user_key_suffix: "lorem" 0x3a6-0x3ab (5)
|
||||
| | | user_key: "lorem.lorem" (inferred)
|
||||
0x03a| 01 | . | type: "value" (0x1) 0x3ab-0x3ac (1)
|
||||
0x03a| 01 00 00 00| ....| sequence_number: 1 0x3ac-0x3b3 (7)
|
||||
0x03b|00 00 00 |... |
|
||||
0x03b| 4c 6f 72 65 6d 20 69 70 73 75 6d 20 64| Lorem ipsum d| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x3b3-0x570 (445)
|
||||
0x03c|6f 6c 6f 72 20 73 69 74 20 61 6d 65 74 2c 20 63|olor sit amet, c|
|
||||
* |until 0x56f.7 (445) | |
|
||||
| | | [3]{}: entry 0x570-0x5f4 (132)
|
||||
0x057|00 |. | shared_bytes: 0 0x570-0x571 (1)
|
||||
0x057| 0b | . | unshared_bytes: 11 0x571-0x572 (1)
|
||||
0x057| 76 | v | value_length: 118 0x572-0x573 (1)
|
||||
| | | key{}: 0x573-0x57e (11)
|
||||
0x057| 72 6f 77 | row | user_key: "row" 0x573-0x576 (3)
|
||||
0x057| 01 | . | type: "value" (0x1) 0x576-0x577 (1)
|
||||
0x057| 04 00 00 00 00 00 00 | ....... | sequence_number: 4 0x577-0x57e (7)
|
||||
0x057| 52 6f| Ro| value: "Row, row, row your boat\nGently down the stream...." 0x57e-0x5f4 (118)
|
||||
0x058|77 2c 20 72 6f 77 2c 20 72 6f 77 20 79 6f 75 72|w, row, row your|
|
||||
* |until 0x5f3.7 (118) | |
|
||||
| | | trailer{}: 0x5f4-0x5fc (8)
|
||||
| | | restarts[0:1]: 0x5f4-0x5f8 (4)
|
||||
0x05f| 00 00 00 00 | .... | [0]: 0 restart 0x5f4-0x5f8 (4)
|
||||
0x05f| 01 00 00 00| | ....| | num_restarts: 1 0x5f8-0x5fc (4)
|
||||
0x00000|fc 0b 44 00 13 bd 03 6c 6f 72 65 6d 2e 64 6f 6c|..D....lorem.dol| compressed: raw bits 0x0-0x266 (614)
|
||||
* |until 0x265.7 (614) | |
|
||||
0x00260| 01 | . | compression: "snappy" (0x1) 0x266-0x267 (1)
|
||||
0x00260| b6 9d 28 0d | ..(. | checksum: 0xd289db6 (valid) 0x267-0x26b (4)
|
||||
| | | metaindex{}: 0x26b-0x278 (13)
|
||||
| | | uncompressed{}: 0x26b-0x273 (8)
|
||||
| | | trailer{}: 0x26b-0x273 (8)
|
||||
| | | restarts[0:1]: 0x26b-0x26f (4)
|
||||
0x00260| 00 00 00 00 | .... | [0]: 0 restart 0x26b-0x26f (4)
|
||||
0x00260| 01| .| num_restarts: 1 0x26f-0x273 (4)
|
||||
0x00270|00 00 00 |... |
|
||||
0x00270| 00 | . | compression: "none" (0x0) 0x273-0x274 (1)
|
||||
0x00270| c0 f2 a1 b0 | .... | checksum: 0xb0a1f2c0 (valid) 0x274-0x278 (4)
|
||||
| | | index{}: 0x278-0x294 (28)
|
||||
| | | uncompressed{}: 0x278-0x28f (23)
|
||||
| | | entries[0:1]: 0x278-0x287 (15)
|
||||
| | | [0]{}: entry 0x278-0x287 (15)
|
||||
0x00270| 00 | . | shared_bytes: 0 0x278-0x279 (1)
|
||||
0x00270| 09 | . | unshared_bytes: 9 0x279-0x27a (1)
|
||||
0x00270| 03 | . | value_length: 3 0x27a-0x27b (1)
|
||||
| | | key{}: 0x27b-0x284 (9)
|
||||
0x00270| 73 | s | user_key: "s" 0x27b-0x27c (1)
|
||||
0x00270| 01 | . | type: "value" (0x1) 0x27c-0x27d (1)
|
||||
0x00270| ff ff ff| ...| sequence_number: 72057594037927935 0x27d-0x284 (7)
|
||||
0x00280|ff ff ff ff |.... |
|
||||
| | | value{}: 0x284-0x287 (3)
|
||||
0x00280| 00 | . | offset: 0 0x284-0x285 (1)
|
||||
0x00280| e6 04 | .. | size: 614 0x285-0x287 (2)
|
||||
| | | trailer{}: 0x287-0x28f (8)
|
||||
| | | restarts[0:1]: 0x287-0x28b (4)
|
||||
0x00280| 00 00 00 00 | .... | [0]: 0 restart 0x287-0x28b (4)
|
||||
0x00280| 01 00 00 00 | .... | num_restarts: 1 0x28b-0x28f (4)
|
||||
0x00280| 00| .| compression: "none" (0x0) 0x28f-0x290 (1)
|
||||
0x00290|68 24 42 91 |h$B. | checksum: 0x91422468 (valid) 0x290-0x294 (4)
|
||||
| | | footer{}: 0x294-0x2c4 (48)
|
||||
| | | metaindex_handle{}: 0x294-0x297 (3)
|
||||
0x00290| eb 04 | .. | offset: 619 0x294-0x296 (2)
|
||||
0x00290| 08 | . | size: 8 0x296-0x297 (1)
|
||||
| | | index_handle{}: 0x297-0x29a (3)
|
||||
0x00290| f8 04 | .. | offset: 632 0x297-0x299 (2)
|
||||
0x00290| 17 | . | size: 23 0x299-0x29a (1)
|
||||
0x00290| 00 00 00 00 00 00| ......| padding: raw bits 0x29a-0x2bc (34)
|
||||
0x002a0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
|
||||
0x002b0|00 00 00 00 00 00 00 00 00 00 00 00 |............ |
|
||||
0x002b0| 57 fb 80 8b| W...| magic_number: 0xdb4775248b80fb57 (valid) 0x2bc-0x2c4 (8)
|
||||
0x002c0|24 75 47 db| |$uG.| |
|
100
format/leveldb/testdata/leveldb_table_uncompressed.fqtest
vendored
Normal file
100
format/leveldb/testdata/leveldb_table_uncompressed.fqtest
vendored
Normal file
@ -0,0 +1,100 @@
|
||||
$ fq -d leveldb_table dv uncompressed.ldb/000005.ldb
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: uncompressed.ldb/000005.ldb (leveldb_table) 0x0-0x65a (1626)
|
||||
| | | data[0:1]: 0x0-0x601 (1537)
|
||||
| | | [0]{}: data_block 0x0-0x601 (1537)
|
||||
| | | uncompressed{}: 0x0-0x5fc (1532)
|
||||
| | | entries[0:4]: 0x0-0x5f4 (1524)
|
||||
| | | [0]{}: entry 0x0-0x1d4 (468)
|
||||
0x000|00 |. | shared_bytes: 0 0x0-0x1 (1)
|
||||
0x000| 13 | . | unshared_bytes: 19 0x1-0x2 (1)
|
||||
0x000| bd 03 | .. | value_length: 445 0x2-0x4 (2)
|
||||
| | | key{}: 0x4-0x17 (19)
|
||||
0x000| 6c 6f 72 65 6d 2e 64 6f 6c 6f 72 | lorem.dolor | user_key: "lorem.dolor" 0x4-0xf (11)
|
||||
0x000| 01| .| type: "value" (0x1) 0xf-0x10 (1)
|
||||
0x010|03 00 00 00 00 00 00 |....... | sequence_number: 3 0x10-0x17 (7)
|
||||
0x010| 4c 6f 72 65 6d 20 69 70 73| Lorem ips| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x17-0x1d4 (445)
|
||||
0x020|75 6d 20 64 6f 6c 6f 72 20 73 69 74 20 61 6d 65|um dolor sit ame|
|
||||
* |until 0x1d3.7 (445) | |
|
||||
| | | [1]{}: entry 0x1d4-0x3a2 (462)
|
||||
0x1d0| 06 | . | shared_bytes: 6 0x1d4-0x1d5 (1)
|
||||
0x1d0| 0d | . | unshared_bytes: 13 0x1d5-0x1d6 (1)
|
||||
0x1d0| bd 03 | .. | value_length: 445 0x1d6-0x1d8 (2)
|
||||
| | | key{}: 0x1d8-0x1e5 (13)
|
||||
0x1d0| 69 70 73 75 6d | ipsum | user_key_suffix: "ipsum" 0x1d8-0x1dd (5)
|
||||
| | | user_key: "lorem.ipsum" (inferred)
|
||||
0x1d0| 01 | . | type: "value" (0x1) 0x1dd-0x1de (1)
|
||||
0x1d0| 02 00| ..| sequence_number: 2 0x1de-0x1e5 (7)
|
||||
0x1e0|00 00 00 00 00 |..... |
|
||||
0x1e0| 4c 6f 72 65 6d 20 69 70 73 75 6d| Lorem ipsum| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x1e5-0x3a2 (445)
|
||||
0x1f0|20 64 6f 6c 6f 72 20 73 69 74 20 61 6d 65 74 2c| dolor sit amet,|
|
||||
* |until 0x3a1.7 (445) | |
|
||||
| | | [2]{}: entry 0x3a2-0x570 (462)
|
||||
0x3a0| 06 | . | shared_bytes: 6 0x3a2-0x3a3 (1)
|
||||
0x3a0| 0d | . | unshared_bytes: 13 0x3a3-0x3a4 (1)
|
||||
0x3a0| bd 03 | .. | value_length: 445 0x3a4-0x3a6 (2)
|
||||
| | | key{}: 0x3a6-0x3b3 (13)
|
||||
0x3a0| 6c 6f 72 65 6d | lorem | user_key_suffix: "lorem" 0x3a6-0x3ab (5)
|
||||
| | | user_key: "lorem.lorem" (inferred)
|
||||
0x3a0| 01 | . | type: "value" (0x1) 0x3ab-0x3ac (1)
|
||||
0x3a0| 01 00 00 00| ....| sequence_number: 1 0x3ac-0x3b3 (7)
|
||||
0x3b0|00 00 00 |... |
|
||||
0x3b0| 4c 6f 72 65 6d 20 69 70 73 75 6d 20 64| Lorem ipsum d| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x3b3-0x570 (445)
|
||||
0x3c0|6f 6c 6f 72 20 73 69 74 20 61 6d 65 74 2c 20 63|olor sit amet, c|
|
||||
* |until 0x56f.7 (445) | |
|
||||
| | | [3]{}: entry 0x570-0x5f4 (132)
|
||||
0x570|00 |. | shared_bytes: 0 0x570-0x571 (1)
|
||||
0x570| 0b | . | unshared_bytes: 11 0x571-0x572 (1)
|
||||
0x570| 76 | v | value_length: 118 0x572-0x573 (1)
|
||||
| | | key{}: 0x573-0x57e (11)
|
||||
0x570| 72 6f 77 | row | user_key: "row" 0x573-0x576 (3)
|
||||
0x570| 01 | . | type: "value" (0x1) 0x576-0x577 (1)
|
||||
0x570| 04 00 00 00 00 00 00 | ....... | sequence_number: 4 0x577-0x57e (7)
|
||||
0x570| 52 6f| Ro| value: "Row, row, row your boat\nGently down the stream...." 0x57e-0x5f4 (118)
|
||||
0x580|77 2c 20 72 6f 77 2c 20 72 6f 77 20 79 6f 75 72|w, row, row your|
|
||||
* |until 0x5f3.7 (118) | |
|
||||
| | | trailer{}: 0x5f4-0x5fc (8)
|
||||
| | | restarts[0:1]: 0x5f4-0x5f8 (4)
|
||||
0x5f0| 00 00 00 00 | .... | [0]: 0 restart 0x5f4-0x5f8 (4)
|
||||
0x5f0| 01 00 00 00 | .... | num_restarts: 1 0x5f8-0x5fc (4)
|
||||
0x5f0| 00 | . | compression: "none" (0x0) 0x5fc-0x5fd (1)
|
||||
0x5f0| 6f 99 1d| o..| checksum: 0xb31d996f (valid) 0x5fd-0x601 (4)
|
||||
0x600|b3 |. |
|
||||
| | | metaindex{}: 0x601-0x60e (13)
|
||||
| | | uncompressed{}: 0x601-0x609 (8)
|
||||
| | | trailer{}: 0x601-0x609 (8)
|
||||
| | | restarts[0:1]: 0x601-0x605 (4)
|
||||
0x600| 00 00 00 00 | .... | [0]: 0 restart 0x601-0x605 (4)
|
||||
0x600| 01 00 00 00 | .... | num_restarts: 1 0x605-0x609 (4)
|
||||
0x600| 00 | . | compression: "none" (0x0) 0x609-0x60a (1)
|
||||
0x600| c0 f2 a1 b0 | .... | checksum: 0xb0a1f2c0 (valid) 0x60a-0x60e (4)
|
||||
| | | index{}: 0x60e-0x62a (28)
|
||||
| | | uncompressed{}: 0x60e-0x625 (23)
|
||||
| | | entries[0:1]: 0x60e-0x61d (15)
|
||||
| | | [0]{}: entry 0x60e-0x61d (15)
|
||||
0x600| 00 | . | shared_bytes: 0 0x60e-0x60f (1)
|
||||
0x600| 09| .| unshared_bytes: 9 0x60f-0x610 (1)
|
||||
0x610|03 |. | value_length: 3 0x610-0x611 (1)
|
||||
| | | key{}: 0x611-0x61a (9)
|
||||
0x610| 73 | s | user_key: "s" 0x611-0x612 (1)
|
||||
0x610| 01 | . | type: "value" (0x1) 0x612-0x613 (1)
|
||||
0x610| ff ff ff ff ff ff ff | ....... | sequence_number: 72057594037927935 0x613-0x61a (7)
|
||||
| | | value{}: 0x61a-0x61d (3)
|
||||
0x610| 00 | . | offset: 0 0x61a-0x61b (1)
|
||||
0x610| fc 0b | .. | size: 1532 0x61b-0x61d (2)
|
||||
| | | trailer{}: 0x61d-0x625 (8)
|
||||
| | | restarts[0:1]: 0x61d-0x621 (4)
|
||||
0x610| 00 00 00| ...| [0]: 0 restart 0x61d-0x621 (4)
|
||||
0x620|00 |. |
|
||||
0x620| 01 00 00 00 | .... | num_restarts: 1 0x621-0x625 (4)
|
||||
0x620| 00 | . | compression: "none" (0x0) 0x625-0x626 (1)
|
||||
0x620| 68 e2 bf 46 | h..F | checksum: 0x46bfe268 (valid) 0x626-0x62a (4)
|
||||
| | | footer{}: 0x62a-0x65a (48)
|
||||
| | | metaindex_handle{}: 0x62a-0x62d (3)
|
||||
0x620| 81 0c | .. | offset: 1537 0x62a-0x62c (2)
|
||||
0x620| 08 | . | size: 8 0x62c-0x62d (1)
|
||||
| | | index_handle{}: 0x62d-0x630 (3)
|
||||
0x620| 8e 0c | .. | offset: 1550 0x62d-0x62f (2)
|
||||
0x620| 17| .| size: 23 0x62f-0x630 (1)
|
||||
0x630|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| padding: raw bits 0x630-0x652 (34)
|
||||
* |until 0x651.7 (34) | |
|
||||
0x650| 57 fb 80 8b 24 75 47 db| | W...$uG.| | magic_number: 0xdb4775248b80fb57 (valid) 0x652-0x65a (8)
|
BIN
format/leveldb/testdata/log_only.ldb/000003.log
vendored
Normal file
BIN
format/leveldb/testdata/log_only.ldb/000003.log
vendored
Normal file
Binary file not shown.
1
format/leveldb/testdata/log_only.ldb/CURRENT
vendored
Normal file
1
format/leveldb/testdata/log_only.ldb/CURRENT
vendored
Normal file
@ -0,0 +1 @@
|
||||
MANIFEST-000002
|
0
format/leveldb/testdata/log_only.ldb/LOCK
vendored
Normal file
0
format/leveldb/testdata/log_only.ldb/LOCK
vendored
Normal file
2
format/leveldb/testdata/log_only.ldb/LOG
vendored
Normal file
2
format/leveldb/testdata/log_only.ldb/LOG
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
2023/12/05-20:38:42.412901 0x107ac5e00 Creating DB ./log_only.ldb since it was missing.
|
||||
2023/12/05-20:38:42.512812 0x107ac5e00 Delete type=3 #1
|
BIN
format/leveldb/testdata/log_only.ldb/MANIFEST-000002
vendored
Normal file
BIN
format/leveldb/testdata/log_only.ldb/MANIFEST-000002
vendored
Normal file
Binary file not shown.
6
format/leveldb/testdata/lorem.json
vendored
Normal file
6
format/leveldb/testdata/lorem.json
vendored
Normal file
@ -0,0 +1,6 @@
|
||||
{
|
||||
"lorem.lorem": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
|
||||
"lorem.ipsum": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
|
||||
"lorem.dolor": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
|
||||
"row": "Row, row, row your boat\nGently down the stream.\nMerrily, merrily, merrily, merrily,\nLife is but a dream. 🚣♂️"
|
||||
}
|
67
format/leveldb/testdata/make_ldb.py
vendored
Normal file
67
format/leveldb/testdata/make_ldb.py
vendored
Normal file
@ -0,0 +1,67 @@
|
||||
# Make LevelDB data: both uncompressed and compressed.
|
||||
# Usage: python3 make_ldb.py
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
|
||||
import plyvel # pip install plyvel
|
||||
import snappy # pip install python-snappy
|
||||
|
||||
|
||||
def main():
|
||||
make("./lorem.json", "./uncompressed.ldb", compression=None, reopen=True)
|
||||
make("./lorem.json", "./snappy.ldb", compression="snappy", reopen=True)
|
||||
make(make_sample_json(), "./repeats.ldb", rounds=2, reopen=True)
|
||||
make("./lorem.json", "./log_only.ldb", compression=None)
|
||||
|
||||
|
||||
def make(input_filepath, output_filepath, compression="snappy", reopen=False, rounds=1):
|
||||
if os.path.exists(output_filepath):
|
||||
raise FileExistsError(f"The file {output_filepath} already exists.")
|
||||
# make a .ldb file and a .log file within
|
||||
db = plyvel.DB(output_filepath, compression=compression, create_if_missing=True)
|
||||
data = read_json(input_filepath)
|
||||
for i in range(rounds):
|
||||
for key, value in data.items():
|
||||
db.put(key.encode(), value.encode())
|
||||
db.close()
|
||||
if reopen:
|
||||
# reopen, so a .ldb file is generated within the .ldb directory;
|
||||
# otherwise there's a .log file only with the fresh changes.
|
||||
db = plyvel.DB(output_filepath, compression=compression)
|
||||
db.close()
|
||||
|
||||
|
||||
def make_sample_json():
|
||||
# in table data blocks, this splits the shared key
|
||||
# inside the sequence_number of the internal key;
|
||||
# see `readInternalKey` in leveldb_table.go for details.
|
||||
data = {}
|
||||
for i in range(0x100):
|
||||
data[f"lorem.{chr(i)}"] = "ipsum"
|
||||
result = io.StringIO()
|
||||
json.dump(data, result)
|
||||
result.seek(0)
|
||||
return result
|
||||
|
||||
|
||||
# Helpers
|
||||
|
||||
|
||||
def compress(value):
|
||||
return snappy.compress(value)
|
||||
|
||||
|
||||
def decompress(value):
|
||||
return snappy.decompress(value)
|
||||
|
||||
|
||||
def read_json(filepath_or_buffer):
|
||||
if hasattr(filepath_or_buffer, "read"):
|
||||
return json.load(filepath_or_buffer)
|
||||
with open(filepath_or_buffer, "r") as file:
|
||||
return json.load(file)
|
||||
|
||||
|
||||
main()
|
BIN
format/leveldb/testdata/repeats.ldb/000005.ldb
vendored
Normal file
BIN
format/leveldb/testdata/repeats.ldb/000005.ldb
vendored
Normal file
Binary file not shown.
0
format/leveldb/testdata/repeats.ldb/000006.log
vendored
Normal file
0
format/leveldb/testdata/repeats.ldb/000006.log
vendored
Normal file
1
format/leveldb/testdata/repeats.ldb/CURRENT
vendored
Normal file
1
format/leveldb/testdata/repeats.ldb/CURRENT
vendored
Normal file
@ -0,0 +1 @@
|
||||
MANIFEST-000004
|
0
format/leveldb/testdata/repeats.ldb/LOCK
vendored
Normal file
0
format/leveldb/testdata/repeats.ldb/LOCK
vendored
Normal file
5
format/leveldb/testdata/repeats.ldb/LOG
vendored
Normal file
5
format/leveldb/testdata/repeats.ldb/LOG
vendored
Normal file
@ -0,0 +1,5 @@
|
||||
2023/12/08-22:52:56.629835 0x10eb27e00 Recovering log #3
|
||||
2023/12/08-22:52:56.631293 0x10eb27e00 Level-0 table #5: started
|
||||
2023/12/08-22:52:56.640224 0x10eb27e00 Level-0 table #5: 2333 bytes OK
|
||||
2023/12/08-22:52:56.682590 0x10eb27e00 Delete type=0 #3
|
||||
2023/12/08-22:52:56.682721 0x10eb27e00 Delete type=3 #2
|
2
format/leveldb/testdata/repeats.ldb/LOG.old
vendored
Normal file
2
format/leveldb/testdata/repeats.ldb/LOG.old
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
2023/12/08-22:52:56.458994 0x10eb27e00 Creating DB ./repeats.ldb since it was missing.
|
||||
2023/12/08-22:52:56.591817 0x10eb27e00 Delete type=3 #1
|
BIN
format/leveldb/testdata/repeats.ldb/MANIFEST-000004
vendored
Normal file
BIN
format/leveldb/testdata/repeats.ldb/MANIFEST-000004
vendored
Normal file
Binary file not shown.
BIN
format/leveldb/testdata/snappy.ldb/000005.ldb
vendored
Normal file
BIN
format/leveldb/testdata/snappy.ldb/000005.ldb
vendored
Normal file
Binary file not shown.
0
format/leveldb/testdata/snappy.ldb/000006.log
vendored
Normal file
0
format/leveldb/testdata/snappy.ldb/000006.log
vendored
Normal file
1
format/leveldb/testdata/snappy.ldb/CURRENT
vendored
Normal file
1
format/leveldb/testdata/snappy.ldb/CURRENT
vendored
Normal file
@ -0,0 +1 @@
|
||||
MANIFEST-000004
|
0
format/leveldb/testdata/snappy.ldb/LOCK
vendored
Normal file
0
format/leveldb/testdata/snappy.ldb/LOCK
vendored
Normal file
5
format/leveldb/testdata/snappy.ldb/LOG
vendored
Normal file
5
format/leveldb/testdata/snappy.ldb/LOG
vendored
Normal file
@ -0,0 +1,5 @@
|
||||
2023/12/05-20:38:42.349463 0x107ac5e00 Recovering log #3
|
||||
2023/12/05-20:38:42.350562 0x107ac5e00 Level-0 table #5: started
|
||||
2023/12/05-20:38:42.359725 0x107ac5e00 Level-0 table #5: 708 bytes OK
|
||||
2023/12/05-20:38:42.411415 0x107ac5e00 Delete type=0 #3
|
||||
2023/12/05-20:38:42.411476 0x107ac5e00 Delete type=3 #2
|
2
format/leveldb/testdata/snappy.ldb/LOG.old
vendored
Normal file
2
format/leveldb/testdata/snappy.ldb/LOG.old
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
2023/12/05-20:38:42.215352 0x107ac5e00 Creating DB ./snappy.ldb since it was missing.
|
||||
2023/12/05-20:38:42.294790 0x107ac5e00 Delete type=3 #1
|
BIN
format/leveldb/testdata/snappy.ldb/MANIFEST-000004
vendored
Normal file
BIN
format/leveldb/testdata/snappy.ldb/MANIFEST-000004
vendored
Normal file
Binary file not shown.
BIN
format/leveldb/testdata/uncompressed.ldb/000005.ldb
vendored
Normal file
BIN
format/leveldb/testdata/uncompressed.ldb/000005.ldb
vendored
Normal file
Binary file not shown.
0
format/leveldb/testdata/uncompressed.ldb/000006.log
vendored
Normal file
0
format/leveldb/testdata/uncompressed.ldb/000006.log
vendored
Normal file
1
format/leveldb/testdata/uncompressed.ldb/CURRENT
vendored
Normal file
1
format/leveldb/testdata/uncompressed.ldb/CURRENT
vendored
Normal file
@ -0,0 +1 @@
|
||||
MANIFEST-000004
|
0
format/leveldb/testdata/uncompressed.ldb/LOCK
vendored
Normal file
0
format/leveldb/testdata/uncompressed.ldb/LOCK
vendored
Normal file
5
format/leveldb/testdata/uncompressed.ldb/LOG
vendored
Normal file
5
format/leveldb/testdata/uncompressed.ldb/LOG
vendored
Normal file
@ -0,0 +1,5 @@
|
||||
2023/12/05-20:38:41.945807 0x107ac5e00 Recovering log #3
|
||||
2023/12/05-20:38:41.945944 0x107ac5e00 Level-0 table #5: started
|
||||
2023/12/05-20:38:41.953310 0x107ac5e00 Level-0 table #5: 1626 bytes OK
|
||||
2023/12/05-20:38:42.214073 0x107ac5e00 Delete type=0 #3
|
||||
2023/12/05-20:38:42.214131 0x107ac5e00 Delete type=3 #2
|
2
format/leveldb/testdata/uncompressed.ldb/LOG.old
vendored
Normal file
2
format/leveldb/testdata/uncompressed.ldb/LOG.old
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
2023/12/05-20:38:41.773771 0x107ac5e00 Creating DB ./uncompressed.ldb since it was missing.
|
||||
2023/12/05-20:38:41.910278 0x107ac5e00 Delete type=3 #1
|
BIN
format/leveldb/testdata/uncompressed.ldb/MANIFEST-000004
vendored
Normal file
BIN
format/leveldb/testdata/uncompressed.ldb/MANIFEST-000004
vendored
Normal file
Binary file not shown.
@ -938,7 +938,7 @@ func (d *D) FramedFn(nBits int64, fn func(d *D)) int64 {
|
||||
return decodeLen
|
||||
}
|
||||
|
||||
// LimitedFn decode from current position nBits forward. When done position will after last bit decoded.
|
||||
// LimitedFn decode from current position nBits forward. When done position will be after last bit decoded.
|
||||
func (d *D) LimitedFn(nBits int64, fn func(d *D)) int64 {
|
||||
if nBits < 0 {
|
||||
d.Fatalf("%d nBits < 0", nBits)
|
||||
|
@ -247,10 +247,15 @@ func (d *D) tryBool() (bool, error) {
|
||||
return n == 1, nil
|
||||
}
|
||||
|
||||
// Unsigned LEB128, description from wasm spec
|
||||
// Unsigned LEB128, also known as "Base 128 Varint".
|
||||
//
|
||||
// Description from wasm spec:
|
||||
//
|
||||
// uN ::= n:byte => n (if n < 2^7 && n < 2^N)
|
||||
// n:byte m:u(N-7) => 2^7 * m + (n - 2^7) (if n >= 2^7 && N > 7)
|
||||
//
|
||||
// Varint description:
|
||||
// https://protobuf.dev/programming-guides/encoding/#varints
|
||||
func (d *D) tryULEB128() (uint64, error) {
|
||||
var result uint64
|
||||
var shift uint
|
||||
@ -260,8 +265,8 @@ func (d *D) tryULEB128() (uint64, error) {
|
||||
if shift >= 63 && b != 0 {
|
||||
return 0, fmt.Errorf("overflow when reading unsigned leb128, shift %d >= 63", shift)
|
||||
}
|
||||
result |= (b & 0x7f) << shift
|
||||
if b&0x80 == 0 {
|
||||
result |= (b & 0b01111111) << shift
|
||||
if b&0b10000000 == 0 {
|
||||
break
|
||||
}
|
||||
shift += 7
|
||||
|
Loading…
Reference in New Issue
Block a user