wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-08-16 23:40:43 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #824 from mikez/leveldb

Browse Source 
leveldb: Add LevelDB support
This commit is contained in:
Mattias Wadman 2023-12-09 16:47:43 +01:00 committed by GitHub
commit b05c7ec54f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
52 changed files with 7193 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -102,6 +102,9 @@ ipv6_packet,
jpeg,
json,
jsonl,
[leveldb_descriptor](doc/formats.md#leveldb_descriptor),
[leveldb_log](doc/formats.md#leveldb_log),
[leveldb_table](doc/formats.md#leveldb_table),
[luajit](doc/formats.md#luajit),
[macho](doc/formats.md#macho),
macho_fat,

316
doc/formats.md
View File

@ -2,138 +2,141 @@
[fq -rn -L . 'include "formats"; formats_table']: sh-start
|Name |Description |Dependencies|
|- |- |-|
|[`aac_frame`](#aac_frame) |Advanced&nbsp;Audio&nbsp;Coding&nbsp;frame |<sub></sub>|
|`adts` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream |<sub>`adts_frame`</sub>|
|`adts_frame` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream&nbsp;frame |<sub>`aac_frame`</sub>|
|`aiff` |Audio&nbsp;Interchange&nbsp;File&nbsp;Format |<sub></sub>|
|`amf0` |Action&nbsp;Message&nbsp;Format&nbsp;0 |<sub></sub>|
|`apev2` |APEv2&nbsp;metadata&nbsp;tag |<sub>`image`</sub>|
|[`apple_bookmark`](#apple_bookmark) |Apple&nbsp;BookmarkData |<sub></sub>|
|`ar` |Unix&nbsp;archive |<sub>`probe`</sub>|
|[`asn1_ber`](#asn1_ber) |ASN1&nbsp;BER&nbsp;(basic&nbsp;encoding&nbsp;rules,&nbsp;also&nbsp;CER&nbsp;and&nbsp;DER) |<sub></sub>|
|`av1_ccr` |AV1&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|`av1_frame` |AV1&nbsp;frame |<sub>`av1_obu`</sub>|
|`av1_obu` |AV1&nbsp;Open&nbsp;Bitstream&nbsp;Unit |<sub></sub>|
|`avc_annexb` |H.264/AVC&nbsp;Annex&nbsp;B |<sub>`avc_nalu`</sub>|
|[`avc_au`](#avc_au) |H.264/AVC&nbsp;Access&nbsp;Unit |<sub>`avc_nalu`</sub>|
|`avc_dcr` |H.264/AVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`avc_nalu`</sub>|
|`avc_nalu` |H.264/AVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`avc_sps` `avc_pps` `avc_sei`</sub>|
|`avc_pps` |H.264/AVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`avc_sei` |H.264/AVC&nbsp;Supplemental&nbsp;Enhancement&nbsp;Information |<sub></sub>|
|`avc_sps` |H.264/AVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`avi`](#avi) |Audio&nbsp;Video&nbsp;Interleaved |<sub>`avc_au` `hevc_au` `mp3_frame` `flac_frame`</sub>|
|[`avro_ocf`](#avro_ocf) |Avro&nbsp;object&nbsp;container&nbsp;file |<sub></sub>|
|[`bencode`](#bencode) |BitTorrent&nbsp;bencoding |<sub></sub>|
|`bitcoin_blkdat` |Bitcoin&nbsp;blk.dat |<sub>`bitcoin_block`</sub>|
|[`bitcoin_block`](#bitcoin_block) |Bitcoin&nbsp;block |<sub>`bitcoin_transaction`</sub>|
|`bitcoin_script` |Bitcoin&nbsp;script |<sub></sub>|
|`bitcoin_transaction` |Bitcoin&nbsp;transaction |<sub>`bitcoin_script`</sub>|
|[`bits`](#bits) |Raw&nbsp;bits |<sub></sub>|
|[`bplist`](#bplist) |Apple&nbsp;Binary&nbsp;Property&nbsp;List |<sub></sub>|
|`bsd_loopback_frame` |BSD&nbsp;loopback&nbsp;frame |<sub>`inet_packet`</sub>|
|[`bson`](#bson) |Binary&nbsp;JSON |<sub></sub>|
|[`bytes`](#bytes) |Raw&nbsp;bytes |<sub></sub>|
|`bzip2` |bzip2&nbsp;compression |<sub>`probe`</sub>|
|[`caff`](#caff) |Live2D&nbsp;Cubism&nbsp;archive |<sub>`probe`</sub>|
|[`cbor`](#cbor) |Concise&nbsp;Binary&nbsp;Object&nbsp;Representation |<sub></sub>|
|[`csv`](#csv) |Comma&nbsp;separated&nbsp;values |<sub></sub>|
|`dns` |DNS&nbsp;packet |<sub></sub>|
|`dns_tcp` |DNS&nbsp;packet&nbsp;(TCP) |<sub></sub>|
|`elf` |Executable&nbsp;and&nbsp;Linkable&nbsp;Format |<sub></sub>|
|`ether8023_frame` |Ethernet&nbsp;802.3&nbsp;frame |<sub>`inet_packet`</sub>|
|`exif` |Exchangeable&nbsp;Image&nbsp;File&nbsp;Format |<sub></sub>|
|`fairplay_spc` |FairPlay&nbsp;Server&nbsp;Playback&nbsp;Context |<sub></sub>|
|`flac` |Free&nbsp;Lossless&nbsp;Audio&nbsp;Codec&nbsp;file |<sub>`flac_metadatablocks` `flac_frame`</sub>|
|[`flac_frame`](#flac_frame) |FLAC&nbsp;frame |<sub></sub>|
|`flac_metadatablock` |FLAC&nbsp;metadatablock |<sub>`flac_streaminfo` `flac_picture` `vorbis_comment`</sub>|
|`flac_metadatablocks` |FLAC&nbsp;metadatablocks |<sub>`flac_metadatablock`</sub>|
|`flac_picture` |FLAC&nbsp;metadatablock&nbsp;picture |<sub>`image`</sub>|
|`flac_streaminfo` |FLAC&nbsp;streaminfo |<sub></sub>|
|`gif` |Graphics&nbsp;Interchange&nbsp;Format |<sub></sub>|
|`gzip` |gzip&nbsp;compression |<sub>`probe`</sub>|
|`hevc_annexb` |H.265/HEVC&nbsp;Annex&nbsp;B |<sub>`hevc_nalu`</sub>|
|[`hevc_au`](#hevc_au) |H.265/HEVC&nbsp;Access&nbsp;Unit |<sub>`hevc_nalu`</sub>|
|`hevc_dcr` |H.265/HEVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`hevc_nalu`</sub>|
|`hevc_nalu` |H.265/HEVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`hevc_vps` `hevc_pps` `hevc_sps`</sub>|
|`hevc_pps` |H.265/HEVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_sps` |H.265/HEVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_vps` |H.265/HEVC&nbsp;Video&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`html`](#html) |HyperText&nbsp;Markup&nbsp;Language |<sub></sub>|
|`icc_profile` |International&nbsp;Color&nbsp;Consortium&nbsp;profile |<sub></sub>|
|`icmp` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol |<sub></sub>|
|`icmpv6` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol&nbsp;v6 |<sub></sub>|
|`id3v1` |ID3v1&nbsp;metadata |<sub></sub>|
|`id3v11` |ID3v1.1&nbsp;metadata |<sub></sub>|
|`id3v2` |ID3v2&nbsp;metadata |<sub>`image`</sub>|
|`ipv4_packet` |Internet&nbsp;protocol&nbsp;v4&nbsp;packet |<sub>`ip_packet`</sub>|
|`ipv6_packet` |Internet&nbsp;protocol&nbsp;v6&nbsp;packet |<sub>`ip_packet`</sub>|
|`jpeg` |Joint&nbsp;Photographic&nbsp;Experts&nbsp;Group&nbsp;file |<sub>`exif` `icc_profile`</sub>|
|`json` |JavaScript&nbsp;Object&nbsp;Notation |<sub></sub>|
|`jsonl` |JavaScript&nbsp;Object&nbsp;Notation&nbsp;Lines |<sub></sub>|
|[`luajit`](#luajit) |LuaJIT&nbsp;2.0&nbsp;bytecode |<sub></sub>|
|[`macho`](#macho) |Mach-O&nbsp;macOS&nbsp;executable |<sub></sub>|
|`macho_fat` |Fat&nbsp;Mach-O&nbsp;macOS&nbsp;executable&nbsp;(multi-architecture) |<sub>`macho`</sub>|
|[`markdown`](#markdown) |Markdown |<sub></sub>|
|[`matroska`](#matroska) |Matroska&nbsp;file |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `image` `mp3_frame` `mpeg_asc` `mpeg_pes_packet` `mpeg_spu` `opus_packet` `vorbis_packet` `vp8_frame` `vp9_cfm` `vp9_frame`</sub>|
|[`moc3`](#moc3) |MOC3&nbsp;file |<sub></sub>|
|[`mp3`](#mp3) |MP3&nbsp;file |<sub>`id3v2` `id3v1` `id3v11` `apev2` `mp3_frame`</sub>|
|`mp3_frame` |MPEG&nbsp;audio&nbsp;layer&nbsp;3&nbsp;frame |<sub>`mp3_frame_tags`</sub>|
|`mp3_frame_vbri` |MP3&nbsp;frame&nbsp;Fraunhofer&nbsp;encoder&nbsp;variable&nbsp;bitrate&nbsp;tag |<sub></sub>|
|`mp3_frame_xing` |MP3&nbsp;frame&nbsp;Xing/Info&nbsp;tag |<sub></sub>|
|[`mp4`](#mp4) |ISOBMFF,&nbsp;QuickTime&nbsp;and&nbsp;similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `icc_profile` `id3v2` `image` `jpeg` `mp3_frame` `mpeg_es` `mpeg_pes_packet` `opus_packet` `png` `prores_frame` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr`</sub>|
|`mpeg_asc` |MPEG-4&nbsp;Audio&nbsp;Specific&nbsp;Config |<sub></sub>|
|`mpeg_es` |MPEG&nbsp;Elementary&nbsp;Stream |<sub>`mpeg_asc` `vorbis_packet`</sub>|
|`mpeg_pes` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream |<sub>`mpeg_pes_packet` `mpeg_spu`</sub>|
|`mpeg_pes_packet` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream&nbsp;packet |<sub></sub>|
|`mpeg_spu` |Sub&nbsp;Picture&nbsp;Unit&nbsp;(DVD&nbsp;subtitle) |<sub></sub>|
|`mpeg_ts` |MPEG&nbsp;Transport&nbsp;Stream |<sub></sub>|
|[`msgpack`](#msgpack) |MessagePack |<sub></sub>|
|`ogg` |OGG&nbsp;file |<sub>`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`</sub>|
|`ogg_page` |OGG&nbsp;page |<sub></sub>|
|[`opentimestamps`](#opentimestamps) |OpenTimestamps&nbsp;file |<sub></sub>|
|`opus_packet` |Opus&nbsp;packet |<sub>`vorbis_comment`</sub>|
|[`pcap`](#pcap) |PCAP&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|`pcapng` |PCAPNG&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|[`pg_btree`](#pg_btree) |PostgreSQL&nbsp;btree&nbsp;index&nbsp;file |<sub></sub>|
|[`pg_control`](#pg_control) |PostgreSQL&nbsp;control&nbsp;file |<sub></sub>|
|[`pg_heap`](#pg_heap) |PostgreSQL&nbsp;heap&nbsp;file |<sub></sub>|
|`png` |Portable&nbsp;Network&nbsp;Graphics&nbsp;file |<sub>`icc_profile` `exif`</sub>|
|`prores_frame` |Apple&nbsp;ProRes&nbsp;frame |<sub></sub>|
|[`protobuf`](#protobuf) |Protobuf |<sub></sub>|
|`protobuf_widevine` |Widevine&nbsp;protobuf |<sub>`protobuf`</sub>|
|`pssh_playready` |PlayReady&nbsp;PSSH |<sub></sub>|
|[`rtmp`](#rtmp) |Real-Time&nbsp;Messaging&nbsp;Protocol |<sub>`amf0` `mpeg_asc`</sub>|
|`sll2_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation&nbsp;v2 |<sub>`inet_packet`</sub>|
|`sll_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation |<sub>`inet_packet`</sub>|
|`tar` |Tar&nbsp;archive |<sub>`probe`</sub>|
|`tcp_segment` |Transmission&nbsp;control&nbsp;protocol&nbsp;segment |<sub></sub>|
|`tiff` |Tag&nbsp;Image&nbsp;File&nbsp;Format |<sub>`icc_profile`</sub>|
|[`tls`](#tls) |Transport&nbsp;layer&nbsp;security |<sub>`asn1_ber`</sub>|
|`toml` |Tom's&nbsp;Obvious,&nbsp;Minimal&nbsp;Language |<sub></sub>|
|[`tzif`](#tzif) |Time&nbsp;Zone&nbsp;Information&nbsp;Format |<sub></sub>|
|`udp_datagram` |User&nbsp;datagram&nbsp;protocol |<sub>`udp_payload`</sub>|
|`vorbis_comment` |Vorbis&nbsp;comment |<sub>`flac_picture`</sub>|
|`vorbis_packet` |Vorbis&nbsp;packet |<sub>`vorbis_comment`</sub>|
|`vp8_frame` |VP8&nbsp;frame |<sub></sub>|
|`vp9_cfm` |VP9&nbsp;Codec&nbsp;Feature&nbsp;Metadata |<sub></sub>|
|`vp9_frame` |VP9&nbsp;frame |<sub></sub>|
|`vpx_ccr` |VPX&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|[`wasm`](#wasm) |WebAssembly&nbsp;Binary&nbsp;Format |<sub></sub>|
|`wav` |WAV&nbsp;file |<sub>`id3v2` `id3v1` `id3v11`</sub>|
|`webp` |WebP&nbsp;image |<sub>`exif` `vp8_frame` `icc_profile` `xml`</sub>|
|[`xml`](#xml) |Extensible&nbsp;Markup&nbsp;Language |<sub></sub>|
|`yaml` |YAML&nbsp;Ain't&nbsp;Markup&nbsp;Language |<sub></sub>|
|[`zip`](#zip) |ZIP&nbsp;archive |<sub>`probe`</sub>|
|`image` |Group |<sub>`gif` `jpeg` `mp4` `png` `tiff` `webp`</sub>|
|`inet_packet` |Group |<sub>`ipv4_packet` `ipv6_packet`</sub>|
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `ipv4_packet` `ipv6_packet` `sll2_packet` `sll_packet`</sub>|
|`mp3_frame_tags` |Group |<sub>`mp3_frame_vbri` `mp3_frame_xing`</sub>|
|`probe` |Group |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `flac` `gif` `gzip` `html` `jpeg` `json` `jsonl` `luajit` `macho` `macho_fat` `matroska` `moc3` `mp3` `mp4` `mpeg_ts` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|`tcp_stream` |Group |<sub>`dns_tcp` `rtmp` `tls`</sub>|
|`udp_payload` |Group |<sub>`dns`</sub>|
|Name |Description |Dependencies|
|- |- |-|
|[`aac_frame`](#aac_frame) |Advanced&nbsp;Audio&nbsp;Coding&nbsp;frame |<sub></sub>|
|`adts` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream |<sub>`adts_frame`</sub>|
|`adts_frame` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream&nbsp;frame |<sub>`aac_frame`</sub>|
|`aiff` |Audio&nbsp;Interchange&nbsp;File&nbsp;Format |<sub></sub>|
|`amf0` |Action&nbsp;Message&nbsp;Format&nbsp;0 |<sub></sub>|
|`apev2` |APEv2&nbsp;metadata&nbsp;tag |<sub>`image`</sub>|
|[`apple_bookmark`](#apple_bookmark) |Apple&nbsp;BookmarkData |<sub></sub>|
|`ar` |Unix&nbsp;archive |<sub>`probe`</sub>|
|[`asn1_ber`](#asn1_ber) |ASN1&nbsp;BER&nbsp;(basic&nbsp;encoding&nbsp;rules,&nbsp;also&nbsp;CER&nbsp;and&nbsp;DER) |<sub></sub>|
|`av1_ccr` |AV1&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|`av1_frame` |AV1&nbsp;frame |<sub>`av1_obu`</sub>|
|`av1_obu` |AV1&nbsp;Open&nbsp;Bitstream&nbsp;Unit |<sub></sub>|
|`avc_annexb` |H.264/AVC&nbsp;Annex&nbsp;B |<sub>`avc_nalu`</sub>|
|[`avc_au`](#avc_au) |H.264/AVC&nbsp;Access&nbsp;Unit |<sub>`avc_nalu`</sub>|
|`avc_dcr` |H.264/AVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`avc_nalu`</sub>|
|`avc_nalu` |H.264/AVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`avc_sps` `avc_pps` `avc_sei`</sub>|
|`avc_pps` |H.264/AVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`avc_sei` |H.264/AVC&nbsp;Supplemental&nbsp;Enhancement&nbsp;Information |<sub></sub>|
|`avc_sps` |H.264/AVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`avi`](#avi) |Audio&nbsp;Video&nbsp;Interleaved |<sub>`avc_au` `hevc_au` `mp3_frame` `flac_frame`</sub>|
|[`avro_ocf`](#avro_ocf) |Avro&nbsp;object&nbsp;container&nbsp;file |<sub></sub>|
|[`bencode`](#bencode) |BitTorrent&nbsp;bencoding |<sub></sub>|
|`bitcoin_blkdat` |Bitcoin&nbsp;blk.dat |<sub>`bitcoin_block`</sub>|
|[`bitcoin_block`](#bitcoin_block) |Bitcoin&nbsp;block |<sub>`bitcoin_transaction`</sub>|
|`bitcoin_script` |Bitcoin&nbsp;script |<sub></sub>|
|`bitcoin_transaction` |Bitcoin&nbsp;transaction |<sub>`bitcoin_script`</sub>|
|[`bits`](#bits) |Raw&nbsp;bits |<sub></sub>|
|[`bplist`](#bplist) |Apple&nbsp;Binary&nbsp;Property&nbsp;List |<sub></sub>|
|`bsd_loopback_frame` |BSD&nbsp;loopback&nbsp;frame |<sub>`inet_packet`</sub>|
|[`bson`](#bson) |Binary&nbsp;JSON |<sub></sub>|
|[`bytes`](#bytes) |Raw&nbsp;bytes |<sub></sub>|
|`bzip2` |bzip2&nbsp;compression |<sub>`probe`</sub>|
|[`caff`](#caff) |Live2D&nbsp;Cubism&nbsp;archive |<sub>`probe`</sub>|
|[`cbor`](#cbor) |Concise&nbsp;Binary&nbsp;Object&nbsp;Representation |<sub></sub>|
|[`csv`](#csv) |Comma&nbsp;separated&nbsp;values |<sub></sub>|
|`dns` |DNS&nbsp;packet |<sub></sub>|
|`dns_tcp` |DNS&nbsp;packet&nbsp;(TCP) |<sub></sub>|
|`elf` |Executable&nbsp;and&nbsp;Linkable&nbsp;Format |<sub></sub>|
|`ether8023_frame` |Ethernet&nbsp;802.3&nbsp;frame |<sub>`inet_packet`</sub>|
|`exif` |Exchangeable&nbsp;Image&nbsp;File&nbsp;Format |<sub></sub>|
|`fairplay_spc` |FairPlay&nbsp;Server&nbsp;Playback&nbsp;Context |<sub></sub>|
|`flac` |Free&nbsp;Lossless&nbsp;Audio&nbsp;Codec&nbsp;file |<sub>`flac_metadatablocks` `flac_frame`</sub>|
|[`flac_frame`](#flac_frame) |FLAC&nbsp;frame |<sub></sub>|
|`flac_metadatablock` |FLAC&nbsp;metadatablock |<sub>`flac_streaminfo` `flac_picture` `vorbis_comment`</sub>|
|`flac_metadatablocks` |FLAC&nbsp;metadatablocks |<sub>`flac_metadatablock`</sub>|
|`flac_picture` |FLAC&nbsp;metadatablock&nbsp;picture |<sub>`image`</sub>|
|`flac_streaminfo` |FLAC&nbsp;streaminfo |<sub></sub>|
|`gif` |Graphics&nbsp;Interchange&nbsp;Format |<sub></sub>|
|`gzip` |gzip&nbsp;compression |<sub>`probe`</sub>|
|`hevc_annexb` |H.265/HEVC&nbsp;Annex&nbsp;B |<sub>`hevc_nalu`</sub>|
|[`hevc_au`](#hevc_au) |H.265/HEVC&nbsp;Access&nbsp;Unit |<sub>`hevc_nalu`</sub>|
|`hevc_dcr` |H.265/HEVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`hevc_nalu`</sub>|
|`hevc_nalu` |H.265/HEVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`hevc_vps` `hevc_pps` `hevc_sps`</sub>|
|`hevc_pps` |H.265/HEVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_sps` |H.265/HEVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_vps` |H.265/HEVC&nbsp;Video&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`html`](#html) |HyperText&nbsp;Markup&nbsp;Language |<sub></sub>|
|`icc_profile` |International&nbsp;Color&nbsp;Consortium&nbsp;profile |<sub></sub>|
|`icmp` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol |<sub></sub>|
|`icmpv6` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol&nbsp;v6 |<sub></sub>|
|`id3v1` |ID3v1&nbsp;metadata |<sub></sub>|
|`id3v11` |ID3v1.1&nbsp;metadata |<sub></sub>|
|`id3v2` |ID3v2&nbsp;metadata |<sub>`image`</sub>|
|`ipv4_packet` |Internet&nbsp;protocol&nbsp;v4&nbsp;packet |<sub>`ip_packet`</sub>|
|`ipv6_packet` |Internet&nbsp;protocol&nbsp;v6&nbsp;packet |<sub>`ip_packet`</sub>|
|`jpeg` |Joint&nbsp;Photographic&nbsp;Experts&nbsp;Group&nbsp;file |<sub>`exif` `icc_profile`</sub>|
|`json` |JavaScript&nbsp;Object&nbsp;Notation |<sub></sub>|
|`jsonl` |JavaScript&nbsp;Object&nbsp;Notation&nbsp;Lines |<sub></sub>|
|[`leveldb_descriptor`](#leveldb_descriptor) |LevelDB&nbsp;Descriptor |<sub></sub>|
|[`leveldb_log`](#leveldb_log) |LevelDB&nbsp;Log |<sub></sub>|
|[`leveldb_table`](#leveldb_table) |LevelDB&nbsp;Table |<sub></sub>|
|[`luajit`](#luajit) |LuaJIT&nbsp;2.0&nbsp;bytecode |<sub></sub>|
|[`macho`](#macho) |Mach-O&nbsp;macOS&nbsp;executable |<sub></sub>|
|`macho_fat` |Fat&nbsp;Mach-O&nbsp;macOS&nbsp;executable&nbsp;(multi-architecture) |<sub>`macho`</sub>|
|[`markdown`](#markdown) |Markdown |<sub></sub>|
|[`matroska`](#matroska) |Matroska&nbsp;file |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `image` `mp3_frame` `mpeg_asc` `mpeg_pes_packet` `mpeg_spu` `opus_packet` `vorbis_packet` `vp8_frame` `vp9_cfm` `vp9_frame`</sub>|
|[`moc3`](#moc3) |MOC3&nbsp;file |<sub></sub>|
|[`mp3`](#mp3) |MP3&nbsp;file |<sub>`id3v2` `id3v1` `id3v11` `apev2` `mp3_frame`</sub>|
|`mp3_frame` |MPEG&nbsp;audio&nbsp;layer&nbsp;3&nbsp;frame |<sub>`mp3_frame_tags`</sub>|
|`mp3_frame_vbri` |MP3&nbsp;frame&nbsp;Fraunhofer&nbsp;encoder&nbsp;variable&nbsp;bitrate&nbsp;tag |<sub></sub>|
|`mp3_frame_xing` |MP3&nbsp;frame&nbsp;Xing/Info&nbsp;tag |<sub></sub>|
|[`mp4`](#mp4) |ISOBMFF,&nbsp;QuickTime&nbsp;and&nbsp;similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `icc_profile` `id3v2` `image` `jpeg` `mp3_frame` `mpeg_es` `mpeg_pes_packet` `opus_packet` `png` `prores_frame` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr`</sub>|
|`mpeg_asc` |MPEG-4&nbsp;Audio&nbsp;Specific&nbsp;Config |<sub></sub>|
|`mpeg_es` |MPEG&nbsp;Elementary&nbsp;Stream |<sub>`mpeg_asc` `vorbis_packet`</sub>|
|`mpeg_pes` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream |<sub>`mpeg_pes_packet` `mpeg_spu`</sub>|
|`mpeg_pes_packet` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream&nbsp;packet |<sub></sub>|
|`mpeg_spu` |Sub&nbsp;Picture&nbsp;Unit&nbsp;(DVD&nbsp;subtitle) |<sub></sub>|
|`mpeg_ts` |MPEG&nbsp;Transport&nbsp;Stream |<sub></sub>|
|[`msgpack`](#msgpack) |MessagePack |<sub></sub>|
|`ogg` |OGG&nbsp;file |<sub>`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`</sub>|
|`ogg_page` |OGG&nbsp;page |<sub></sub>|
|[`opentimestamps`](#opentimestamps) |OpenTimestamps&nbsp;file |<sub></sub>|
|`opus_packet` |Opus&nbsp;packet |<sub>`vorbis_comment`</sub>|
|[`pcap`](#pcap) |PCAP&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|`pcapng` |PCAPNG&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|[`pg_btree`](#pg_btree) |PostgreSQL&nbsp;btree&nbsp;index&nbsp;file |<sub></sub>|
|[`pg_control`](#pg_control) |PostgreSQL&nbsp;control&nbsp;file |<sub></sub>|
|[`pg_heap`](#pg_heap) |PostgreSQL&nbsp;heap&nbsp;file |<sub></sub>|
|`png` |Portable&nbsp;Network&nbsp;Graphics&nbsp;file |<sub>`icc_profile` `exif`</sub>|
|`prores_frame` |Apple&nbsp;ProRes&nbsp;frame |<sub></sub>|
|[`protobuf`](#protobuf) |Protobuf |<sub></sub>|
|`protobuf_widevine` |Widevine&nbsp;protobuf |<sub>`protobuf`</sub>|
|`pssh_playready` |PlayReady&nbsp;PSSH |<sub></sub>|
|[`rtmp`](#rtmp) |Real-Time&nbsp;Messaging&nbsp;Protocol |<sub>`amf0` `mpeg_asc`</sub>|
|`sll2_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation&nbsp;v2 |<sub>`inet_packet`</sub>|
|`sll_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation |<sub>`inet_packet`</sub>|
|`tar` |Tar&nbsp;archive |<sub>`probe`</sub>|
|`tcp_segment` |Transmission&nbsp;control&nbsp;protocol&nbsp;segment |<sub></sub>|
|`tiff` |Tag&nbsp;Image&nbsp;File&nbsp;Format |<sub>`icc_profile`</sub>|
|[`tls`](#tls) |Transport&nbsp;layer&nbsp;security |<sub>`asn1_ber`</sub>|
|`toml` |Tom's&nbsp;Obvious,&nbsp;Minimal&nbsp;Language |<sub></sub>|
|[`tzif`](#tzif) |Time&nbsp;Zone&nbsp;Information&nbsp;Format |<sub></sub>|
|`udp_datagram` |User&nbsp;datagram&nbsp;protocol |<sub>`udp_payload`</sub>|
|`vorbis_comment` |Vorbis&nbsp;comment |<sub>`flac_picture`</sub>|
|`vorbis_packet` |Vorbis&nbsp;packet |<sub>`vorbis_comment`</sub>|
|`vp8_frame` |VP8&nbsp;frame |<sub></sub>|
|`vp9_cfm` |VP9&nbsp;Codec&nbsp;Feature&nbsp;Metadata |<sub></sub>|
|`vp9_frame` |VP9&nbsp;frame |<sub></sub>|
|`vpx_ccr` |VPX&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|[`wasm`](#wasm) |WebAssembly&nbsp;Binary&nbsp;Format |<sub></sub>|
|`wav` |WAV&nbsp;file |<sub>`id3v2` `id3v1` `id3v11`</sub>|
|`webp` |WebP&nbsp;image |<sub>`exif` `vp8_frame` `icc_profile` `xml`</sub>|
|[`xml`](#xml) |Extensible&nbsp;Markup&nbsp;Language |<sub></sub>|
|`yaml` |YAML&nbsp;Ain't&nbsp;Markup&nbsp;Language |<sub></sub>|
|[`zip`](#zip) |ZIP&nbsp;archive |<sub>`probe`</sub>|
|`image` |Group |<sub>`gif` `jpeg` `mp4` `png` `tiff` `webp`</sub>|
|`inet_packet` |Group |<sub>`ipv4_packet` `ipv6_packet`</sub>|
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `ipv4_packet` `ipv6_packet` `sll2_packet` `sll_packet`</sub>|
|`mp3_frame_tags` |Group |<sub>`mp3_frame_vbri` `mp3_frame_xing`</sub>|
|`probe` |Group |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `flac` `gif` `gzip` `html` `jpeg` `json` `jsonl` `leveldb_table` `luajit` `macho` `macho_fat` `matroska` `moc3` `mp3` `mp4` `mpeg_ts` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|`tcp_stream` |Group |<sub>`dns_tcp` `rtmp` `tls`</sub>|
|`udp_payload` |Group |<sub>`dns`</sub>|
[#]: sh-end
@ -690,6 +693,55 @@ $ fq -n -d html '[inputs | {key: input_filename, value: .html.head.title?}] | fr
$ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href' file.html
```
## leveldb_descriptor
### Limitations
- fragmented non-"full" records are not merged and decoded further.
### Authors
- [@mikez](https://github.com/mikez), original author
### References
- https://github.com/google/leveldb/blob/main/doc/impl.md#manifest
- https://github.com/google/leveldb/blob/main/doc/log_format.md
- https://github.com/google/leveldb/blob/main/db/version_edit.cc
## leveldb_log
### Limitations
- fragmented non-"full" records are not merged and decoded further.
### Authors
- [@mikez](https://github.com/mikez), original author
### References
- https://github.com/google/leveldb/blob/main/doc/impl.md#log-files
- https://github.com/google/leveldb/blob/main/doc/log_format.md
- https://github.com/google/leveldb/blob/main/db/write_batch.cc
## leveldb_table
### Limitations
- no Meta Blocks (like "filter") are decoded yet.
- Zstandard uncompression is not implemented yet.
### Authors
- [@mikez](https://github.com/mikez), original author
### References
- https://github.com/google/leveldb/blob/main/doc/table_format.md
- https://github.com/google/leveldb/blob/main/doc/impl.md
- https://github.com/google/leveldb/blob/main/doc/index.md
## luajit
### Authors

4
format/all/all.fqtest
View File

@ -14,6 +14,7 @@ $ fq -n _registry.groups.probe
"gif",
"gzip",
"jpeg",
"leveldb_table",
"luajit",
"macho",
"macho_fat",
@ -111,6 +112,9 @@ ipv6_packet Internet protocol v6 packet
jpeg Joint Photographic Experts Group file
json JavaScript Object Notation
jsonl JavaScript Object Notation Lines
leveldb_descriptor LevelDB Descriptor
leveldb_log LevelDB Log
leveldb_table LevelDB Table
luajit LuaJIT 2.0 bytecode
macho Mach-O macOS executable
macho_fat Fat Mach-O macOS executable (multi-architecture)

1
format/all/all.go
View File

@ -30,6 +30,7 @@ import (
_ "github.com/wader/fq/format/inet"
_ "github.com/wader/fq/format/jpeg"
_ "github.com/wader/fq/format/json"
_ "github.com/wader/fq/format/leveldb"
_ "github.com/wader/fq/format/luajit"
_ "github.com/wader/fq/format/markdown"
_ "github.com/wader/fq/format/math"

3
format/format.go
View File

@ -125,6 +125,9 @@ var (
JPEG = &decode.Group{Name: "jpeg"}
JSON = &decode.Group{Name: "json"}
JSONL = &decode.Group{Name: "jsonl"}
LevelDB_Descriptor = &decode.Group{Name: "leveldb_descriptor"}
LevelDB_LDB = &decode.Group{Name: "leveldb_table"}
LevelDB_LOG = &decode.Group{Name: "leveldb_log"}
LuaJIT = &decode.Group{Name: "luajit"}
MachO = &decode.Group{Name: "macho"}
MachO_Fat = &decode.Group{Name: "macho_fat"}

121
format/leveldb/leveldb_descriptor.go Normal file
View File

@ -0,0 +1,121 @@
package leveldb
// https://github.com/google/leveldb/blob/main/doc/impl.md#manifest
// https://github.com/google/leveldb/blob/main/db/version_edit.cc
//
// Files in LevelDB using this format include:
// - MANIFEST-*
import (
"embed"
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
)
//go:embed leveldb_descriptor.jq
//go:embed leveldb_descriptor.md
var leveldbDescriptorFS embed.FS
func init() {
interp.RegisterFormat(
format.LevelDB_Descriptor,
&decode.Format{
Description: "LevelDB Descriptor",
DecodeFn: ldbDescriptorDecode,
Functions: []string{"torepr"},
})
interp.RegisterFS(leveldbDescriptorFS)
}
const (
tagTypeComparator = 1
tagTypeLogNumber = 2
tagTypeNextFileNumber = 3
tagTypeLastSequence = 4
tagTypeCompactPointer = 5
tagTypeDeletedFile = 6
tagTypeNewFile = 7
// 8 not used anymore
tagTypePrevLogNumber = 9
)
var tagTypes = scalar.UintMapSymStr{
tagTypeComparator: "comparator",
tagTypeLogNumber: "log_number",
tagTypeNextFileNumber: "next_file_number",
tagTypeLastSequence: "last_sequence",
tagTypeCompactPointer: "compact_pointer",
tagTypeDeletedFile: "deleted_file",
tagTypeNewFile: "new_file",
tagTypePrevLogNumber: "previous_log_number",
}
func ldbDescriptorDecode(d *decode.D) any {
rro := recordReadOptions{readDataFn: func(size int64, recordType int, d *decode.D) {
if recordType == recordTypeFull {
d.FieldStruct("data", func(d *decode.D) {
d.LimitedFn(size, readManifest)
})
} else {
d.FieldRawLen("data", size)
}
}}
readBlockSequence(rro, d)
return nil
}
// List of sorted tables for each level involving key ranges and other metadata.
func readManifest(d *decode.D) {
d.FieldArray("tags", func(d *decode.D) {
for !d.End() {
d.FieldStruct("tag", func(d *decode.D) {
tag := d.FieldULEB128("key", tagTypes)
switch tag {
case tagTypeComparator:
readLengthPrefixedString("value", d)
case tagTypeLogNumber,
tagTypePrevLogNumber,
tagTypeNextFileNumber,
tagTypeLastSequence:
d.FieldULEB128("value")
case tagTypeCompactPointer:
d.FieldStruct("value", func(d *decode.D) {
d.FieldULEB128("level")
readTagInternalKey("internal_key", d)
})
case tagTypeDeletedFile:
d.FieldStruct("value", func(d *decode.D) {
d.FieldULEB128("level")
d.FieldULEB128("file_number")
})
case tagTypeNewFile:
d.FieldStruct("value", func(d *decode.D) {
d.FieldULEB128("level")
d.FieldULEB128("file_number")
d.FieldULEB128("file_size")
readTagInternalKey("smallest_internal_key", d)
readTagInternalKey("largest_internal_key", d)
})
default:
d.Fatalf("unknown tag: %d", tag)
}
})
}
})
}
func readTagInternalKey(name string, d *decode.D) {
d.FieldStruct(name, func(d *decode.D) {
length := d.FieldULEB128("length")
d.FieldStruct("data", func(d *decode.D) {
err := readInternalKey(nil, int(length), d)
if err != nil {
d.Errorf("%v", err)
}
})
})
}

37
format/leveldb/leveldb_descriptor.jq Normal file
View File

@ -0,0 +1,37 @@
# TK(2023-12-07): in the output, object-properties are
# sorted alphabetically... how can one prevent this
# and keep the original order?
def _leveldb_descriptor_torepr:
def _f:
if .type == "root" then
[ .value.blocks[].records[]
| {type: "record", value: .}
| _f
]
elif .type == "record" then
if .value.header.record_type != "full" then
empty
else
[ .value.data.tags[]
| {type: "tag", value: {(.key): .value}}
| _f
]
end
elif .type == "tag" then
( .value
| if .comparator then .comparator |= .data else . end
| if .new_file then
( .new_file.largest_internal_key |= .data
| .new_file.smallest_internal_key |= .data
)
else .
end
| if .compact_pointer then
.compact_pointer.internal_key |= .data
else .
end
)
end;
( {type: "root", value: .}
| _f
);

13
format/leveldb/leveldb_descriptor.md Normal file
View File

@ -0,0 +1,13 @@
### Limitations
- fragmented non-"full" records are not merged and decoded further.
### Authors
- [@mikez](https://github.com/mikez), original author
### References
- https://github.com/google/leveldb/blob/main/doc/impl.md#manifest
- https://github.com/google/leveldb/blob/main/doc/log_format.md
- https://github.com/google/leveldb/blob/main/db/version_edit.cc

86
format/leveldb/leveldb_log.go Normal file
View File

@ -0,0 +1,86 @@
package leveldb
// https://github.com/google/leveldb/blob/main/doc/impl.md#log-files
// https://github.com/google/leveldb/blob/main/db/write_batch.cc
//
// Files in LevelDB using this format include:
// - *.log
import (
"embed"
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
)
//go:embed leveldb_log.md
var leveldbLogFS embed.FS
func init() {
interp.RegisterFormat(
format.LevelDB_LOG,
&decode.Format{
Description: "LevelDB Log",
DecodeFn: ldbLogDecode,
})
interp.RegisterFS(leveldbLogFS)
}
func ldbLogDecode(d *decode.D) any {
rro := recordReadOptions{readDataFn: func(size int64, recordType int, d *decode.D) {
if recordType == recordTypeFull {
d.FieldStruct("data", func(d *decode.D) {
d.LimitedFn(size, readBatch)
})
} else {
d.FieldRawLen("data", size)
}
}}
readBlockSequence(rro, d)
return nil
}
// https://github.com/google/leveldb/blob/main/db/write_batch.cc#L5-L14
//
// WriteBatch::rep_ :=
//
// sequence: fixed64
// count: fixed32
// data: record[count]
//
// record :=
//
// kTypeValue varstring varstring
// kTypeDeletion varstring
//
// varstring :=
//
// len: varint32
// data: uint8[len]
func readBatch(d *decode.D) {
d.FieldU64("sequence")
expectedCount := d.FieldU32("count")
actualCount := uint64(0)
d.FieldArray("records", func(d *decode.D) {
for !d.End() {
d.FieldStruct("record", func(d *decode.D) {
valueType := d.FieldULEB128("type", valueTypes)
switch valueType {
case valueTypeDeletion:
readLengthPrefixedString("key", d)
case valueTypeValue:
readLengthPrefixedString("key", d)
readLengthPrefixedString("value", d)
default:
d.Fatalf("unknown value type: %d", valueType)
}
})
actualCount++
}
})
if actualCount != expectedCount {
d.Errorf("actual record count (%d) does not equal expected count (%d)", actualCount, expectedCount)
}
}

13
format/leveldb/leveldb_log.md Normal file
View File

@ -0,0 +1,13 @@
### Limitations
- fragmented non-"full" records are not merged and decoded further.
### Authors
- [@mikez](https://github.com/mikez), original author
### References
- https://github.com/google/leveldb/blob/main/doc/impl.md#log-files
- https://github.com/google/leveldb/blob/main/doc/log_format.md
- https://github.com/google/leveldb/blob/main/db/write_batch.cc

133
format/leveldb/leveldb_log_blocks.go Normal file
View File

@ -0,0 +1,133 @@
package leveldb
// https://github.com/google/leveldb/blob/main/doc/log_format.md
//
// Files in LevelDB using the "log-format" of block sequences include:
// - *.log
// - MANIFEST-*
import (
"github.com/wader/fq/internal/mathex"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
)
type recordReadOptions struct {
// Both .log- and MANIFEST-files use the Log-format,
// i.e., a sequence of records split into 32KB blocks.
// However, the format of the data within the records differ.
// This function specifies how to read said data.
readDataFn func(size int64, recordType int, d *decode.D)
}
// https://github.com/google/leveldb/blob/main/db/log_format.h
const (
// checksum (4 bytes) + length (2 bytes) + record type (1 byte)
headerSize = (4 + 2 + 1) * 8
blockSize = (32 * 1024) * 8 // 32KB
recordTypeZero = 0 // preallocated file regions
recordTypeFull = 1
recordTypeFirst = 2 // fragments
recordTypeMiddle = 3
recordTypeLast = 4
)
var recordTypes = scalar.UintMapSymStr{
recordTypeZero: "zero",
recordTypeFull: "full",
recordTypeFirst: "first",
recordTypeMiddle: "middle",
recordTypeLast: "last",
}
// Read a sequence of 32KB-blocks (the last one may be less).
// https://github.com/google/leveldb/blob/main/db/log_reader.cc#L189
func readBlockSequence(rro recordReadOptions, d *decode.D) {
d.Endian = decode.LittleEndian
d.FieldArray("blocks", func(d *decode.D) {
for d.BitsLeft() >= headerSize {
d.LimitedFn(mathex.Min(blockSize, d.BitsLeft()), func(d *decode.D) {
d.FieldStruct("block", bind(readLogBlock, rro))
})
}
})
if d.BitsLeft() > 0 {
// The reference implementation says:
// "[...] if buffer_ is non-empty, we have a truncated header at the
// end of the file, which can be caused by the writer crashing in the
// middle of writing the header. Instead of considering this an error,
// just report EOF."
d.FieldRawLen("truncated_block", d.BitsLeft())
}
}
// Read a Log-block, consisting of up to 32KB of records and an optional trailer.
//
// block := record* trailer?
func readLogBlock(rro recordReadOptions, d *decode.D) {
if d.BitsLeft() > blockSize {
d.Fatalf("Bits left greater than maximum log-block size of 32KB.")
}
// record*
d.FieldArray("records", func(d *decode.D) {
for d.BitsLeft() >= headerSize {
d.FieldStruct("record", bind(readLogRecord, rro))
}
})
// trailer?
if d.BitsLeft() > 0 {
d.FieldRawLen("trailer", d.BitsLeft())
}
}
// Read a Log-record.
//
// checksum: uint32 // crc32c of type and data[] ; little-endian
// length: uint16 // little-endian
// type: uint8 // One of FULL, FIRST, MIDDLE, LAST
// data: uint8[length]
//
// via https://github.com/google/leveldb/blob/main/doc/log_format.md
func readLogRecord(rro recordReadOptions, d *decode.D) {
// header
var checksumValue *decode.Value
var length int64
var recordType int
d.LimitedFn(headerSize, func(d *decode.D) {
d.FieldStruct("header", func(d *decode.D) {
d.FieldU32("checksum", scalar.UintHex)
checksumValue = d.FieldGet("checksum")
length = int64(d.FieldU16("length"))
recordType = int(d.FieldU8("record_type", recordTypes))
})
})
// verify checksum: record type (1 byte) + data (`length` bytes)
d.RangeFn(d.Pos()-8, (1+length)*8, func(d *decode.D) {
bytesToCheck := d.Bits(int(d.BitsLeft()))
actualChecksum := computeChecksum(bytesToCheck)
_ = checksumValue.TryUintScalarFn(d.UintAssert(uint64(actualChecksum)))
})
// data
dataSize := length * 8
rro.readDataFn(dataSize, recordType, d)
}
func readLengthPrefixedString(name string, d *decode.D) {
d.FieldStruct(name, func(d *decode.D) {
length := d.FieldULEB128("length")
d.FieldUTF8("data", int(length))
})
}
// simplified `functools.partial` (Python) or `Function.prototype.bind` (JavaScript)
func bind(f func(recordReadOptions, *decode.D), rro recordReadOptions) func(*decode.D) {
return func(d *decode.D) {
f(rro, d)
}
}

473
format/leveldb/leveldb_table.go Normal file
View File

@ -0,0 +1,473 @@
package leveldb
// https://github.com/google/leveldb/blob/main/doc/table_format.md
// https://github.com/google/leveldb/blob/main/doc/impl.md
// https://github.com/google/leveldb/blob/main/doc/index.md
//
// Files in LevelDB using this format include:
// - *.ldb
import (
"bytes"
"embed"
"fmt"
"hash/crc32"
"github.com/golang/snappy"
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
)
//go:embed leveldb_table.md
var leveldbTableFS embed.FS
func init() {
interp.RegisterFormat(
format.LevelDB_LDB,
&decode.Format{
Description: "LevelDB Table",
Groups: []*decode.Group{format.Probe},
DecodeFn: ldbTableDecode,
})
interp.RegisterFS(leveldbTableFS)
}
const (
// four varints (each max 10 bytes) + magic number (8 bytes)
// https://github.com/google/leveldb/blob/main/table/format.h#L53
footerEncodedLength = (4*10 + 8) * 8
magicNumberLength = 8 * 8
// leading 64 bits of
// echo http://code.google.com/p/leveldb/ | sha1sum
// https://github.com/google/leveldb/blob/main/table/format.h#L76
tableMagicNumber = 0xdb4775248b80fb57
uint32BitSize = 32
)
// https://github.com/google/leveldb/blob/main/include/leveldb/options.h#L25
const (
compressionTypeNone = 0x0
compressionTypeSnappy = 0x1
compressionTypeZstandard = 0x2
)
var compressionTypes = scalar.UintMapSymStr{
compressionTypeNone: "none",
compressionTypeSnappy: "snappy",
compressionTypeZstandard: "zstd",
}
// https://github.com/google/leveldb/blob/main/db/dbformat.h#L54
const (
valueTypeDeletion = 0x0
valueTypeValue = 0x1
)
var valueTypes = scalar.UintMapSymStr{
valueTypeDeletion: "deletion",
valueTypeValue: "value",
}
type blockHandle struct {
offset uint64
size uint64
}
func ldbTableDecode(d *decode.D) any {
d.Endian = decode.LittleEndian
// footer
var indexOffset int64
var indexSize int64
var metaIndexOffset int64
var metaIndexSize int64
d.FieldStruct("footer", func(d *decode.D) {
// check for magic number and fail fast if it isn't there
d.SeekAbs(d.Len() - magicNumberLength)
d.FieldU64("magic_number", d.UintAssert(tableMagicNumber), scalar.UintHex)
d.SeekAbs(d.Len() - footerEncodedLength)
d.LimitedFn(footerEncodedLength-magicNumberLength, func(d *decode.D) {
d.FieldStruct("metaindex_handle", func(d *decode.D) {
metaIndexOffset = int64(d.FieldULEB128("offset"))
metaIndexSize = int64(d.FieldULEB128("size"))
})
d.FieldStruct("index_handle", func(d *decode.D) {
indexOffset = int64(d.FieldULEB128("offset"))
indexSize = int64(d.FieldULEB128("size"))
})
d.FieldRawLen("padding", d.BitsLeft())
})
})
// metaindex
d.SeekAbs(metaIndexOffset * 8)
var metaHandles []blockHandle
readTableBlock(
"metaindex",
metaIndexSize,
keyValueContentsReader(
nil,
func(d *decode.D) {
handle := readBlockHandle(d)
metaHandles = append(metaHandles, handle)
},
),
d,
)
// index
d.SeekAbs(indexOffset * 8)
var dataHandles []blockHandle
readTableBlock(
"index",
indexSize,
keyValueContentsReader(
readInternalKey,
func(d *decode.D) {
handle := readBlockHandle(d)
dataHandles = append(dataHandles, handle)
},
),
d,
)
// meta
if len(metaHandles) > 0 {
d.FieldArray("meta", func(d *decode.D) {
for _, handle := range metaHandles {
d.SeekAbs(int64(handle.offset) * 8)
readTableBlock("meta_block", int64(handle.size), readMetaContent, d)
}
})
}
// data
if len(dataHandles) > 0 {
d.FieldArray("data", func(d *decode.D) {
for _, handle := range dataHandles {
d.SeekAbs(int64(handle.offset) * 8)
readTableBlock(
"data_block",
int64(handle.size),
keyValueContentsReader(readInternalKey, nil),
d,
)
}
})
}
return nil
}
// Readers
// Read block contents as well as compression + checksum bytes following it.
// The function `readTableBlockContents` gets the _uncompressed_ bytebuffer.
// https://github.com/google/leveldb/blob/main/table/format.cc#L69
func readTableBlock(
name string,
size int64,
readTableBlockContents func(size int64, d *decode.D),
d *decode.D,
) {
d.FieldStruct(name, func(d *decode.D) {
start := d.Pos()
br := d.RawLen(size * 8)
// compression (1 byte)
compressionType := d.FieldU8("compression", compressionTypes, scalar.UintHex)
// checksum (4 bytes)
data := d.ReadAllBits(br)
bytesToCheck := append(data, uint8(compressionType))
checksum := computeChecksum(bytesToCheck)
d.FieldU32("checksum", d.UintAssert(uint64(checksum)), scalar.UintHex)
// decompress if needed
d.SeekAbs(start)
if compressionType == compressionTypeNone {
d.FieldStruct("uncompressed", func(d *decode.D) {
readTableBlockContents(size, d)
})
} else {
compressedSize := size
compressed := data
bb := &bytes.Buffer{}
switch compressionType {
case compressionTypeSnappy:
decompressed, err := snappy.Decode(nil, compressed)
if err != nil {
d.Errorf("failed decompressing data: %v", err)
}
d.Copy(bb, bytes.NewReader(decompressed))
default:
d.Errorf("Unsupported compression type: %x", compressionType)
}
if bb.Len() > 0 {
d.FieldStructRootBitBufFn(
"uncompressed",
bitio.NewBitReader(bb.Bytes(), -1),
func(d *decode.D) {
readTableBlockContents(int64(bb.Len()), d)
},
)
}
d.FieldRawLen("compressed", compressedSize*8)
}
})
}
// Read content encoded as a sequence of key/value-entries and a trailer of restarts.
// https://github.com/google/leveldb/blob/main/table/block_builder.cc#L16
// https://github.com/google/leveldb/blob/main/table/block.cc#L48
func readKeyValueContents(
keyCallbackFn func(sharedBytes []byte, unsharedSize int, d *decode.D) error,
valueCallbackFn func(d *decode.D),
size int64,
d *decode.D,
) {
start := d.Pos()
end := start + size*8
var restartOffset int64
d.SeekAbs(end - uint32BitSize)
d.FieldStruct("trailer", func(d *decode.D) {
numRestarts := int64(d.FieldU32("num_restarts"))
restartOffset = size*8 - (1+numRestarts)*uint32BitSize
d.SeekAbs(start + restartOffset)
d.FieldArray("restarts", func(d *decode.D) {
for i := 0; i < int(numRestarts); i++ {
d.FieldU32("restart")
}
})
})
// TK: how do you make an empty entries-array appear _above_ the trailer?
// Right now, its omitted if empty.
if restartOffset <= 0 {
return
}
d.SeekAbs(start)
d.FieldArray("entries", func(d *decode.D) {
var lastKey []byte
for d.Pos() < start+restartOffset {
d.FieldStruct("entry", func(d *decode.D) {
// https://github.com/google/leveldb/blob/main/table/block.cc#L48-L75
shared := int64(d.FieldULEB128("shared_bytes"))
unshared := int64(d.FieldULEB128("unshared_bytes"))
valueLength := int64(d.FieldULEB128("value_length"))
// read key
// https://github.com/google/leveldb/blob/main/table/block.cc#L261
if int(shared) > len(lastKey) {
d.Fatalf("`shared` size is larger than `lastKey` length")
}
keyPrefix := lastKey[:shared]
keySuffix := readBytesWithoutChangingPosition(unshared, d)
lastKey = append(keyPrefix, keySuffix...)
if keyCallbackFn == nil && shared == 0 {
d.FieldUTF8("key", int(unshared))
} else {
d.FieldStruct("key", func(d *decode.D) {
if keyCallbackFn == nil {
keyCallbackFn = readPrefixedBytes
}
err := keyCallbackFn(keyPrefix, int(unshared), d)
if err != nil {
d.Errorf("%v", err)
}
})
}
// read value
if valueCallbackFn == nil {
d.FieldUTF8("value", int(valueLength))
} else {
d.FieldStruct("value", valueCallbackFn)
}
})
}
})
}
func readBytesWithoutChangingPosition(nBytes int64, d *decode.D) []byte {
var result []byte
d.RangeFn(d.Pos(), nBytes*8, func(d *decode.D) {
br := d.RawLen(d.BitsLeft())
result = d.ReadAllBits(br)
})
return result
}
// https://github.com/google/leveldb/blob/main/table/format.cc#L24
func readBlockHandle(d *decode.D) blockHandle {
return blockHandle{
offset: d.FieldULEB128("offset"),
size: d.FieldULEB128("size"),
}
}
// Read bytes and prefix with given bytes;
// name read bytes "suffix" and the merged bytes "full" (synthetic field).
func readPrefixedBytes(prefixBytes []byte, nBytes int, d *decode.D) error {
br, err := d.TryFieldRawLen("suffix", int64(nBytes)*8)
if err != nil {
return err
}
full := append(prefixBytes, d.ReadAllBits(br)...)
d.FieldValueStr("full", string(full), strInferred)
return nil
}
// An "internal key" consists of the triple (user_key, type, sequence_number).
// https://github.com/google/leveldb/blob/main/db/dbformat.h#L171
func readInternalKey(sharedBytes []byte, unsharedSize int, d *decode.D) error {
// In the LevelDB encoding, the internal key can be cut at any byte:
// including the user_key, type, or sequence_number:
// https://github.com/google/leveldb/blob/main/table/block_builder.cc#L79-L83
//
// The resulting prefix is then shared among subsequent keys and not
// specified explicitly by them. Here, we handle each cutoff case.
//
// All sizes are in bytes unless mentioned otherwise.
keySize := len(sharedBytes) + unsharedSize
typeAndSequenceNumberSize := 8
// key
// +-----------------------------------------------+
// user_key
// +---------------------------------+
// ⁞ user_key_suffix type sequence_number
// [AAAAAAAAAAAA]⁞[BBBBBBBBBBBBBBBBBB] [T] [SSSSSSS]
// ⁞ 1 7 bytes
// +------------+⁞+--------------------------------+
// shared ⁞ unshared
// ⁞
// cutoff
if keySize < typeAndSequenceNumberSize || int64(unsharedSize) > d.BitsLeft()/8 {
return fmt.Errorf("key size %d or unshared size %d invalid", keySize, unsharedSize)
}
// case 1: user_key, type, and sequence_number fit fully in unshared.
if len(sharedBytes) == 0 {
d.FieldUTF8("user_key", keySize-typeAndSequenceNumberSize)
d.FieldU8("type", valueTypes, scalar.UintHex)
d.FieldU56("sequence_number")
return nil
}
// case 2: type and sequence_number fit fully in unshared: simulate user_key value.
if unsharedSize >= typeAndSequenceNumberSize {
suffix := fieldUTF8ReturnBytes("user_key_suffix", unsharedSize-typeAndSequenceNumberSize, d)
d.FieldValueStr("user_key", stringify(sharedBytes, suffix), strInferred)
d.FieldU8("type", valueTypes, scalar.UintHex)
d.FieldU56("sequence_number")
return nil
}
// case 3: sequence_number fits fully in unshared: simulate user_key and type value,
sequenceNumberSize := typeAndSequenceNumberSize - 1
if unsharedSize == sequenceNumberSize {
lastIndex := len(sharedBytes) - 1
d.FieldValueStr("user_key", string(sharedBytes[:lastIndex]), strInferred)
d.FieldValueUint("type", uint64(sharedBytes[lastIndex]), valueTypes, scalar.UintHex, uintInferred)
d.FieldU56("sequence_number")
return nil
}
// case 4: sequence_number cut: simulate user_key, type, and sequence_number value.
typeByteIndex := keySize - typeAndSequenceNumberSize
d.FieldValueStr("user_key", string(sharedBytes[:typeByteIndex]), strInferred)
d.FieldValueUint("type", uint64(sharedBytes[typeByteIndex]), valueTypes, scalar.UintHex, uintInferred)
var suffixBytes []byte
if unsharedSize > 0 {
br := d.FieldRawLen("sequence_number_suffix", int64(unsharedSize)*8)
suffixBytes = d.ReadAllBits(br)
}
sequenceNumberBytes := append(
sharedBytes[typeByteIndex+1:keySize-unsharedSize],
suffixBytes...,
)
sequenceNumberBE := bitio.Read64(sequenceNumberBytes[:], 0, int64(sequenceNumberSize*8))
sequenceNumberLE := bitio.ReverseBytes64(56, sequenceNumberBE)
d.FieldValueUint("sequence_number", sequenceNumberLE, uintInferred)
return nil
}
// Read content encoded in the "filter" or "stats" Meta Block format.
// https://github.com/google/leveldb/blob/main/doc/table_format.md#filter-meta-block
// https://github.com/google/leveldb/blob/main/table/filter_block.cc
func readMetaContent(nBytes int64, d *decode.D) {
// TK(2023-12-04)
d.FieldRawLen("raw", nBytes*8)
}
// Helpers
var strInferred = scalar.StrFn(func(s scalar.Str) (scalar.Str, error) {
s.Description = "inferred"
return s, nil
})
var uintInferred = scalar.UintFn(func(s scalar.Uint) (scalar.Uint, error) {
s.Description = "inferred"
return s, nil
})
func keyValueContentsReader(
keyCallbackFn func(sharedPrefix []byte, unsharedSize int, d *decode.D) error,
valueCallbackFn func(d *decode.D),
) func(size int64, d *decode.D) {
return func(size int64, d *decode.D) {
readKeyValueContents(keyCallbackFn, valueCallbackFn, size, d)
}
}
// Compute the checksum: a CRC32 as in RFC3720 + custom mask.
// https://datatracker.ietf.org/doc/html/rfc3720#appendix-B.4
func computeChecksum(bytes []uint8) uint32 {
crc32C := crc32.New(crc32.MakeTable(crc32.Castagnoli))
crc32C.Write(bytes)
return mask(crc32C.Sum32())
}
// Return a masked representation of the CRC.
// https://github.com/google/leveldb/blob/main/util/crc32c.h#L29
func mask(crc uint32) uint32 {
const kMaskDelta = 0xa282ead8
// Rotate right by 15 bits and add a constant.
return ((crc >> 15) | (crc << 17)) + kMaskDelta
}
// Concatinate byteslices and convert into a string.
func stringify(byteSlices ...[]byte) string {
totalSize := 0
for _, b := range byteSlices {
totalSize += len(b)
}
result := make([]byte, 0, totalSize)
for _, b := range byteSlices {
result = append(result, b...)
}
return string(result)
}
func fieldUTF8ReturnBytes(name string, nBytes int, d *decode.D) []byte {
var result []byte
d.RangeFn(d.Pos(), int64(nBytes)*8, func(d *decode.D) {
result = d.BytesLen(nBytes)
})
d.FieldUTF8(name, nBytes)
return result
}

14
format/leveldb/leveldb_table.md Normal file
View File

@ -0,0 +1,14 @@
### Limitations
- no Meta Blocks (like "filter") are decoded yet.
- Zstandard uncompression is not implemented yet.
### Authors
- [@mikez](https://github.com/mikez), original author
### References
- https://github.com/google/leveldb/blob/main/doc/table_format.md
- https://github.com/google/leveldb/blob/main/doc/impl.md
- https://github.com/google/leveldb/blob/main/doc/index.md

24
format/leveldb/testdata/help_leveldb_descriptor.fqtest vendored Normal file
View File

@ -0,0 +1,24 @@
$ fq -h leveldb_descriptor
leveldb_descriptor: LevelDB Descriptor decoder
Decode examples
===============
# Decode file as leveldb_descriptor
$ fq -d leveldb_descriptor . file
# Decode value as leveldb_descriptor
... | leveldb_descriptor
Limitations
===========
- fragmented non-"full" records are not merged and decoded further.
Authors
=======
- @mikez (https://github.com/mikez), original author
References
==========
- https://github.com/google/leveldb/blob/main/doc/impl.md#manifest
- https://github.com/google/leveldb/blob/main/doc/log_format.md
- https://github.com/google/leveldb/blob/main/db/version_edit.cc

24
format/leveldb/testdata/help_leveldb_log.fqtest vendored Normal file
View File

@ -0,0 +1,24 @@
$ fq -h leveldb_log
leveldb_log: LevelDB Log decoder
Decode examples
===============
# Decode file as leveldb_log
$ fq -d leveldb_log . file
# Decode value as leveldb_log
... | leveldb_log
Limitations
===========
- fragmented non-"full" records are not merged and decoded further.
Authors
=======
- @mikez (https://github.com/mikez), original author
References
==========
- https://github.com/google/leveldb/blob/main/doc/impl.md#log-files
- https://github.com/google/leveldb/blob/main/doc/log_format.md
- https://github.com/google/leveldb/blob/main/db/write_batch.cc

25
format/leveldb/testdata/help_leveldb_table.fqtest vendored Normal file
View File

@ -0,0 +1,25 @@
$ fq -h leveldb_table
leveldb_table: LevelDB Table decoder
Decode examples
===============
# Decode file as leveldb_table
$ fq -d leveldb_table . file
# Decode value as leveldb_table
... | leveldb_table
Limitations
===========
- no Meta Blocks (like "filter") are decoded yet.
- Zstandard uncompression is not implemented yet.
Authors
=======
- @mikez (https://github.com/mikez), original author
References
==========
- https://github.com/google/leveldb/blob/main/doc/table_format.md
- https://github.com/google/leveldb/blob/main/doc/impl.md
- https://github.com/google/leveldb/blob/main/doc/index.md

87
format/leveldb/testdata/leveldb_log.fqtest vendored Normal file
View File

@ -0,0 +1,87 @@
$ fq -d leveldb_log dv log_only.ldb/000003.log
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: log_only.ldb/000003.log (leveldb_log) 0x0-0x62c (1580)
| | | blocks[0:1]: 0x0-0x62c (1580)
| | | [0]{}: block 0x0-0x62c (1580)
| | | records[0:4]: 0x0-0x62c (1580)
| | | [0]{}: record 0x0-0x1df (479)
| | | header{}: 0x0-0x7 (7)
0x000|18 93 40 61 |..@a | checksum: 0x61409318 (valid) 0x0-0x4 (4)
0x000| d8 01 | .. | length: 472 0x4-0x6 (2)
0x000| 01 | . | record_type: "full" (1) 0x6-0x7 (1)
| | | data{}: 0x7-0x1df (472)
0x000| 01 00 00 00 00 00 00 00 | ........ | sequence: 1 0x7-0xf (8)
0x000| 01| .| count: 1 0xf-0x13 (4)
0x010|00 00 00 |... |
| | | records[0:1]: 0x13-0x1df (460)
| | | [0]{}: record 0x13-0x1df (460)
0x010| 01 | . | type: "value" (1) 0x13-0x14 (1)
| | | key{}: 0x14-0x20 (12)
0x010| 0b | . | length: 11 0x14-0x15 (1)
0x010| 6c 6f 72 65 6d 2e 6c 6f 72 65 6d| lorem.lorem| data: "lorem.lorem" 0x15-0x20 (11)
| | | value{}: 0x20-0x1df (447)
0x020|bd 03 |.. | length: 445 0x20-0x22 (2)
0x020| 4c 6f 72 65 6d 20 69 70 73 75 6d 20 64 6f| Lorem ipsum do| data: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x22-0x1df (445)
0x030|6c 6f 72 20 73 69 74 20 61 6d 65 74 2c 20 63 6f|lor sit amet, co|
* |until 0x1de.7 (445) | |
| | | [1]{}: record 0x1df-0x3be (479)
| | | header{}: 0x1df-0x1e6 (7)
0x1d0| 5a| Z| checksum: 0x12ba655a (valid) 0x1df-0x1e3 (4)
0x1e0|65 ba 12 |e.. |
0x1e0| d8 01 | .. | length: 472 0x1e3-0x1e5 (2)
0x1e0| 01 | . | record_type: "full" (1) 0x1e5-0x1e6 (1)
| | | data{}: 0x1e6-0x3be (472)
0x1e0| 02 00 00 00 00 00 00 00 | ........ | sequence: 2 0x1e6-0x1ee (8)
0x1e0| 01 00| ..| count: 1 0x1ee-0x1f2 (4)
0x1f0|00 00 |.. |
| | | records[0:1]: 0x1f2-0x3be (460)
| | | [0]{}: record 0x1f2-0x3be (460)
0x1f0| 01 | . | type: "value" (1) 0x1f2-0x1f3 (1)
| | | key{}: 0x1f3-0x1ff (12)
0x1f0| 0b | . | length: 11 0x1f3-0x1f4 (1)
0x1f0| 6c 6f 72 65 6d 2e 69 70 73 75 6d | lorem.ipsum | data: "lorem.ipsum" 0x1f4-0x1ff (11)
| | | value{}: 0x1ff-0x3be (447)
0x1f0| bd| .| length: 445 0x1ff-0x201 (2)
0x200|03 |. |
0x200| 4c 6f 72 65 6d 20 69 70 73 75 6d 20 64 6f 6c| Lorem ipsum dol| data: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x201-0x3be (445)
0x210|6f 72 20 73 69 74 20 61 6d 65 74 2c 20 63 6f 6e|or sit amet, con|
* |until 0x3bd.7 (445) | |
| | | [2]{}: record 0x3be-0x59d (479)
| | | header{}: 0x3be-0x3c5 (7)
0x3b0| 09 81| ..| checksum: 0x890d8109 (valid) 0x3be-0x3c2 (4)
0x3c0|0d 89 |.. |
0x3c0| d8 01 | .. | length: 472 0x3c2-0x3c4 (2)
0x3c0| 01 | . | record_type: "full" (1) 0x3c4-0x3c5 (1)
| | | data{}: 0x3c5-0x59d (472)
0x3c0| 03 00 00 00 00 00 00 00 | ........ | sequence: 3 0x3c5-0x3cd (8)
0x3c0| 01 00 00| ...| count: 1 0x3cd-0x3d1 (4)
0x3d0|00 |. |
| | | records[0:1]: 0x3d1-0x59d (460)
| | | [0]{}: record 0x3d1-0x59d (460)
0x3d0| 01 | . | type: "value" (1) 0x3d1-0x3d2 (1)
| | | key{}: 0x3d2-0x3de (12)
0x3d0| 0b | . | length: 11 0x3d2-0x3d3 (1)
0x3d0| 6c 6f 72 65 6d 2e 64 6f 6c 6f 72 | lorem.dolor | data: "lorem.dolor" 0x3d3-0x3de (11)
| | | value{}: 0x3de-0x59d (447)
0x3d0| bd 03| ..| length: 445 0x3de-0x3e0 (2)
0x3e0|4c 6f 72 65 6d 20 69 70 73 75 6d 20 64 6f 6c 6f|Lorem ipsum dolo| data: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x3e0-0x59d (445)
* |until 0x59c.7 (445) | |
| | | [3]{}: record 0x59d-0x62c (143)
| | | header{}: 0x59d-0x5a4 (7)
0x590| a0 86 3e| ..>| checksum: 0xb3e86a0 (valid) 0x59d-0x5a1 (4)
0x5a0|0b |. |
0x5a0| 88 00 | .. | length: 136 0x5a1-0x5a3 (2)
0x5a0| 01 | . | record_type: "full" (1) 0x5a3-0x5a4 (1)
| | | data{}: 0x5a4-0x62c (136)
0x5a0| 04 00 00 00 00 00 00 00 | ........ | sequence: 4 0x5a4-0x5ac (8)
0x5a0| 01 00 00 00| ....| count: 1 0x5ac-0x5b0 (4)
| | | records[0:1]: 0x5b0-0x62c (124)
| | | [0]{}: record 0x5b0-0x62c (124)
0x5b0|01 |. | type: "value" (1) 0x5b0-0x5b1 (1)
| | | key{}: 0x5b1-0x5b5 (4)
0x5b0| 03 | . | length: 3 0x5b1-0x5b2 (1)
0x5b0| 72 6f 77 | row | data: "row" 0x5b2-0x5b5 (3)
| | | value{}: 0x5b5-0x62c (119)
0x5b0| 76 | v | length: 118 0x5b5-0x5b6 (1)
0x5b0| 52 6f 77 2c 20 72 6f 77 2c 20| Row, row, | data: "Row, row, row your boat\nGently down the stream...." 0x5b6-0x62c (118)
0x5c0|72 6f 77 20 79 6f 75 72 20 62 6f 61 74 0a 47 65|row your boat.Ge|
* |until 0x62b.7 (end) (118) | |

57
format/leveldb/testdata/leveldb_manifest.fqtest vendored Normal file
View File

@ -0,0 +1,57 @@
$ fq -d leveldb_descriptor dv uncompressed.ldb/MANIFEST-000004
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: uncompressed.ldb/MANIFEST-000004 (leveldb_descriptor) 0x0-0x57 (87)
| | | blocks[0:1]: 0x0-0x57 (87)
| | | [0]{}: block 0x0-0x57 (87)
| | | records[0:2]: 0x0-0x57 (87)
| | | [0]{}: record 0x0-0x23 (35)
| | | header{}: 0x0-0x7 (7)
0x00|56 f9 b8 f8 |V... | checksum: 0xf8b8f956 (valid) 0x0-0x4 (4)
0x00| 1c 00 | .. | length: 28 0x4-0x6 (2)
0x00| 01 | . | record_type: "full" (1) 0x6-0x7 (1)
| | | data{}: 0x7-0x23 (28)
| | | tags[0:1]: 0x7-0x23 (28)
| | | [0]{}: tag 0x7-0x23 (28)
0x00| 01 | . | key: "comparator" (1) 0x7-0x8 (1)
| | | value{}: 0x8-0x23 (27)
0x00| 1a | . | length: 26 0x8-0x9 (1)
0x00| 6c 65 76 65 6c 64 62| leveldb| data: "leveldb.BytewiseComparator" 0x9-0x23 (26)
0x10|2e 42 79 74 65 77 69 73 65 43 6f 6d 70 61 72 61|.BytewiseCompara|
0x20|74 6f 72 |tor |
| | | [1]{}: record 0x23-0x57 (52)
| | | header{}: 0x23-0x2a (7)
0x20| d2 53 ab 33 | .S.3 | checksum: 0x33ab53d2 (valid) 0x23-0x27 (4)
0x20| 2d 00 | -. | length: 45 0x27-0x29 (2)
0x20| 01 | . | record_type: "full" (1) 0x29-0x2a (1)
| | | data{}: 0x2a-0x57 (45)
| | | tags[0:5]: 0x2a-0x57 (45)
| | | [0]{}: tag 0x2a-0x2c (2)
0x20| 02 | . | key: "log_number" (2) 0x2a-0x2b (1)
0x20| 06 | . | value: 6 0x2b-0x2c (1)
| | | [1]{}: tag 0x2c-0x2e (2)
0x20| 09 | . | key: "previous_log_number" (9) 0x2c-0x2d (1)
0x20| 00 | . | value: 0 0x2d-0x2e (1)
| | | [2]{}: tag 0x2e-0x30 (2)
0x20| 03 | . | key: "next_file_number" (3) 0x2e-0x2f (1)
0x20| 07| .| value: 7 0x2f-0x30 (1)
| | | [3]{}: tag 0x30-0x32 (2)
0x30|04 |. | key: "last_sequence" (4) 0x30-0x31 (1)
0x30| 04 | . | value: 4 0x31-0x32 (1)
| | | [4]{}: tag 0x32-0x57 (37)
0x30| 07 | . | key: "new_file" (7) 0x32-0x33 (1)
| | | value{}: 0x33-0x57 (36)
0x30| 00 | . | level: 0 0x33-0x34 (1)
0x30| 05 | . | file_number: 5 0x34-0x35 (1)
0x30| da 0c | .. | file_size: 1626 0x35-0x37 (2)
| | | smallest_internal_key{}: 0x37-0x4b (20)
0x30| 13 | . | length: 19 0x37-0x38 (1)
| | | data{}: 0x38-0x4b (19)
0x30| 6c 6f 72 65 6d 2e 64 6f| lorem.do| user_key: "lorem.dolor" 0x38-0x43 (11)
0x40|6c 6f 72 |lor |
0x40| 01 | . | type: "value" (0x1) 0x43-0x44 (1)
0x40| 03 00 00 00 00 00 00 | ....... | sequence_number: 3 0x44-0x4b (7)
| | | largest_internal_key{}: 0x4b-0x57 (12)
0x40| 0b | . | length: 11 0x4b-0x4c (1)
| | | data{}: 0x4c-0x57 (11)
0x40| 72 6f 77 | row | user_key: "row" 0x4c-0x4f (3)
0x40| 01| .| type: "value" (0x1) 0x4f-0x50 (1)
0x50|04 00 00 00 00 00 00| |.......| | sequence_number: 4 0x50-0x57 (7)

39
format/leveldb/testdata/leveldb_manifest_torepr.fqtest vendored Normal file
View File

@ -0,0 +1,39 @@
$ fq -d leveldb_descriptor torepr uncompressed.ldb/MANIFEST-000004
[
[
{
"comparator": "leveldb.BytewiseComparator"
}
],
[
{
"log_number": 6
},
{
"previous_log_number": 0
},
{
"next_file_number": 7
},
{
"last_sequence": 4
},
{
"new_file": {
"file_number": 5,
"file_size": 1626,
"largest_internal_key": {
"sequence_number": 4,
"type": "value",
"user_key": "row"
},
"level": 0,
"smallest_internal_key": {
"sequence_number": 3,
"type": "value",
"user_key": "lorem.dolor"
}
}
}
]
]

5539
format/leveldb/testdata/leveldb_table_repeats.fqtest vendored Normal file
View File

File diff suppressed because it is too large Load Diff

104
format/leveldb/testdata/leveldb_table_snappy.fqtest vendored Normal file
View File

@ -0,0 +1,104 @@
$ fq -d leveldb_table dv snappy.ldb/000005.ldb
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: snappy.ldb/000005.ldb (leveldb_table) 0x0-0x2c4 (708)
| | | data[0:1]: 0x0-0x26b (619)
| | | [0]{}: data_block 0x0-0x26b (619)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| uncompressed{}: 0x0-0x5fc (1532)
| | | entries[0:4]: 0x0-0x5f4 (1524)
| | | [0]{}: entry 0x0-0x1d4 (468)
0x000|00 |. | shared_bytes: 0 0x0-0x1 (1)
0x000| 13 | . | unshared_bytes: 19 0x1-0x2 (1)
0x000| bd 03 | .. | value_length: 445 0x2-0x4 (2)
| | | key{}: 0x4-0x17 (19)
0x000| 6c 6f 72 65 6d 2e 64 6f 6c 6f 72 | lorem.dolor | user_key: "lorem.dolor" 0x4-0xf (11)
0x000| 01| .| type: "value" (0x1) 0xf-0x10 (1)
0x001|03 00 00 00 00 00 00 |....... | sequence_number: 3 0x10-0x17 (7)
0x001| 4c 6f 72 65 6d 20 69 70 73| Lorem ips| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x17-0x1d4 (445)
0x002|75 6d 20 64 6f 6c 6f 72 20 73 69 74 20 61 6d 65|um dolor sit ame|
* |until 0x1d3.7 (445) | |
| | | [1]{}: entry 0x1d4-0x3a2 (462)
0x01d| 06 | . | shared_bytes: 6 0x1d4-0x1d5 (1)
0x01d| 0d | . | unshared_bytes: 13 0x1d5-0x1d6 (1)
0x01d| bd 03 | .. | value_length: 445 0x1d6-0x1d8 (2)
| | | key{}: 0x1d8-0x1e5 (13)
0x01d| 69 70 73 75 6d | ipsum | user_key_suffix: "ipsum" 0x1d8-0x1dd (5)
| | | user_key: "lorem.ipsum" (inferred)
0x01d| 01 | . | type: "value" (0x1) 0x1dd-0x1de (1)
0x01d| 02 00| ..| sequence_number: 2 0x1de-0x1e5 (7)
0x01e|00 00 00 00 00 |..... |
0x01e| 4c 6f 72 65 6d 20 69 70 73 75 6d| Lorem ipsum| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x1e5-0x3a2 (445)
0x01f|20 64 6f 6c 6f 72 20 73 69 74 20 61 6d 65 74 2c| dolor sit amet,|
* |until 0x3a1.7 (445) | |
| | | [2]{}: entry 0x3a2-0x570 (462)
0x03a| 06 | . | shared_bytes: 6 0x3a2-0x3a3 (1)
0x03a| 0d | . | unshared_bytes: 13 0x3a3-0x3a4 (1)
0x03a| bd 03 | .. | value_length: 445 0x3a4-0x3a6 (2)
| | | key{}: 0x3a6-0x3b3 (13)
0x03a| 6c 6f 72 65 6d | lorem | user_key_suffix: "lorem" 0x3a6-0x3ab (5)
| | | user_key: "lorem.lorem" (inferred)
0x03a| 01 | . | type: "value" (0x1) 0x3ab-0x3ac (1)
0x03a| 01 00 00 00| ....| sequence_number: 1 0x3ac-0x3b3 (7)
0x03b|00 00 00 |... |
0x03b| 4c 6f 72 65 6d 20 69 70 73 75 6d 20 64| Lorem ipsum d| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x3b3-0x570 (445)
0x03c|6f 6c 6f 72 20 73 69 74 20 61 6d 65 74 2c 20 63|olor sit amet, c|
* |until 0x56f.7 (445) | |
| | | [3]{}: entry 0x570-0x5f4 (132)
0x057|00 |. | shared_bytes: 0 0x570-0x571 (1)
0x057| 0b | . | unshared_bytes: 11 0x571-0x572 (1)
0x057| 76 | v | value_length: 118 0x572-0x573 (1)
| | | key{}: 0x573-0x57e (11)
0x057| 72 6f 77 | row | user_key: "row" 0x573-0x576 (3)
0x057| 01 | . | type: "value" (0x1) 0x576-0x577 (1)
0x057| 04 00 00 00 00 00 00 | ....... | sequence_number: 4 0x577-0x57e (7)
0x057| 52 6f| Ro| value: "Row, row, row your boat\nGently down the stream...." 0x57e-0x5f4 (118)
0x058|77 2c 20 72 6f 77 2c 20 72 6f 77 20 79 6f 75 72|w, row, row your|
* |until 0x5f3.7 (118) | |
| | | trailer{}: 0x5f4-0x5fc (8)
| | | restarts[0:1]: 0x5f4-0x5f8 (4)
0x05f| 00 00 00 00 | .... | [0]: 0 restart 0x5f4-0x5f8 (4)
0x05f| 01 00 00 00| | ....| | num_restarts: 1 0x5f8-0x5fc (4)
0x00000|fc 0b 44 00 13 bd 03 6c 6f 72 65 6d 2e 64 6f 6c|..D....lorem.dol| compressed: raw bits 0x0-0x266 (614)
* |until 0x265.7 (614) | |
0x00260| 01 | . | compression: "snappy" (0x1) 0x266-0x267 (1)
0x00260| b6 9d 28 0d | ..(. | checksum: 0xd289db6 (valid) 0x267-0x26b (4)
| | | metaindex{}: 0x26b-0x278 (13)
| | | uncompressed{}: 0x26b-0x273 (8)
| | | trailer{}: 0x26b-0x273 (8)
| | | restarts[0:1]: 0x26b-0x26f (4)
0x00260| 00 00 00 00 | .... | [0]: 0 restart 0x26b-0x26f (4)
0x00260| 01| .| num_restarts: 1 0x26f-0x273 (4)
0x00270|00 00 00 |... |
0x00270| 00 | . | compression: "none" (0x0) 0x273-0x274 (1)
0x00270| c0 f2 a1 b0 | .... | checksum: 0xb0a1f2c0 (valid) 0x274-0x278 (4)
| | | index{}: 0x278-0x294 (28)
| | | uncompressed{}: 0x278-0x28f (23)
| | | entries[0:1]: 0x278-0x287 (15)
| | | [0]{}: entry 0x278-0x287 (15)
0x00270| 00 | . | shared_bytes: 0 0x278-0x279 (1)
0x00270| 09 | . | unshared_bytes: 9 0x279-0x27a (1)
0x00270| 03 | . | value_length: 3 0x27a-0x27b (1)
| | | key{}: 0x27b-0x284 (9)
0x00270| 73 | s | user_key: "s" 0x27b-0x27c (1)
0x00270| 01 | . | type: "value" (0x1) 0x27c-0x27d (1)
0x00270| ff ff ff| ...| sequence_number: 72057594037927935 0x27d-0x284 (7)
0x00280|ff ff ff ff |.... |
| | | value{}: 0x284-0x287 (3)
0x00280| 00 | . | offset: 0 0x284-0x285 (1)
0x00280| e6 04 | .. | size: 614 0x285-0x287 (2)
| | | trailer{}: 0x287-0x28f (8)
| | | restarts[0:1]: 0x287-0x28b (4)
0x00280| 00 00 00 00 | .... | [0]: 0 restart 0x287-0x28b (4)
0x00280| 01 00 00 00 | .... | num_restarts: 1 0x28b-0x28f (4)
0x00280| 00| .| compression: "none" (0x0) 0x28f-0x290 (1)
0x00290|68 24 42 91 |h$B. | checksum: 0x91422468 (valid) 0x290-0x294 (4)
| | | footer{}: 0x294-0x2c4 (48)
| | | metaindex_handle{}: 0x294-0x297 (3)
0x00290| eb 04 | .. | offset: 619 0x294-0x296 (2)
0x00290| 08 | . | size: 8 0x296-0x297 (1)
| | | index_handle{}: 0x297-0x29a (3)
0x00290| f8 04 | .. | offset: 632 0x297-0x299 (2)
0x00290| 17 | . | size: 23 0x299-0x29a (1)
0x00290| 00 00 00 00 00 00| ......| padding: raw bits 0x29a-0x2bc (34)
0x002a0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
0x002b0|00 00 00 00 00 00 00 00 00 00 00 00 |............ |
0x002b0| 57 fb 80 8b| W...| magic_number: 0xdb4775248b80fb57 (valid) 0x2bc-0x2c4 (8)
0x002c0|24 75 47 db| |$uG.| |

100
format/leveldb/testdata/leveldb_table_uncompressed.fqtest vendored Normal file
View File

@ -0,0 +1,100 @@
$ fq -d leveldb_table dv uncompressed.ldb/000005.ldb
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: uncompressed.ldb/000005.ldb (leveldb_table) 0x0-0x65a (1626)
| | | data[0:1]: 0x0-0x601 (1537)
| | | [0]{}: data_block 0x0-0x601 (1537)
| | | uncompressed{}: 0x0-0x5fc (1532)
| | | entries[0:4]: 0x0-0x5f4 (1524)
| | | [0]{}: entry 0x0-0x1d4 (468)
0x000|00 |. | shared_bytes: 0 0x0-0x1 (1)
0x000| 13 | . | unshared_bytes: 19 0x1-0x2 (1)
0x000| bd 03 | .. | value_length: 445 0x2-0x4 (2)
| | | key{}: 0x4-0x17 (19)
0x000| 6c 6f 72 65 6d 2e 64 6f 6c 6f 72 | lorem.dolor | user_key: "lorem.dolor" 0x4-0xf (11)
0x000| 01| .| type: "value" (0x1) 0xf-0x10 (1)
0x010|03 00 00 00 00 00 00 |....... | sequence_number: 3 0x10-0x17 (7)
0x010| 4c 6f 72 65 6d 20 69 70 73| Lorem ips| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x17-0x1d4 (445)
0x020|75 6d 20 64 6f 6c 6f 72 20 73 69 74 20 61 6d 65|um dolor sit ame|
* |until 0x1d3.7 (445) | |
| | | [1]{}: entry 0x1d4-0x3a2 (462)
0x1d0| 06 | . | shared_bytes: 6 0x1d4-0x1d5 (1)
0x1d0| 0d | . | unshared_bytes: 13 0x1d5-0x1d6 (1)
0x1d0| bd 03 | .. | value_length: 445 0x1d6-0x1d8 (2)
| | | key{}: 0x1d8-0x1e5 (13)
0x1d0| 69 70 73 75 6d | ipsum | user_key_suffix: "ipsum" 0x1d8-0x1dd (5)
| | | user_key: "lorem.ipsum" (inferred)
0x1d0| 01 | . | type: "value" (0x1) 0x1dd-0x1de (1)
0x1d0| 02 00| ..| sequence_number: 2 0x1de-0x1e5 (7)
0x1e0|00 00 00 00 00 |..... |
0x1e0| 4c 6f 72 65 6d 20 69 70 73 75 6d| Lorem ipsum| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x1e5-0x3a2 (445)
0x1f0|20 64 6f 6c 6f 72 20 73 69 74 20 61 6d 65 74 2c| dolor sit amet,|
* |until 0x3a1.7 (445) | |
| | | [2]{}: entry 0x3a2-0x570 (462)
0x3a0| 06 | . | shared_bytes: 6 0x3a2-0x3a3 (1)
0x3a0| 0d | . | unshared_bytes: 13 0x3a3-0x3a4 (1)
0x3a0| bd 03 | .. | value_length: 445 0x3a4-0x3a6 (2)
| | | key{}: 0x3a6-0x3b3 (13)
0x3a0| 6c 6f 72 65 6d | lorem | user_key_suffix: "lorem" 0x3a6-0x3ab (5)
| | | user_key: "lorem.lorem" (inferred)
0x3a0| 01 | . | type: "value" (0x1) 0x3ab-0x3ac (1)
0x3a0| 01 00 00 00| ....| sequence_number: 1 0x3ac-0x3b3 (7)
0x3b0|00 00 00 |... |
0x3b0| 4c 6f 72 65 6d 20 69 70 73 75 6d 20 64| Lorem ipsum d| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x3b3-0x570 (445)
0x3c0|6f 6c 6f 72 20 73 69 74 20 61 6d 65 74 2c 20 63|olor sit amet, c|
* |until 0x56f.7 (445) | |
| | | [3]{}: entry 0x570-0x5f4 (132)
0x570|00 |. | shared_bytes: 0 0x570-0x571 (1)
0x570| 0b | . | unshared_bytes: 11 0x571-0x572 (1)
0x570| 76 | v | value_length: 118 0x572-0x573 (1)
| | | key{}: 0x573-0x57e (11)
0x570| 72 6f 77 | row | user_key: "row" 0x573-0x576 (3)
0x570| 01 | . | type: "value" (0x1) 0x576-0x577 (1)
0x570| 04 00 00 00 00 00 00 | ....... | sequence_number: 4 0x577-0x57e (7)
0x570| 52 6f| Ro| value: "Row, row, row your boat\nGently down the stream...." 0x57e-0x5f4 (118)
0x580|77 2c 20 72 6f 77 2c 20 72 6f 77 20 79 6f 75 72|w, row, row your|
* |until 0x5f3.7 (118) | |
| | | trailer{}: 0x5f4-0x5fc (8)
| | | restarts[0:1]: 0x5f4-0x5f8 (4)
0x5f0| 00 00 00 00 | .... | [0]: 0 restart 0x5f4-0x5f8 (4)
0x5f0| 01 00 00 00 | .... | num_restarts: 1 0x5f8-0x5fc (4)
0x5f0| 00 | . | compression: "none" (0x0) 0x5fc-0x5fd (1)
0x5f0| 6f 99 1d| o..| checksum: 0xb31d996f (valid) 0x5fd-0x601 (4)
0x600|b3 |. |
| | | metaindex{}: 0x601-0x60e (13)
| | | uncompressed{}: 0x601-0x609 (8)
| | | trailer{}: 0x601-0x609 (8)
| | | restarts[0:1]: 0x601-0x605 (4)
0x600| 00 00 00 00 | .... | [0]: 0 restart 0x601-0x605 (4)
0x600| 01 00 00 00 | .... | num_restarts: 1 0x605-0x609 (4)
0x600| 00 | . | compression: "none" (0x0) 0x609-0x60a (1)
0x600| c0 f2 a1 b0 | .... | checksum: 0xb0a1f2c0 (valid) 0x60a-0x60e (4)
| | | index{}: 0x60e-0x62a (28)
| | | uncompressed{}: 0x60e-0x625 (23)
| | | entries[0:1]: 0x60e-0x61d (15)
| | | [0]{}: entry 0x60e-0x61d (15)
0x600| 00 | . | shared_bytes: 0 0x60e-0x60f (1)
0x600| 09| .| unshared_bytes: 9 0x60f-0x610 (1)
0x610|03 |. | value_length: 3 0x610-0x611 (1)
| | | key{}: 0x611-0x61a (9)
0x610| 73 | s | user_key: "s" 0x611-0x612 (1)
0x610| 01 | . | type: "value" (0x1) 0x612-0x613 (1)
0x610| ff ff ff ff ff ff ff | ....... | sequence_number: 72057594037927935 0x613-0x61a (7)
| | | value{}: 0x61a-0x61d (3)
0x610| 00 | . | offset: 0 0x61a-0x61b (1)
0x610| fc 0b | .. | size: 1532 0x61b-0x61d (2)
| | | trailer{}: 0x61d-0x625 (8)
| | | restarts[0:1]: 0x61d-0x621 (4)
0x610| 00 00 00| ...| [0]: 0 restart 0x61d-0x621 (4)
0x620|00 |. |
0x620| 01 00 00 00 | .... | num_restarts: 1 0x621-0x625 (4)
0x620| 00 | . | compression: "none" (0x0) 0x625-0x626 (1)
0x620| 68 e2 bf 46 | h..F | checksum: 0x46bfe268 (valid) 0x626-0x62a (4)
| | | footer{}: 0x62a-0x65a (48)
| | | metaindex_handle{}: 0x62a-0x62d (3)
0x620| 81 0c | .. | offset: 1537 0x62a-0x62c (2)
0x620| 08 | . | size: 8 0x62c-0x62d (1)
| | | index_handle{}: 0x62d-0x630 (3)
0x620| 8e 0c | .. | offset: 1550 0x62d-0x62f (2)
0x620| 17| .| size: 23 0x62f-0x630 (1)
0x630|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| padding: raw bits 0x630-0x652 (34)
* |until 0x651.7 (34) | |
0x650| 57 fb 80 8b 24 75 47 db| | W...$uG.| | magic_number: 0xdb4775248b80fb57 (valid) 0x652-0x65a (8)

BIN
format/leveldb/testdata/log_only.ldb/000003.log vendored Normal file
View File

Binary file not shown.

1
format/leveldb/testdata/log_only.ldb/CURRENT vendored Normal file
View File

@ -0,0 +1 @@
MANIFEST-000002

0
format/leveldb/testdata/log_only.ldb/LOCK vendored Normal file
View File

2
format/leveldb/testdata/log_only.ldb/LOG vendored Normal file
View File

@ -0,0 +1,2 @@
2023/12/05-20:38:42.412901 0x107ac5e00 Creating DB ./log_only.ldb since it was missing.
2023/12/05-20:38:42.512812 0x107ac5e00 Delete type=3 #1

BIN
format/leveldb/testdata/log_only.ldb/MANIFEST-000002 vendored Normal file
View File

Binary file not shown.

6
format/leveldb/testdata/lorem.json vendored Normal file
View File

@ -0,0 +1,6 @@
{
"lorem.lorem": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
"lorem.ipsum": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
"lorem.dolor": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
"row": "Row, row, row your boat\nGently down the stream.\nMerrily, merrily, merrily, merrily,\nLife is but a dream. 🚣‍♂️"
}

67
format/leveldb/testdata/make_ldb.py vendored Normal file
View File

@ -0,0 +1,67 @@
# Make LevelDB data: both uncompressed and compressed.
# Usage: python3 make_ldb.py
import io
import json
import os
import plyvel # pip install plyvel
import snappy # pip install python-snappy
def main():
make("./lorem.json", "./uncompressed.ldb", compression=None, reopen=True)
make("./lorem.json", "./snappy.ldb", compression="snappy", reopen=True)
make(make_sample_json(), "./repeats.ldb", rounds=2, reopen=True)
make("./lorem.json", "./log_only.ldb", compression=None)
def make(input_filepath, output_filepath, compression="snappy", reopen=False, rounds=1):
if os.path.exists(output_filepath):
raise FileExistsError(f"The file {output_filepath} already exists.")
# make a .ldb file and a .log file within
db = plyvel.DB(output_filepath, compression=compression, create_if_missing=True)
data = read_json(input_filepath)
for i in range(rounds):
for key, value in data.items():
db.put(key.encode(), value.encode())
db.close()
if reopen:
# reopen, so a .ldb file is generated within the .ldb directory;
# otherwise there's a .log file only with the fresh changes.
db = plyvel.DB(output_filepath, compression=compression)
db.close()
def make_sample_json():
# in table data blocks, this splits the shared key
# inside the sequence_number of the internal key;
# see `readInternalKey` in leveldb_table.go for details.
data = {}
for i in range(0x100):
data[f"lorem.{chr(i)}"] = "ipsum"
result = io.StringIO()
json.dump(data, result)
result.seek(0)
return result
# Helpers
def compress(value):
return snappy.compress(value)
def decompress(value):
return snappy.decompress(value)
def read_json(filepath_or_buffer):
if hasattr(filepath_or_buffer, "read"):
return json.load(filepath_or_buffer)
with open(filepath_or_buffer, "r") as file:
return json.load(file)
main()

BIN
format/leveldb/testdata/repeats.ldb/000005.ldb vendored Normal file
View File

Binary file not shown.

0
format/leveldb/testdata/repeats.ldb/000006.log vendored Normal file
View File

1
format/leveldb/testdata/repeats.ldb/CURRENT vendored Normal file
View File

@ -0,0 +1 @@
MANIFEST-000004

0
format/leveldb/testdata/repeats.ldb/LOCK vendored Normal file
View File

5
format/leveldb/testdata/repeats.ldb/LOG vendored Normal file
View File

@ -0,0 +1,5 @@
2023/12/08-22:52:56.629835 0x10eb27e00 Recovering log #3
2023/12/08-22:52:56.631293 0x10eb27e00 Level-0 table #5: started
2023/12/08-22:52:56.640224 0x10eb27e00 Level-0 table #5: 2333 bytes OK
2023/12/08-22:52:56.682590 0x10eb27e00 Delete type=0 #3
2023/12/08-22:52:56.682721 0x10eb27e00 Delete type=3 #2

2
format/leveldb/testdata/repeats.ldb/LOG.old vendored Normal file
View File

@ -0,0 +1,2 @@
2023/12/08-22:52:56.458994 0x10eb27e00 Creating DB ./repeats.ldb since it was missing.
2023/12/08-22:52:56.591817 0x10eb27e00 Delete type=3 #1

BIN
format/leveldb/testdata/repeats.ldb/MANIFEST-000004 vendored Normal file
View File

Binary file not shown.

BIN
format/leveldb/testdata/snappy.ldb/000005.ldb vendored Normal file
View File

Binary file not shown.

0
format/leveldb/testdata/snappy.ldb/000006.log vendored Normal file
View File

1
format/leveldb/testdata/snappy.ldb/CURRENT vendored Normal file
View File

@ -0,0 +1 @@
MANIFEST-000004

0
format/leveldb/testdata/snappy.ldb/LOCK vendored Normal file
View File

5
format/leveldb/testdata/snappy.ldb/LOG vendored Normal file
View File

@ -0,0 +1,5 @@
2023/12/05-20:38:42.349463 0x107ac5e00 Recovering log #3
2023/12/05-20:38:42.350562 0x107ac5e00 Level-0 table #5: started
2023/12/05-20:38:42.359725 0x107ac5e00 Level-0 table #5: 708 bytes OK
2023/12/05-20:38:42.411415 0x107ac5e00 Delete type=0 #3
2023/12/05-20:38:42.411476 0x107ac5e00 Delete type=3 #2

2
format/leveldb/testdata/snappy.ldb/LOG.old vendored Normal file
View File

@ -0,0 +1,2 @@
2023/12/05-20:38:42.215352 0x107ac5e00 Creating DB ./snappy.ldb since it was missing.
2023/12/05-20:38:42.294790 0x107ac5e00 Delete type=3 #1

BIN
format/leveldb/testdata/snappy.ldb/MANIFEST-000004 vendored Normal file
View File

Binary file not shown.

BIN
format/leveldb/testdata/uncompressed.ldb/000005.ldb vendored Normal file
View File

Binary file not shown.

0
format/leveldb/testdata/uncompressed.ldb/000006.log vendored Normal file
View File

1
format/leveldb/testdata/uncompressed.ldb/CURRENT vendored Normal file
View File

@ -0,0 +1 @@
MANIFEST-000004

0
format/leveldb/testdata/uncompressed.ldb/LOCK vendored Normal file
View File

5
format/leveldb/testdata/uncompressed.ldb/LOG vendored Normal file
View File

@ -0,0 +1,5 @@
2023/12/05-20:38:41.945807 0x107ac5e00 Recovering log #3
2023/12/05-20:38:41.945944 0x107ac5e00 Level-0 table #5: started
2023/12/05-20:38:41.953310 0x107ac5e00 Level-0 table #5: 1626 bytes OK
2023/12/05-20:38:42.214073 0x107ac5e00 Delete type=0 #3
2023/12/05-20:38:42.214131 0x107ac5e00 Delete type=3 #2

2
format/leveldb/testdata/uncompressed.ldb/LOG.old vendored Normal file
View File

@ -0,0 +1,2 @@
2023/12/05-20:38:41.773771 0x107ac5e00 Creating DB ./uncompressed.ldb since it was missing.
2023/12/05-20:38:41.910278 0x107ac5e00 Delete type=3 #1

BIN
format/leveldb/testdata/uncompressed.ldb/MANIFEST-000004 vendored Normal file
View File

Binary file not shown.

2
pkg/decode/decode.go
View File

@ -938,7 +938,7 @@ func (d *D) FramedFn(nBits int64, fn func(d *D)) int64 {
return decodeLen
}
// LimitedFn decode from current position nBits forward. When done position will after last bit decoded.
// LimitedFn decode from current position nBits forward. When done position will be after last bit decoded.
func (d *D) LimitedFn(nBits int64, fn func(d *D)) int64 {
if nBits < 0 {
d.Fatalf("%d nBits < 0", nBits)

11
pkg/decode/read.go
View File

@ -247,10 +247,15 @@ func (d *D) tryBool() (bool, error) {
return n == 1, nil
}
// Unsigned LEB128, description from wasm spec
// Unsigned LEB128, also known as "Base 128 Varint".
//
// Description from wasm spec:
//
// uN ::= n:byte => n (if n < 2^7 && n < 2^N)
// n:byte m:u(N-7) => 2^7 * m + (n - 2^7) (if n >= 2^7 && N > 7)
//
// Varint description:
// https://protobuf.dev/programming-guides/encoding/#varints
func (d *D) tryULEB128() (uint64, error) {
var result uint64
var shift uint
@ -260,8 +265,8 @@ func (d *D) tryULEB128() (uint64, error) {
if shift >= 63 && b != 0 {
return 0, fmt.Errorf("overflow when reading unsigned leb128, shift %d >= 63", shift)
}
result |= (b & 0x7f) << shift
if b&0x80 == 0 {
result |= (b & 0b01111111) << shift
if b&0b10000000 == 0 {
break
}
shift += 7