mirror of
https://github.com/wader/fq.git
synced 2024-09-11 12:05:39 +03:00
leveldb: address PR comments
This commit is contained in:
parent
efc59a8130
commit
b05aa997ab
@ -102,6 +102,7 @@ ipv6_packet,
|
||||
jpeg,
|
||||
json,
|
||||
jsonl,
|
||||
[leveldb_ldb](doc/formats.md#leveldb_ldb),
|
||||
[luajit](doc/formats.md#luajit),
|
||||
[macho](doc/formats.md#macho),
|
||||
macho_fat,
|
||||
|
@ -72,6 +72,7 @@
|
||||
|`jpeg` |Joint Photographic Experts Group file |<sub>`exif` `icc_profile`</sub>|
|
||||
|`json` |JavaScript Object Notation |<sub></sub>|
|
||||
|`jsonl` |JavaScript Object Notation Lines |<sub></sub>|
|
||||
|[`leveldb_ldb`](#leveldb_ldb) |LevelDB Table |<sub></sub>|
|
||||
|[`luajit`](#luajit) |LuaJIT 2.0 bytecode |<sub></sub>|
|
||||
|[`macho`](#macho) |Mach-O macOS executable |<sub></sub>|
|
||||
|`macho_fat` |Fat Mach-O macOS executable (multi-architecture) |<sub>`macho`</sub>|
|
||||
@ -131,7 +132,7 @@
|
||||
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|
||||
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `ipv4_packet` `ipv6_packet` `sll2_packet` `sll_packet`</sub>|
|
||||
|`mp3_frame_tags` |Group |<sub>`mp3_frame_vbri` `mp3_frame_xing`</sub>|
|
||||
|`probe` |Group |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `flac` `gif` `gzip` `html` `jpeg` `json` `jsonl` `luajit` `macho` `macho_fat` `matroska` `moc3` `mp3` `mp4` `mpeg_ts` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|
||||
|`probe` |Group |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `flac` `gif` `gzip` `html` `jpeg` `json` `jsonl` `leveldb_ldb` `luajit` `macho` `macho_fat` `matroska` `moc3` `mp3` `mp4` `mpeg_ts` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|
||||
|`tcp_stream` |Group |<sub>`dns_tcp` `rtmp` `tls`</sub>|
|
||||
|`udp_payload` |Group |<sub>`dns`</sub>|
|
||||
|
||||
@ -690,6 +691,23 @@ $ fq -n -d html '[inputs | {key: input_filename, value: .html.head.title?}] | fr
|
||||
$ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href' file.html
|
||||
```
|
||||
|
||||
## leveldb_ldb
|
||||
|
||||
### Limitations
|
||||
|
||||
- no Meta Blocks (like "filter") are decoded yet.
|
||||
- Zstandard uncompression is not implemented yet.
|
||||
|
||||
### Authors
|
||||
|
||||
- [@mikez](https://github.com/mikez), original author
|
||||
|
||||
### References
|
||||
|
||||
- https://github.com/google/leveldb/blob/main/doc/table_format.md
|
||||
- https://github.com/google/leveldb/blob/main/doc/impl.md
|
||||
- https://github.com/google/leveldb/blob/main/doc/index.md
|
||||
|
||||
## luajit
|
||||
|
||||
### Authors
|
||||
|
@ -30,7 +30,7 @@ import (
|
||||
_ "github.com/wader/fq/format/inet"
|
||||
_ "github.com/wader/fq/format/jpeg"
|
||||
_ "github.com/wader/fq/format/json"
|
||||
_ "github.com/wader/fq/format/ldb"
|
||||
_ "github.com/wader/fq/format/leveldb"
|
||||
_ "github.com/wader/fq/format/luajit"
|
||||
_ "github.com/wader/fq/format/markdown"
|
||||
_ "github.com/wader/fq/format/math"
|
||||
|
@ -125,7 +125,7 @@ var (
|
||||
JPEG = &decode.Group{Name: "jpeg"}
|
||||
JSON = &decode.Group{Name: "json"}
|
||||
JSONL = &decode.Group{Name: "jsonl"}
|
||||
LDB = &decode.Group{Name: "ldb"}
|
||||
LDB = &decode.Group{Name: "leveldb_ldb"}
|
||||
LuaJIT = &decode.Group{Name: "luajit"}
|
||||
MachO = &decode.Group{Name: "macho"}
|
||||
MachO_Fat = &decode.Group{Name: "macho_fat"}
|
||||
|
@ -1,4 +1,4 @@
|
||||
package ldb
|
||||
package leveldb
|
||||
|
||||
// https://github.com/google/leveldb/blob/main/doc/table_format.md
|
||||
// https://github.com/google/leveldb/blob/main/doc/impl.md
|
||||
@ -6,8 +6,7 @@ package ldb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"embed"
|
||||
"hash/crc32"
|
||||
|
||||
"github.com/golang/snappy"
|
||||
@ -18,6 +17,9 @@ import (
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
|
||||
//go:embed leveldb_ldb.md
|
||||
var leveldbFS embed.FS
|
||||
|
||||
func init() {
|
||||
interp.RegisterFormat(
|
||||
format.LDB,
|
||||
@ -26,19 +28,18 @@ func init() {
|
||||
Groups: []*decode.Group{format.Probe},
|
||||
DecodeFn: ldbDecode,
|
||||
})
|
||||
interp.RegisterFS(leveldbFS)
|
||||
}
|
||||
|
||||
const (
|
||||
// four varints (each max 10 bytes) + magic number (8 bytes)
|
||||
// https://github.com/google/leveldb/blob/main/table/format.h#L53
|
||||
footerEncodedLength = 4*10 + 8
|
||||
footerEncodedLength = (4*10 + 8) * 8
|
||||
magicNumberLength = 8 * 8
|
||||
// leading 64 bits of
|
||||
// echo http://code.google.com/p/leveldb/ | sha1sum
|
||||
// https://github.com/google/leveldb/blob/main/table/format.h#L76
|
||||
tableMagicNumber = 0xdb4775248b80fb57
|
||||
// 1-byte compression type + 4-bytes CRC
|
||||
// https://github.com/google/leveldb/blob/main/table/format.h#L79
|
||||
blockTrailerSize = 5
|
||||
)
|
||||
|
||||
// https://github.com/google/leveldb/blob/main/include/leveldb/options.h#L25
|
||||
@ -50,8 +51,8 @@ const (
|
||||
|
||||
var compressionTypes = scalar.UintMapSymStr{
|
||||
compressionTypeNone: "none",
|
||||
compressionTypeSnappy: "Snappy",
|
||||
compressionTypeZstandard: "Zstandard",
|
||||
compressionTypeSnappy: "snappy",
|
||||
compressionTypeZstandard: "zstd",
|
||||
}
|
||||
|
||||
// https://github.com/google/leveldb/blob/main/db/dbformat.h#L54
|
||||
@ -70,22 +71,24 @@ func ldbDecode(d *decode.D) any {
|
||||
|
||||
// footer
|
||||
|
||||
d.SeekAbs(d.Len() - footerEncodedLength*8)
|
||||
var indexOffset int64
|
||||
var indexSize int64
|
||||
var metaIndexOffset int64
|
||||
var metaIndexSize int64
|
||||
|
||||
d.SeekAbs(d.Len() - footerEncodedLength)
|
||||
d.FieldStruct("footer", func(d *decode.D) {
|
||||
d.FieldStruct("metaindex_handle", func(d *decode.D) {
|
||||
metaIndexOffset = int64(d.FieldUintFn("offset", decodeVarInt))
|
||||
metaIndexSize = int64(d.FieldUintFn("size", decodeVarInt))
|
||||
handleLength := d.LimitedFn(footerEncodedLength, func(d *decode.D) {
|
||||
d.FieldStruct("metaindex_handle", func(d *decode.D) {
|
||||
metaIndexOffset = int64(d.FieldULEB128("offset"))
|
||||
metaIndexSize = int64(d.FieldULEB128("size"))
|
||||
})
|
||||
d.FieldStruct("index_handle", func(d *decode.D) {
|
||||
indexOffset = int64(d.FieldULEB128("offset"))
|
||||
indexSize = int64(d.FieldULEB128("size"))
|
||||
})
|
||||
})
|
||||
d.FieldStruct("index_handle", func(d *decode.D) {
|
||||
indexOffset = int64(d.FieldUintFn("offset", decodeVarInt))
|
||||
indexSize = int64(d.FieldUintFn("size", decodeVarInt))
|
||||
})
|
||||
d.FieldRawLen("padding", d.Len()-d.Pos()-8*8)
|
||||
d.FieldRawLen("padding", footerEncodedLength-handleLength-magicNumberLength)
|
||||
d.FieldU64("magic_number", d.UintAssert(tableMagicNumber), scalar.UintHex)
|
||||
})
|
||||
|
||||
@ -93,12 +96,12 @@ func ldbDecode(d *decode.D) any {
|
||||
|
||||
d.SeekAbs(metaIndexOffset * 8)
|
||||
var metaHandles []BlockHandle
|
||||
fieldStructBlock("metaindex", metaIndexSize, readKeyValueContent, func(d *decode.D) {
|
||||
readBlock("metaindex", metaIndexSize, readKeyValueContent, func(d *decode.D) {
|
||||
// BlockHandle
|
||||
// https://github.com/google/leveldb/blob/main/table/format.cc#L24
|
||||
handle := BlockHandle{
|
||||
Offset: d.FieldUintFn("offset", decodeVarInt),
|
||||
Size: d.FieldUintFn("size", decodeVarInt),
|
||||
Offset: d.FieldULEB128("offset"),
|
||||
Size: d.FieldULEB128("size"),
|
||||
}
|
||||
metaHandles = append(metaHandles, handle)
|
||||
}, d)
|
||||
@ -107,12 +110,12 @@ func ldbDecode(d *decode.D) any {
|
||||
|
||||
d.SeekAbs(indexOffset * 8)
|
||||
var dataHandles []BlockHandle
|
||||
fieldStructBlock("index", indexSize, readKeyValueContent, func(d *decode.D) {
|
||||
readBlock("index", indexSize, readKeyValueContent, func(d *decode.D) {
|
||||
// BlockHandle
|
||||
// https://github.com/google/leveldb/blob/main/table/format.cc#L24
|
||||
handle := BlockHandle{
|
||||
Offset: d.FieldUintFn("offset", decodeVarInt),
|
||||
Size: d.FieldUintFn("size", decodeVarInt),
|
||||
Offset: d.FieldULEB128("offset"),
|
||||
Size: d.FieldULEB128("size"),
|
||||
}
|
||||
dataHandles = append(dataHandles, handle)
|
||||
}, d)
|
||||
@ -123,7 +126,7 @@ func ldbDecode(d *decode.D) any {
|
||||
d.FieldArray("meta", func(d *decode.D) {
|
||||
for _, handle := range metaHandles {
|
||||
d.SeekAbs(int64(handle.Offset) * 8)
|
||||
fieldStructBlock("meta_block", int64(handle.Size), readMetaContent, nil, d)
|
||||
readBlock("meta_block", int64(handle.Size), readMetaContent, nil, d)
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -134,7 +137,7 @@ func ldbDecode(d *decode.D) any {
|
||||
d.FieldArray("data", func(d *decode.D) {
|
||||
for _, handle := range dataHandles {
|
||||
d.SeekAbs(int64(handle.Offset) * 8)
|
||||
fieldStructBlock("data_block", int64(handle.Size), readKeyValueContent, nil, d)
|
||||
readBlock("data_block", int64(handle.Size), readKeyValueContent, nil, d)
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -142,20 +145,21 @@ func ldbDecode(d *decode.D) any {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Helpers
|
||||
// Readers
|
||||
|
||||
func fieldStructBlock(name string, size int64, readBlockContent func(size int64, valueCallbackFn func(d *decode.D), d *decode.D), valueCallbackFn func(d *decode.D), d *decode.D) *decode.D {
|
||||
func readBlock(name string, size int64, readBlockContent func(size int64, valueCallbackFn func(d *decode.D), d *decode.D), valueCallbackFn func(d *decode.D), d *decode.D) {
|
||||
// ReadBlock: https://github.com/google/leveldb/blob/main/table/format.cc#L69
|
||||
return d.FieldStruct(name, func(d *decode.D) {
|
||||
d.FieldStruct(name, func(d *decode.D) {
|
||||
start := d.Pos()
|
||||
br := d.RawLen(size * 8)
|
||||
// compression (1 byte)
|
||||
compressionType := d.FieldU8("compression", compressionTypes, scalar.UintHex)
|
||||
// validate crc
|
||||
// crc (4 bytes)
|
||||
data := d.ReadAllBits(br)
|
||||
bytesToCheck := append(data, uint8(compressionType))
|
||||
maskedCRCInt := maskedCrc32(bytesToCheck)
|
||||
d.FieldU32("crc", d.UintAssert(uint64(maskedCRCInt)), scalar.UintHex)
|
||||
|
||||
// decompress if needed
|
||||
d.SeekAbs(start)
|
||||
if compressionType == compressionTypeNone {
|
||||
d.FieldStruct("uncompressed", func(d *decode.D) {
|
||||
@ -165,7 +169,6 @@ func fieldStructBlock(name string, size int64, readBlockContent func(size int64,
|
||||
compressedSize := size
|
||||
compressed := data
|
||||
bb := &bytes.Buffer{}
|
||||
_ = bb
|
||||
switch compressionType {
|
||||
case compressionTypeSnappy:
|
||||
decompressed, err := snappy.Decode(nil, compressed)
|
||||
@ -206,7 +209,7 @@ func readKeyValueContent(size int64, valueCallbackFn func(d *decode.D), d *decod
|
||||
})
|
||||
})
|
||||
// TK: how do you make an empty entries-array appear _above_ the trailer?
|
||||
// Right now, its omited if empty.
|
||||
// Right now, its omitted if empty.
|
||||
if restartOffset <= 0 {
|
||||
return
|
||||
}
|
||||
@ -214,9 +217,9 @@ func readKeyValueContent(size int64, valueCallbackFn func(d *decode.D), d *decod
|
||||
d.FieldArray("entries", func(d *decode.D) {
|
||||
for d.Pos() < start+restartOffset {
|
||||
d.FieldStruct("entry", func(d *decode.D) {
|
||||
d.FieldUintFn("shared_bytes", decodeVarInt)
|
||||
unshared := int64(d.FieldUintFn("unshared_bytes", decodeVarInt))
|
||||
valueLength := d.FieldUintFn("value_length", decodeVarInt)
|
||||
d.FieldULEB128("shared_bytes")
|
||||
unshared := int64(d.FieldULEB128("unshared_bytes"))
|
||||
valueLength := d.FieldULEB128("value_length")
|
||||
// InternalKey
|
||||
// https://github.com/google/leveldb/blob/main/db/dbformat.h#L171
|
||||
d.FieldStruct("key_delta", func(d *decode.D) {
|
||||
@ -241,29 +244,7 @@ func readMetaContent(size int64, valueCallbackFn func(d *decode.D), d *decode.D)
|
||||
d.FieldRawLen("raw", size*8)
|
||||
}
|
||||
|
||||
func decodeVarInt(d *decode.D) uint64 {
|
||||
var value uint64 = 0
|
||||
var shift uint64 = 0
|
||||
|
||||
for {
|
||||
b := d.U8()
|
||||
value |= (b & 0b01111111) << shift
|
||||
shift += 7
|
||||
if b&0b10000000 == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
// Return a masked representation of the crc.
|
||||
// https://github.com/google/leveldb/blob/main/util/crc32c.h#L29
|
||||
func mask(crc uint32) uint32 {
|
||||
const kMaskDelta = 0xa282ead8
|
||||
// Rotate right by 15 bits and add a constant.
|
||||
return ((crc >> 15) | (crc << 17)) + kMaskDelta
|
||||
}
|
||||
// Helpers
|
||||
|
||||
func maskedCrc32(bytes []uint8) uint32 {
|
||||
crc32C := crc32.New(crc32.MakeTable(crc32.Castagnoli))
|
||||
@ -271,9 +252,10 @@ func maskedCrc32(bytes []uint8) uint32 {
|
||||
return mask(crc32C.Sum32())
|
||||
}
|
||||
|
||||
// Print the hexadecimal representation in little-endian format.
|
||||
func printLE(name string, value uint32) {
|
||||
buf := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(buf, value)
|
||||
fmt.Printf("%s: % x\n", name, buf)
|
||||
// Return a masked representation of a CRC.
|
||||
// https://github.com/google/leveldb/blob/main/util/crc32c.h#L29
|
||||
func mask(crc uint32) uint32 {
|
||||
const kMaskDelta = 0xa282ead8
|
||||
// Rotate right by 15 bits and add a constant.
|
||||
return ((crc >> 15) | (crc << 17)) + kMaskDelta
|
||||
}
|
14
format/leveldb/leveldb_ldb.md
Normal file
14
format/leveldb/leveldb_ldb.md
Normal file
@ -0,0 +1,14 @@
|
||||
### Limitations
|
||||
|
||||
- no Meta Blocks (like "filter") are decoded yet.
|
||||
- Zstandard uncompression is not implemented yet.
|
||||
|
||||
### Authors
|
||||
|
||||
- [@mikez](https://github.com/mikez), original author
|
||||
|
||||
### References
|
||||
|
||||
- https://github.com/google/leveldb/blob/main/doc/table_format.md
|
||||
- https://github.com/google/leveldb/blob/main/doc/impl.md
|
||||
- https://github.com/google/leveldb/blob/main/doc/index.md
|
25
format/leveldb/testdata/help_leveldb_ldb.fqtest
vendored
Normal file
25
format/leveldb/testdata/help_leveldb_ldb.fqtest
vendored
Normal file
@ -0,0 +1,25 @@
|
||||
$ fq -h leveldb_ldb
|
||||
leveldb_ldb: LevelDB Table decoder
|
||||
|
||||
Decode examples
|
||||
===============
|
||||
|
||||
# Decode file as leveldb_ldb
|
||||
$ fq -d leveldb_ldb . file
|
||||
# Decode value as leveldb_ldb
|
||||
... | leveldb_ldb
|
||||
|
||||
Limitations
|
||||
===========
|
||||
- no Meta Blocks (like "filter") are decoded yet.
|
||||
- Zstandard uncompression is not implemented yet.
|
||||
|
||||
Authors
|
||||
=======
|
||||
- @mikez (https://github.com/mikez), original author
|
||||
|
||||
References
|
||||
==========
|
||||
- https://github.com/google/leveldb/blob/main/doc/table_format.md
|
||||
- https://github.com/google/leveldb/blob/main/doc/impl.md
|
||||
- https://github.com/google/leveldb/blob/main/doc/index.md
|
102
format/leveldb/testdata/ldb_snappy.fqtest
vendored
Normal file
102
format/leveldb/testdata/ldb_snappy.fqtest
vendored
Normal file
@ -0,0 +1,102 @@
|
||||
$ fq -d leveldb_ldb dv snappy.ldb/000005.ldb
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: snappy.ldb/000005.ldb (leveldb_ldb) 0x0-0x2c4 (708)
|
||||
| | | data[0:1]: 0x0-0x26b (619)
|
||||
| | | [0]{}: data_block 0x0-0x26b (619)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| uncompressed{}: 0x0-0x5fc (1532)
|
||||
| | | entries[0:4]: 0x0-0x5f4 (1524)
|
||||
| | | [0]{}: entry 0x0-0x1d4 (468)
|
||||
0x000|00 |. | shared_bytes: 0 0x0-0x1 (1)
|
||||
0x000| 13 | . | unshared_bytes: 19 0x1-0x2 (1)
|
||||
0x000| bd 03 | .. | value_length: 445 0x2-0x4 (2)
|
||||
| | | key_delta{}: 0x4-0x17 (19)
|
||||
0x000| 6c 6f 72 65 6d 2e 64 6f 6c 6f 72 | lorem.dolor | user_key: "lorem.dolor" 0x4-0xf (11)
|
||||
0x000| 01| .| type: "value" (0x1) 0xf-0x10 (1)
|
||||
0x001|03 00 00 00 00 00 00 |....... | sequence_number: 3 0x10-0x17 (7)
|
||||
0x001| 4c 6f 72 65 6d 20 69 70 73| Lorem ips| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x17-0x1d4 (445)
|
||||
0x002|75 6d 20 64 6f 6c 6f 72 20 73 69 74 20 61 6d 65|um dolor sit ame|
|
||||
* |until 0x1d3.7 (445) | |
|
||||
| | | [1]{}: entry 0x1d4-0x3a2 (462)
|
||||
0x01d| 06 | . | shared_bytes: 6 0x1d4-0x1d5 (1)
|
||||
0x01d| 0d | . | unshared_bytes: 13 0x1d5-0x1d6 (1)
|
||||
0x01d| bd 03 | .. | value_length: 445 0x1d6-0x1d8 (2)
|
||||
| | | key_delta{}: 0x1d8-0x1e5 (13)
|
||||
0x01d| 69 70 73 75 6d | ipsum | user_key: "ipsum" 0x1d8-0x1dd (5)
|
||||
0x01d| 01 | . | type: "value" (0x1) 0x1dd-0x1de (1)
|
||||
0x01d| 02 00| ..| sequence_number: 2 0x1de-0x1e5 (7)
|
||||
0x01e|00 00 00 00 00 |..... |
|
||||
0x01e| 4c 6f 72 65 6d 20 69 70 73 75 6d| Lorem ipsum| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x1e5-0x3a2 (445)
|
||||
0x01f|20 64 6f 6c 6f 72 20 73 69 74 20 61 6d 65 74 2c| dolor sit amet,|
|
||||
* |until 0x3a1.7 (445) | |
|
||||
| | | [2]{}: entry 0x3a2-0x570 (462)
|
||||
0x03a| 06 | . | shared_bytes: 6 0x3a2-0x3a3 (1)
|
||||
0x03a| 0d | . | unshared_bytes: 13 0x3a3-0x3a4 (1)
|
||||
0x03a| bd 03 | .. | value_length: 445 0x3a4-0x3a6 (2)
|
||||
| | | key_delta{}: 0x3a6-0x3b3 (13)
|
||||
0x03a| 6c 6f 72 65 6d | lorem | user_key: "lorem" 0x3a6-0x3ab (5)
|
||||
0x03a| 01 | . | type: "value" (0x1) 0x3ab-0x3ac (1)
|
||||
0x03a| 01 00 00 00| ....| sequence_number: 1 0x3ac-0x3b3 (7)
|
||||
0x03b|00 00 00 |... |
|
||||
0x03b| 4c 6f 72 65 6d 20 69 70 73 75 6d 20 64| Lorem ipsum d| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x3b3-0x570 (445)
|
||||
0x03c|6f 6c 6f 72 20 73 69 74 20 61 6d 65 74 2c 20 63|olor sit amet, c|
|
||||
* |until 0x56f.7 (445) | |
|
||||
| | | [3]{}: entry 0x570-0x5f4 (132)
|
||||
0x057|00 |. | shared_bytes: 0 0x570-0x571 (1)
|
||||
0x057| 0b | . | unshared_bytes: 11 0x571-0x572 (1)
|
||||
0x057| 76 | v | value_length: 118 0x572-0x573 (1)
|
||||
| | | key_delta{}: 0x573-0x57e (11)
|
||||
0x057| 72 6f 77 | row | user_key: "row" 0x573-0x576 (3)
|
||||
0x057| 01 | . | type: "value" (0x1) 0x576-0x577 (1)
|
||||
0x057| 04 00 00 00 00 00 00 | ....... | sequence_number: 4 0x577-0x57e (7)
|
||||
0x057| 52 6f| Ro| value: "Row, row, row your boat\nGently down the stream...." 0x57e-0x5f4 (118)
|
||||
0x058|77 2c 20 72 6f 77 2c 20 72 6f 77 20 79 6f 75 72|w, row, row your|
|
||||
* |until 0x5f3.7 (118) | |
|
||||
| | | trailer{}: 0x5f4-0x5fc (8)
|
||||
| | | restarts[0:1]: 0x5f4-0x5f8 (4)
|
||||
0x05f| 00 00 00 00 | .... | [0]: 0 restart 0x5f4-0x5f8 (4)
|
||||
0x05f| 01 00 00 00| | ....| | num_restarts: 1 0x5f8-0x5fc (4)
|
||||
0x00000|fc 0b 44 00 13 bd 03 6c 6f 72 65 6d 2e 64 6f 6c|..D....lorem.dol| compressed: raw bits 0x0-0x266 (614)
|
||||
* |until 0x265.7 (614) | |
|
||||
0x00260| 01 | . | compression: "snappy" (0x1) 0x266-0x267 (1)
|
||||
0x00260| b6 9d 28 0d | ..(. | crc: 0xd289db6 (valid) 0x267-0x26b (4)
|
||||
| | | metaindex{}: 0x26b-0x278 (13)
|
||||
| | | uncompressed{}: 0x26b-0x273 (8)
|
||||
| | | trailer{}: 0x26b-0x273 (8)
|
||||
| | | restarts[0:1]: 0x26b-0x26f (4)
|
||||
0x00260| 00 00 00 00 | .... | [0]: 0 restart 0x26b-0x26f (4)
|
||||
0x00260| 01| .| num_restarts: 1 0x26f-0x273 (4)
|
||||
0x00270|00 00 00 |... |
|
||||
0x00270| 00 | . | compression: "none" (0x0) 0x273-0x274 (1)
|
||||
0x00270| c0 f2 a1 b0 | .... | crc: 0xb0a1f2c0 (valid) 0x274-0x278 (4)
|
||||
| | | index{}: 0x278-0x294 (28)
|
||||
| | | uncompressed{}: 0x278-0x28f (23)
|
||||
| | | entries[0:1]: 0x278-0x287 (15)
|
||||
| | | [0]{}: entry 0x278-0x287 (15)
|
||||
0x00270| 00 | . | shared_bytes: 0 0x278-0x279 (1)
|
||||
0x00270| 09 | . | unshared_bytes: 9 0x279-0x27a (1)
|
||||
0x00270| 03 | . | value_length: 3 0x27a-0x27b (1)
|
||||
| | | key_delta{}: 0x27b-0x284 (9)
|
||||
0x00270| 73 | s | user_key: "s" 0x27b-0x27c (1)
|
||||
0x00270| 01 | . | type: "value" (0x1) 0x27c-0x27d (1)
|
||||
0x00270| ff ff ff| ...| sequence_number: 72057594037927935 0x27d-0x284 (7)
|
||||
0x00280|ff ff ff ff |.... |
|
||||
| | | value{}: 0x284-0x287 (3)
|
||||
0x00280| 00 | . | offset: 0 0x284-0x285 (1)
|
||||
0x00280| e6 04 | .. | size: 614 0x285-0x287 (2)
|
||||
| | | trailer{}: 0x287-0x28f (8)
|
||||
| | | restarts[0:1]: 0x287-0x28b (4)
|
||||
0x00280| 00 00 00 00 | .... | [0]: 0 restart 0x287-0x28b (4)
|
||||
0x00280| 01 00 00 00 | .... | num_restarts: 1 0x28b-0x28f (4)
|
||||
0x00280| 00| .| compression: "none" (0x0) 0x28f-0x290 (1)
|
||||
0x00290|68 24 42 91 |h$B. | crc: 0x91422468 (valid) 0x290-0x294 (4)
|
||||
| | | footer{}: 0x294-0x2c4 (48)
|
||||
| | | metaindex_handle{}: 0x294-0x297 (3)
|
||||
0x00290| eb 04 | .. | offset: 619 0x294-0x296 (2)
|
||||
0x00290| 08 | . | size: 8 0x296-0x297 (1)
|
||||
| | | index_handle{}: 0x297-0x29a (3)
|
||||
0x00290| f8 04 | .. | offset: 632 0x297-0x299 (2)
|
||||
0x00290| 17 | . | size: 23 0x299-0x29a (1)
|
||||
0x00290| 00 00 00 00 00 00| ......| padding: raw bits 0x29a-0x2bc (34)
|
||||
0x002a0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
|
||||
0x002b0|00 00 00 00 00 00 00 00 00 00 00 00 |............ |
|
||||
0x002b0| 57 fb 80 8b| W...| magic_number: 0xdb4775248b80fb57 (valid) 0x2bc-0x2c4 (8)
|
||||
0x002c0|24 75 47 db| |$uG.| |
|
@ -1,5 +1,5 @@
|
||||
$ fq -d ldb dv uncompressed.ldb/000005.ldb
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: uncompressed.ldb/000005.ldb (ldb) 0x0-0x65a (1626)
|
||||
$ fq -d leveldb_ldb dv uncompressed.ldb/000005.ldb
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: uncompressed.ldb/000005.ldb (leveldb_ldb) 0x0-0x65a (1626)
|
||||
| | | data[0:1]: 0x0-0x601 (1537)
|
||||
| | | [0]{}: data_block 0x0-0x601 (1537)
|
||||
| | | uncompressed{}: 0x0-0x5fc (1532)
|
@ -938,7 +938,7 @@ func (d *D) FramedFn(nBits int64, fn func(d *D)) int64 {
|
||||
return decodeLen
|
||||
}
|
||||
|
||||
// LimitedFn decode from current position nBits forward. When done position will after last bit decoded.
|
||||
// LimitedFn decode from current position nBits forward. When done position will be after last bit decoded.
|
||||
func (d *D) LimitedFn(nBits int64, fn func(d *D)) int64 {
|
||||
if nBits < 0 {
|
||||
d.Fatalf("%d nBits < 0", nBits)
|
||||
|
@ -247,10 +247,15 @@ func (d *D) tryBool() (bool, error) {
|
||||
return n == 1, nil
|
||||
}
|
||||
|
||||
// Unsigned LEB128, description from wasm spec
|
||||
// Unsigned LEB128, also known as "Base 128 Varint".
|
||||
//
|
||||
// Description from wasm spec:
|
||||
//
|
||||
// uN ::= n:byte => n (if n < 2^7 && n < 2^N)
|
||||
// n:byte m:u(N-7) => 2^7 * m + (n - 2^7) (if n >= 2^7 && N > 7)
|
||||
//
|
||||
// Varint description:
|
||||
// https://protobuf.dev/programming-guides/encoding/#varints
|
||||
func (d *D) tryULEB128() (uint64, error) {
|
||||
var result uint64
|
||||
var shift uint
|
||||
@ -260,8 +265,8 @@ func (d *D) tryULEB128() (uint64, error) {
|
||||
if shift >= 63 && b != 0 {
|
||||
return 0, fmt.Errorf("overflow when reading unsigned leb128, shift %d >= 63", shift)
|
||||
}
|
||||
result |= (b & 0x7f) << shift
|
||||
if b&0x80 == 0 {
|
||||
result |= (b & 0b01111111) << shift
|
||||
if b&0b10000000 == 0 {
|
||||
break
|
||||
}
|
||||
shift += 7
|
||||
|
Loading…
Reference in New Issue
Block a user