wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-09-11 12:05:39 +03:00
Code Issues Projects Releases Wiki Activity

leveldb: address PR comments

Browse Source
This commit is contained in:
Michael B. 2023-12-05 11:31:30 +01:00
parent efc59a8130
commit b05aa997ab
27 changed files with 220 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -102,6 +102,7 @@ ipv6_packet,
jpeg,
json,
jsonl,
[leveldb_ldb](doc/formats.md#leveldb_ldb),
[luajit](doc/formats.md#luajit),
[macho](doc/formats.md#macho),
macho_fat,

20
doc/formats.md
View File

@ -72,6 +72,7 @@
|`jpeg` |Joint&nbsp;Photographic&nbsp;Experts&nbsp;Group&nbsp;file |<sub>`exif` `icc_profile`</sub>|
|`json` |JavaScript&nbsp;Object&nbsp;Notation |<sub></sub>|
|`jsonl` |JavaScript&nbsp;Object&nbsp;Notation&nbsp;Lines |<sub></sub>|
|[`leveldb_ldb`](#leveldb_ldb) |LevelDB&nbsp;Table |<sub></sub>|
|[`luajit`](#luajit) |LuaJIT&nbsp;2.0&nbsp;bytecode |<sub></sub>|
|[`macho`](#macho) |Mach-O&nbsp;macOS&nbsp;executable |<sub></sub>|
|`macho_fat` |Fat&nbsp;Mach-O&nbsp;macOS&nbsp;executable&nbsp;(multi-architecture) |<sub>`macho`</sub>|
@ -131,7 +132,7 @@
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `ipv4_packet` `ipv6_packet` `sll2_packet` `sll_packet`</sub>|
|`mp3_frame_tags` |Group |<sub>`mp3_frame_vbri` `mp3_frame_xing`</sub>|
|`probe` |Group |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `flac` `gif` `gzip` `html` `jpeg` `json` `jsonl` `luajit` `macho` `macho_fat` `matroska` `moc3` `mp3` `mp4` `mpeg_ts` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|`probe` |Group |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `flac` `gif` `gzip` `html` `jpeg` `json` `jsonl` `leveldb_ldb` `luajit` `macho` `macho_fat` `matroska` `moc3` `mp3` `mp4` `mpeg_ts` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|`tcp_stream` |Group |<sub>`dns_tcp` `rtmp` `tls`</sub>|
|`udp_payload` |Group |<sub>`dns`</sub>|
@ -690,6 +691,23 @@ $ fq -n -d html '[inputs | {key: input_filename, value: .html.head.title?}] | fr
$ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href' file.html
```
## leveldb_ldb
### Limitations
- no Meta Blocks (like "filter") are decoded yet.
- Zstandard uncompression is not implemented yet.
### Authors
- [@mikez](https://github.com/mikez), original author
### References
- https://github.com/google/leveldb/blob/main/doc/table_format.md
- https://github.com/google/leveldb/blob/main/doc/impl.md
- https://github.com/google/leveldb/blob/main/doc/index.md
## luajit
### Authors

2
format/all/all.go
View File

@ -30,7 +30,7 @@ import (
_ "github.com/wader/fq/format/inet"
_ "github.com/wader/fq/format/jpeg"
_ "github.com/wader/fq/format/json"
_ "github.com/wader/fq/format/ldb"
_ "github.com/wader/fq/format/leveldb"
_ "github.com/wader/fq/format/luajit"
_ "github.com/wader/fq/format/markdown"
_ "github.com/wader/fq/format/math"

2
format/format.go
View File

@ -125,7 +125,7 @@ var (
JPEG = &decode.Group{Name: "jpeg"}
JSON = &decode.Group{Name: "json"}
JSONL = &decode.Group{Name: "jsonl"}
LDB = &decode.Group{Name: "ldb"}
LDB = &decode.Group{Name: "leveldb_ldb"}
LuaJIT = &decode.Group{Name: "luajit"}
MachO = &decode.Group{Name: "macho"}
MachO_Fat = &decode.Group{Name: "macho_fat"}

110
format/ldb/ldb.go → format/leveldb/leveldb_ldb.go
View File

@ -1,4 +1,4 @@
package ldb
package leveldb
// https://github.com/google/leveldb/blob/main/doc/table_format.md
// https://github.com/google/leveldb/blob/main/doc/impl.md
@ -6,8 +6,7 @@ package ldb
import (
"bytes"
"encoding/binary"
"fmt"
"embed"
"hash/crc32"
"github.com/golang/snappy"
@ -18,6 +17,9 @@ import (
"github.com/wader/fq/pkg/scalar"
)
//go:embed leveldb_ldb.md
var leveldbFS embed.FS
func init() {
interp.RegisterFormat(
format.LDB,
@ -26,19 +28,18 @@ func init() {
Groups: []*decode.Group{format.Probe},
DecodeFn: ldbDecode,
})
interp.RegisterFS(leveldbFS)
}
const (
// four varints (each max 10 bytes) + magic number (8 bytes)
// https://github.com/google/leveldb/blob/main/table/format.h#L53
footerEncodedLength = 4*10 + 8
footerEncodedLength = (4*10 + 8) * 8
magicNumberLength = 8 * 8
// leading 64 bits of
// echo http://code.google.com/p/leveldb/ | sha1sum
// https://github.com/google/leveldb/blob/main/table/format.h#L76
tableMagicNumber = 0xdb4775248b80fb57
// 1-byte compression type + 4-bytes CRC
// https://github.com/google/leveldb/blob/main/table/format.h#L79
blockTrailerSize = 5
)
// https://github.com/google/leveldb/blob/main/include/leveldb/options.h#L25
@ -50,8 +51,8 @@ const (
var compressionTypes = scalar.UintMapSymStr{
compressionTypeNone: "none",
compressionTypeSnappy: "Snappy",
compressionTypeZstandard: "Zstandard",
compressionTypeSnappy: "snappy",
compressionTypeZstandard: "zstd",
}
// https://github.com/google/leveldb/blob/main/db/dbformat.h#L54
@ -70,22 +71,24 @@ func ldbDecode(d *decode.D) any {
// footer
d.SeekAbs(d.Len() - footerEncodedLength*8)
var indexOffset int64
var indexSize int64
var metaIndexOffset int64
var metaIndexSize int64
d.SeekAbs(d.Len() - footerEncodedLength)
d.FieldStruct("footer", func(d *decode.D) {
d.FieldStruct("metaindex_handle", func(d *decode.D) {
metaIndexOffset = int64(d.FieldUintFn("offset", decodeVarInt))
metaIndexSize = int64(d.FieldUintFn("size", decodeVarInt))
handleLength := d.LimitedFn(footerEncodedLength, func(d *decode.D) {
d.FieldStruct("metaindex_handle", func(d *decode.D) {
metaIndexOffset = int64(d.FieldULEB128("offset"))
metaIndexSize = int64(d.FieldULEB128("size"))
})
d.FieldStruct("index_handle", func(d *decode.D) {
indexOffset = int64(d.FieldULEB128("offset"))
indexSize = int64(d.FieldULEB128("size"))
})
})
d.FieldStruct("index_handle", func(d *decode.D) {
indexOffset = int64(d.FieldUintFn("offset", decodeVarInt))
indexSize = int64(d.FieldUintFn("size", decodeVarInt))
})
d.FieldRawLen("padding", d.Len()-d.Pos()-8*8)
d.FieldRawLen("padding", footerEncodedLength-handleLength-magicNumberLength)
d.FieldU64("magic_number", d.UintAssert(tableMagicNumber), scalar.UintHex)
})
@ -93,12 +96,12 @@ func ldbDecode(d *decode.D) any {
d.SeekAbs(metaIndexOffset * 8)
var metaHandles []BlockHandle
fieldStructBlock("metaindex", metaIndexSize, readKeyValueContent, func(d *decode.D) {
readBlock("metaindex", metaIndexSize, readKeyValueContent, func(d *decode.D) {
// BlockHandle
// https://github.com/google/leveldb/blob/main/table/format.cc#L24
handle := BlockHandle{
Offset: d.FieldUintFn("offset", decodeVarInt),
Size: d.FieldUintFn("size", decodeVarInt),
Offset: d.FieldULEB128("offset"),
Size: d.FieldULEB128("size"),
}
metaHandles = append(metaHandles, handle)
}, d)
@ -107,12 +110,12 @@ func ldbDecode(d *decode.D) any {
d.SeekAbs(indexOffset * 8)
var dataHandles []BlockHandle
fieldStructBlock("index", indexSize, readKeyValueContent, func(d *decode.D) {
readBlock("index", indexSize, readKeyValueContent, func(d *decode.D) {
// BlockHandle
// https://github.com/google/leveldb/blob/main/table/format.cc#L24
handle := BlockHandle{
Offset: d.FieldUintFn("offset", decodeVarInt),
Size: d.FieldUintFn("size", decodeVarInt),
Offset: d.FieldULEB128("offset"),
Size: d.FieldULEB128("size"),
}
dataHandles = append(dataHandles, handle)
}, d)
@ -123,7 +126,7 @@ func ldbDecode(d *decode.D) any {
d.FieldArray("meta", func(d *decode.D) {
for _, handle := range metaHandles {
d.SeekAbs(int64(handle.Offset) * 8)
fieldStructBlock("meta_block", int64(handle.Size), readMetaContent, nil, d)
readBlock("meta_block", int64(handle.Size), readMetaContent, nil, d)
}
})
}
@ -134,7 +137,7 @@ func ldbDecode(d *decode.D) any {
d.FieldArray("data", func(d *decode.D) {
for _, handle := range dataHandles {
d.SeekAbs(int64(handle.Offset) * 8)
fieldStructBlock("data_block", int64(handle.Size), readKeyValueContent, nil, d)
readBlock("data_block", int64(handle.Size), readKeyValueContent, nil, d)
}
})
}
@ -142,20 +145,21 @@ func ldbDecode(d *decode.D) any {
return nil
}
// Helpers
// Readers
func fieldStructBlock(name string, size int64, readBlockContent func(size int64, valueCallbackFn func(d *decode.D), d *decode.D), valueCallbackFn func(d *decode.D), d *decode.D) *decode.D {
func readBlock(name string, size int64, readBlockContent func(size int64, valueCallbackFn func(d *decode.D), d *decode.D), valueCallbackFn func(d *decode.D), d *decode.D) {
// ReadBlock: https://github.com/google/leveldb/blob/main/table/format.cc#L69
return d.FieldStruct(name, func(d *decode.D) {
d.FieldStruct(name, func(d *decode.D) {
start := d.Pos()
br := d.RawLen(size * 8)
// compression (1 byte)
compressionType := d.FieldU8("compression", compressionTypes, scalar.UintHex)
// validate crc
// crc (4 bytes)
data := d.ReadAllBits(br)
bytesToCheck := append(data, uint8(compressionType))
maskedCRCInt := maskedCrc32(bytesToCheck)
d.FieldU32("crc", d.UintAssert(uint64(maskedCRCInt)), scalar.UintHex)
// decompress if needed
d.SeekAbs(start)
if compressionType == compressionTypeNone {
d.FieldStruct("uncompressed", func(d *decode.D) {
@ -165,7 +169,6 @@ func fieldStructBlock(name string, size int64, readBlockContent func(size int64,
compressedSize := size
compressed := data
bb := &bytes.Buffer{}
_ = bb
switch compressionType {
case compressionTypeSnappy:
decompressed, err := snappy.Decode(nil, compressed)
@ -206,7 +209,7 @@ func readKeyValueContent(size int64, valueCallbackFn func(d *decode.D), d *decod
})
})
// TK: how do you make an empty entries-array appear _above_ the trailer?
// Right now, its omited if empty.
// Right now, its omitted if empty.
if restartOffset <= 0 {
return
}
@ -214,9 +217,9 @@ func readKeyValueContent(size int64, valueCallbackFn func(d *decode.D), d *decod
d.FieldArray("entries", func(d *decode.D) {
for d.Pos() < start+restartOffset {
d.FieldStruct("entry", func(d *decode.D) {
d.FieldUintFn("shared_bytes", decodeVarInt)
unshared := int64(d.FieldUintFn("unshared_bytes", decodeVarInt))
valueLength := d.FieldUintFn("value_length", decodeVarInt)
d.FieldULEB128("shared_bytes")
unshared := int64(d.FieldULEB128("unshared_bytes"))
valueLength := d.FieldULEB128("value_length")
// InternalKey
// https://github.com/google/leveldb/blob/main/db/dbformat.h#L171
d.FieldStruct("key_delta", func(d *decode.D) {
@ -241,29 +244,7 @@ func readMetaContent(size int64, valueCallbackFn func(d *decode.D), d *decode.D)
d.FieldRawLen("raw", size*8)
}
func decodeVarInt(d *decode.D) uint64 {
var value uint64 = 0
var shift uint64 = 0
for {
b := d.U8()
value |= (b & 0b01111111) << shift
shift += 7
if b&0b10000000 == 0 {
break
}
}
return value
}
// Return a masked representation of the crc.
// https://github.com/google/leveldb/blob/main/util/crc32c.h#L29
func mask(crc uint32) uint32 {
const kMaskDelta = 0xa282ead8
// Rotate right by 15 bits and add a constant.
return ((crc >> 15) | (crc << 17)) + kMaskDelta
}
// Helpers
func maskedCrc32(bytes []uint8) uint32 {
crc32C := crc32.New(crc32.MakeTable(crc32.Castagnoli))
@ -271,9 +252,10 @@ func maskedCrc32(bytes []uint8) uint32 {
return mask(crc32C.Sum32())
}
// Print the hexadecimal representation in little-endian format.
func printLE(name string, value uint32) {
buf := make([]byte, 4)
binary.LittleEndian.PutUint32(buf, value)
fmt.Printf("%s: % x\n", name, buf)
// Return a masked representation of a CRC.
// https://github.com/google/leveldb/blob/main/util/crc32c.h#L29
func mask(crc uint32) uint32 {
const kMaskDelta = 0xa282ead8
// Rotate right by 15 bits and add a constant.
return ((crc >> 15) | (crc << 17)) + kMaskDelta
}

14
format/leveldb/leveldb_ldb.md Normal file
View File

@ -0,0 +1,14 @@
### Limitations
- no Meta Blocks (like "filter") are decoded yet.
- Zstandard uncompression is not implemented yet.
### Authors
- [@mikez](https://github.com/mikez), original author
### References
- https://github.com/google/leveldb/blob/main/doc/table_format.md
- https://github.com/google/leveldb/blob/main/doc/impl.md
- https://github.com/google/leveldb/blob/main/doc/index.md

25
format/leveldb/testdata/help_leveldb_ldb.fqtest vendored Normal file
View File

@ -0,0 +1,25 @@
$ fq -h leveldb_ldb
leveldb_ldb: LevelDB Table decoder
Decode examples
===============
# Decode file as leveldb_ldb
$ fq -d leveldb_ldb . file
# Decode value as leveldb_ldb
... | leveldb_ldb
Limitations
===========
- no Meta Blocks (like "filter") are decoded yet.
- Zstandard uncompression is not implemented yet.
Authors
=======
- @mikez (https://github.com/mikez), original author
References
==========
- https://github.com/google/leveldb/blob/main/doc/table_format.md
- https://github.com/google/leveldb/blob/main/doc/impl.md
- https://github.com/google/leveldb/blob/main/doc/index.md

102
format/leveldb/testdata/ldb_snappy.fqtest vendored Normal file
View File

@ -0,0 +1,102 @@
$ fq -d leveldb_ldb dv snappy.ldb/000005.ldb
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: snappy.ldb/000005.ldb (leveldb_ldb) 0x0-0x2c4 (708)
| | | data[0:1]: 0x0-0x26b (619)
| | | [0]{}: data_block 0x0-0x26b (619)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| uncompressed{}: 0x0-0x5fc (1532)
| | | entries[0:4]: 0x0-0x5f4 (1524)
| | | [0]{}: entry 0x0-0x1d4 (468)
0x000|00 |. | shared_bytes: 0 0x0-0x1 (1)
0x000| 13 | . | unshared_bytes: 19 0x1-0x2 (1)
0x000| bd 03 | .. | value_length: 445 0x2-0x4 (2)
| | | key_delta{}: 0x4-0x17 (19)
0x000| 6c 6f 72 65 6d 2e 64 6f 6c 6f 72 | lorem.dolor | user_key: "lorem.dolor" 0x4-0xf (11)
0x000| 01| .| type: "value" (0x1) 0xf-0x10 (1)
0x001|03 00 00 00 00 00 00 |....... | sequence_number: 3 0x10-0x17 (7)
0x001| 4c 6f 72 65 6d 20 69 70 73| Lorem ips| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x17-0x1d4 (445)
0x002|75 6d 20 64 6f 6c 6f 72 20 73 69 74 20 61 6d 65|um dolor sit ame|
* |until 0x1d3.7 (445) | |
| | | [1]{}: entry 0x1d4-0x3a2 (462)
0x01d| 06 | . | shared_bytes: 6 0x1d4-0x1d5 (1)
0x01d| 0d | . | unshared_bytes: 13 0x1d5-0x1d6 (1)
0x01d| bd 03 | .. | value_length: 445 0x1d6-0x1d8 (2)
| | | key_delta{}: 0x1d8-0x1e5 (13)
0x01d| 69 70 73 75 6d | ipsum | user_key: "ipsum" 0x1d8-0x1dd (5)
0x01d| 01 | . | type: "value" (0x1) 0x1dd-0x1de (1)
0x01d| 02 00| ..| sequence_number: 2 0x1de-0x1e5 (7)
0x01e|00 00 00 00 00 |..... |
0x01e| 4c 6f 72 65 6d 20 69 70 73 75 6d| Lorem ipsum| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x1e5-0x3a2 (445)
0x01f|20 64 6f 6c 6f 72 20 73 69 74 20 61 6d 65 74 2c| dolor sit amet,|
* |until 0x3a1.7 (445) | |
| | | [2]{}: entry 0x3a2-0x570 (462)
0x03a| 06 | . | shared_bytes: 6 0x3a2-0x3a3 (1)
0x03a| 0d | . | unshared_bytes: 13 0x3a3-0x3a4 (1)
0x03a| bd 03 | .. | value_length: 445 0x3a4-0x3a6 (2)
| | | key_delta{}: 0x3a6-0x3b3 (13)
0x03a| 6c 6f 72 65 6d | lorem | user_key: "lorem" 0x3a6-0x3ab (5)
0x03a| 01 | . | type: "value" (0x1) 0x3ab-0x3ac (1)
0x03a| 01 00 00 00| ....| sequence_number: 1 0x3ac-0x3b3 (7)
0x03b|00 00 00 |... |
0x03b| 4c 6f 72 65 6d 20 69 70 73 75 6d 20 64| Lorem ipsum d| value: "Lorem ipsum dolor sit amet, consectetur adipisc..." 0x3b3-0x570 (445)
0x03c|6f 6c 6f 72 20 73 69 74 20 61 6d 65 74 2c 20 63|olor sit amet, c|
* |until 0x56f.7 (445) | |
| | | [3]{}: entry 0x570-0x5f4 (132)
0x057|00 |. | shared_bytes: 0 0x570-0x571 (1)
0x057| 0b | . | unshared_bytes: 11 0x571-0x572 (1)
0x057| 76 | v | value_length: 118 0x572-0x573 (1)
| | | key_delta{}: 0x573-0x57e (11)
0x057| 72 6f 77 | row | user_key: "row" 0x573-0x576 (3)
0x057| 01 | . | type: "value" (0x1) 0x576-0x577 (1)
0x057| 04 00 00 00 00 00 00 | ....... | sequence_number: 4 0x577-0x57e (7)
0x057| 52 6f| Ro| value: "Row, row, row your boat\nGently down the stream...." 0x57e-0x5f4 (118)
0x058|77 2c 20 72 6f 77 2c 20 72 6f 77 20 79 6f 75 72|w, row, row your|
* |until 0x5f3.7 (118) | |
| | | trailer{}: 0x5f4-0x5fc (8)
| | | restarts[0:1]: 0x5f4-0x5f8 (4)
0x05f| 00 00 00 00 | .... | [0]: 0 restart 0x5f4-0x5f8 (4)
0x05f| 01 00 00 00| | ....| | num_restarts: 1 0x5f8-0x5fc (4)
0x00000|fc 0b 44 00 13 bd 03 6c 6f 72 65 6d 2e 64 6f 6c|..D....lorem.dol| compressed: raw bits 0x0-0x266 (614)
* |until 0x265.7 (614) | |
0x00260| 01 | . | compression: "snappy" (0x1) 0x266-0x267 (1)
0x00260| b6 9d 28 0d | ..(. | crc: 0xd289db6 (valid) 0x267-0x26b (4)
| | | metaindex{}: 0x26b-0x278 (13)
| | | uncompressed{}: 0x26b-0x273 (8)
| | | trailer{}: 0x26b-0x273 (8)
| | | restarts[0:1]: 0x26b-0x26f (4)
0x00260| 00 00 00 00 | .... | [0]: 0 restart 0x26b-0x26f (4)
0x00260| 01| .| num_restarts: 1 0x26f-0x273 (4)
0x00270|00 00 00 |... |
0x00270| 00 | . | compression: "none" (0x0) 0x273-0x274 (1)
0x00270| c0 f2 a1 b0 | .... | crc: 0xb0a1f2c0 (valid) 0x274-0x278 (4)
| | | index{}: 0x278-0x294 (28)
| | | uncompressed{}: 0x278-0x28f (23)
| | | entries[0:1]: 0x278-0x287 (15)
| | | [0]{}: entry 0x278-0x287 (15)
0x00270| 00 | . | shared_bytes: 0 0x278-0x279 (1)
0x00270| 09 | . | unshared_bytes: 9 0x279-0x27a (1)
0x00270| 03 | . | value_length: 3 0x27a-0x27b (1)
| | | key_delta{}: 0x27b-0x284 (9)
0x00270| 73 | s | user_key: "s" 0x27b-0x27c (1)
0x00270| 01 | . | type: "value" (0x1) 0x27c-0x27d (1)
0x00270| ff ff ff| ...| sequence_number: 72057594037927935 0x27d-0x284 (7)
0x00280|ff ff ff ff |.... |
| | | value{}: 0x284-0x287 (3)
0x00280| 00 | . | offset: 0 0x284-0x285 (1)
0x00280| e6 04 | .. | size: 614 0x285-0x287 (2)
| | | trailer{}: 0x287-0x28f (8)
| | | restarts[0:1]: 0x287-0x28b (4)
0x00280| 00 00 00 00 | .... | [0]: 0 restart 0x287-0x28b (4)
0x00280| 01 00 00 00 | .... | num_restarts: 1 0x28b-0x28f (4)
0x00280| 00| .| compression: "none" (0x0) 0x28f-0x290 (1)
0x00290|68 24 42 91 |h$B. | crc: 0x91422468 (valid) 0x290-0x294 (4)
| | | footer{}: 0x294-0x2c4 (48)
| | | metaindex_handle{}: 0x294-0x297 (3)
0x00290| eb 04 | .. | offset: 619 0x294-0x296 (2)
0x00290| 08 | . | size: 8 0x296-0x297 (1)
| | | index_handle{}: 0x297-0x29a (3)
0x00290| f8 04 | .. | offset: 632 0x297-0x299 (2)
0x00290| 17 | . | size: 23 0x299-0x29a (1)
0x00290| 00 00 00 00 00 00| ......| padding: raw bits 0x29a-0x2bc (34)
0x002a0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
0x002b0|00 00 00 00 00 00 00 00 00 00 00 00 |............ |
0x002b0| 57 fb 80 8b| W...| magic_number: 0xdb4775248b80fb57 (valid) 0x2bc-0x2c4 (8)
0x002c0|24 75 47 db| |$uG.| |

4
format/ldb/testdata/ldb.fqtest → format/leveldb/testdata/ldb_uncompressed.fqtest vendored
View File

@ -1,5 +1,5 @@
$ fq -d ldb dv uncompressed.ldb/000005.ldb
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: uncompressed.ldb/000005.ldb (ldb) 0x0-0x65a (1626)
$ fq -d leveldb_ldb dv uncompressed.ldb/000005.ldb
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: uncompressed.ldb/000005.ldb (leveldb_ldb) 0x0-0x65a (1626)
| | | data[0:1]: 0x0-0x601 (1537)
| | | [0]{}: data_block 0x0-0x601 (1537)
| | | uncompressed{}: 0x0-0x5fc (1532)

0
format/ldb/testdata/lorem.json → format/leveldb/testdata/lorem.json vendored
View File

0
format/ldb/testdata/make_ldb.py → format/leveldb/testdata/make_ldb.py vendored
View File

0
format/ldb/testdata/snappy.ldb/000005.ldb → format/leveldb/testdata/snappy.ldb/000005.ldb vendored
View File

0
format/ldb/testdata/snappy.ldb/000006.log → format/leveldb/testdata/snappy.ldb/000006.log vendored
View File

0
format/ldb/testdata/snappy.ldb/CURRENT → format/leveldb/testdata/snappy.ldb/CURRENT vendored
View File

0
format/ldb/testdata/snappy.ldb/LOCK → format/leveldb/testdata/snappy.ldb/LOCK vendored
View File

0
format/ldb/testdata/snappy.ldb/LOG → format/leveldb/testdata/snappy.ldb/LOG vendored
View File

0
format/ldb/testdata/snappy.ldb/LOG.old → format/leveldb/testdata/snappy.ldb/LOG.old vendored
View File

0
format/ldb/testdata/snappy.ldb/MANIFEST-000004 → format/leveldb/testdata/snappy.ldb/MANIFEST-000004 vendored
View File

0
format/ldb/testdata/uncompressed.ldb/000005.ldb → format/leveldb/testdata/uncompressed.ldb/000005.ldb vendored
View File

0
format/ldb/testdata/uncompressed.ldb/000006.log → format/leveldb/testdata/uncompressed.ldb/000006.log vendored
View File

0
format/ldb/testdata/uncompressed.ldb/CURRENT → format/leveldb/testdata/uncompressed.ldb/CURRENT vendored
View File

0
format/ldb/testdata/uncompressed.ldb/LOCK → format/leveldb/testdata/uncompressed.ldb/LOCK vendored
View File

0
format/ldb/testdata/uncompressed.ldb/LOG → format/leveldb/testdata/uncompressed.ldb/LOG vendored
View File

0
format/ldb/testdata/uncompressed.ldb/LOG.old → format/leveldb/testdata/uncompressed.ldb/LOG.old vendored
View File

0
format/ldb/testdata/uncompressed.ldb/MANIFEST-000004 → format/leveldb/testdata/uncompressed.ldb/MANIFEST-000004 vendored
View File

2
pkg/decode/decode.go
View File

@ -938,7 +938,7 @@ func (d *D) FramedFn(nBits int64, fn func(d *D)) int64 {
return decodeLen
}
// LimitedFn decode from current position nBits forward. When done position will after last bit decoded.
// LimitedFn decode from current position nBits forward. When done position will be after last bit decoded.
func (d *D) LimitedFn(nBits int64, fn func(d *D)) int64 {
if nBits < 0 {
d.Fatalf("%d nBits < 0", nBits)

11
pkg/decode/read.go
View File

@ -247,10 +247,15 @@ func (d *D) tryBool() (bool, error) {
return n == 1, nil
}
// Unsigned LEB128, description from wasm spec
// Unsigned LEB128, also known as "Base 128 Varint".
//
// Description from wasm spec:
//
// uN ::= n:byte => n (if n < 2^7 && n < 2^N)
// n:byte m:u(N-7) => 2^7 * m + (n - 2^7) (if n >= 2^7 && N > 7)
//
// Varint description:
// https://protobuf.dev/programming-guides/encoding/#varints
func (d *D) tryULEB128() (uint64, error) {
var result uint64
var shift uint
@ -260,8 +265,8 @@ func (d *D) tryULEB128() (uint64, error) {
if shift >= 63 && b != 0 {
return 0, fmt.Errorf("overflow when reading unsigned leb128, shift %d >= 63", shift)
}
result |= (b & 0x7f) << shift
if b&0x80 == 0 {
result |= (b & 0b01111111) << shift
if b&0b10000000 == 0 {
break
}
shift += 7