1
1
mirror of https://github.com/wader/fq.git synced 2024-12-18 02:41:44 +03:00

negentropy: add format.

This commit is contained in:
fiatjaf 2024-09-18 15:57:13 -03:00
parent f150085735
commit 49bd7c27fd
11 changed files with 1661 additions and 9 deletions

View File

@ -124,6 +124,7 @@ mpeg_pes_packet,
mpeg_spu,
mpeg_ts,
[msgpack](doc/formats.md#msgpack),
[negentropy](doc/formats.md#negentropy),
[nes](doc/formats.md#nes),
ogg,
ogg_page,

View File

@ -96,6 +96,7 @@
|`mpeg_spu` |Sub&nbsp;Picture&nbsp;Unit&nbsp;(DVD&nbsp;subtitle) |<sub></sub>|
|`mpeg_ts` |MPEG&nbsp;Transport&nbsp;Stream |<sub></sub>|
|[`msgpack`](#msgpack) |MessagePack |<sub></sub>|
|[`negentropy`](#negentropy) |Negentropy&nbsp;message |<sub></sub>|
|[`nes`](#nes) |iNES/NES&nbsp;2.0&nbsp;cartridge&nbsp;ROM&nbsp;format |<sub></sub>|
|`ogg` |OGG&nbsp;file |<sub>`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`</sub>|
|`ogg_page` |OGG&nbsp;page |<sub></sub>|
@ -140,7 +141,7 @@
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `ipv4_packet` `ipv6_packet` `sll2_packet` `sll_packet`</sub>|
|`mp3_frame_tags` |Group |<sub>`mp3_frame_vbri` `mp3_frame_xing`</sub>|
|`probe` |Group |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `fit` `flac` `gif` `gzip` `html` `jp2c` `jpeg` `json` `jsonl` `leveldb_table` `luajit` `macho` `macho_fat` `matroska` `moc3` `mp3` `mp4` `mpeg_ts` `nes` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `tzx` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|`probe` |Group |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `caff` `elf` `fit` `flac` `gif` `gzip` `html` `jp2c` `jpeg` `json` `jsonl` `leveldb_table` `luajit` `macho` `macho_fat` `matroska` `midi` `moc3` `mp3` `mp4` `mpeg_ts` `nes` `ogg` `opentimestamps` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `tzx` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|`tcp_stream` |Group |<sub>`dns_tcp` `rtmp` `tls`</sub>|
|`udp_payload` |Group |<sub>`dns`</sub>|
@ -873,37 +874,41 @@ Standard MIDI file.
### Notes
1. Only supports the MIDI 1.0 specification.
2. Does only basic validation on the MIDI data.
1. Only supports the MIDI 1.0 MIDI file specification.
2. Only supports _MThd_ and _MTrk_ chunks.
3. Does only basic validation on the MIDI data.
### Sample queries
1. Extract the track names from a MIDI file
```
fq -d midi -d midi '.. | select(.event=="Track Name")? | "\(.name)"' twinkle.mid
fq -d midi -d midi '.. | select(.event=="track_name")? | "\(.track_name)"' midi/twinkle.mid
```
2. Extract the tempo changes from a MIDI file
```
fq -d midi '.. | select(.event=="Tempo")?.tempo' twinkle.mid
fq -d midi '.. | select(.event=="tempo")?.tempo' midi/twinkle.mid
```
3. Extract the key changes from a MIDI file
```
fq -d midi '.. | select(.event=="Key Signature")?.key' key-signatures.mid
fq -d midi '.. | select(.event=="key_signature")?.key_signature' midi/twinkle.mid
```
4. Extract NoteOn and NoteOff events:
4. Extract NoteOn events:
```
fq -d midi 'grep_by(.event=="Note On" or .event=="Note Off") | "\(.event) \(.time.tick) \(.note)"' twinkle.mid
fq -d midi 'grep_by(.event=="note_on") | [.time.tick, .note_on.note] | join(" ")' midi/twinkle.mid
```
### Authors
- transcriptaze.development@gmail.com
- [transcriptaze](https://github.com/transcriptaze)
### References
1. [The Complete MIDI 1.0 Detailed Specification](https://www.midi.org/specifications/item/the-midi-1-0-specification)
2. [Standard MIDI Files](https://midi.org/standard-midi-files)
3. [Standard MIDI File (SMF) Format](http://midi.teragonaudio.com/tech/midifile.htm)
4. [MIDI Files Specification](http://www.somascape.org/midi/tech/mfile.html)
## moc3
MOC3 file.
@ -1012,6 +1017,33 @@ $ fq -d msgpack torepr file.msgpack
### References
- https://github.com/msgpack/msgpack/blob/master/spec.md
## negentropy
Negentropy message.
### View a full Negentropy message
```
$ fq -d negentropy dd file
```
### Check how many ranges the message has and how many of those are of 'fingerprint' mode
```
$ fq -d negentropy '.bounds | length as $total | map(select(.mode == "fingerprint")) | length | {$total, fingerprint: .}' message
```
### Check get all ids in all idlists
```
$ fq -d negentropy '.bounds | map(select(.mode == "idlist") | .idlist | .ids) | flatten' message
```
### Authors
- fiatjaf, https://fiatjaf.com
### References
- https://github.com/hoytech/negentropy
## nes
iNES/NES 2.0 cartridge ROM format.

View File

@ -141,6 +141,7 @@ mpeg_pes_packet MPEG Packetized elementary stream packet
mpeg_spu Sub Picture Unit (DVD subtitle)
mpeg_ts MPEG Transport Stream
msgpack MessagePack
negentropy Negentropy message
nes iNES/NES 2.0 cartridge ROM format
ogg OGG file
ogg_page OGG page

View File

@ -42,6 +42,7 @@ import (
_ "github.com/wader/fq/format/mp4"
_ "github.com/wader/fq/format/mpeg"
_ "github.com/wader/fq/format/msgpack"
_ "github.com/wader/fq/format/negentropy"
_ "github.com/wader/fq/format/nes"
_ "github.com/wader/fq/format/ogg"
_ "github.com/wader/fq/format/opentimestamps"

View File

@ -148,6 +148,7 @@ var (
MPEG_TS = &decode.Group{Name: "mpeg_ts"}
MPES_PES = &decode.Group{Name: "mpeg_pes"}
MsgPack = &decode.Group{Name: "msgpack"}
Negentropy = &decode.Group{Name: "negentropy"}
NES = &decode.Group{Name: "nes"}
Ogg = &decode.Group{Name: "ogg"}
Ogg_Page = &decode.Group{Name: "ogg_page"}

View File

@ -0,0 +1,116 @@
package negentropy
// https://github.com/hoytech/negentropy
import (
"embed"
"math"
"time"
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
)
//go:embed negentropy.md
var negFS embed.FS
func init() {
interp.RegisterFormat(
format.Negentropy,
&decode.Format{
Description: "Negentropy message",
DecodeFn: decodeNegentropyMessage,
Groups: []*decode.Group{},
})
interp.RegisterFS(negFS)
}
const (
version = 0x61
fingerprintSize = 16
)
var modeMapper = scalar.SintMapSymStr{
0: "skip",
1: "fingerprint",
2: "idlist",
}
type timestampTranslator struct{ last time.Time }
func (tt *timestampTranslator) MapSint(s scalar.Sint) (scalar.Sint, error) {
if s.Actual == 0 {
s.Description = "infinity"
s.Actual = -1
tt.last = time.Unix(math.MaxInt64, 0)
return s, nil
} else {
s.Actual--
timestamp := tt.last.Add(time.Second * time.Duration(s.Actual-1))
tt.last = timestamp
s.Description = timestamp.Format(time.DateTime)
return s, nil
}
}
func decodeNegentropyMessage(d *decode.D) any {
tt := &timestampTranslator{last: time.Unix(0, 0)}
d.Endian = decode.BigEndian
v := d.FieldU8("version")
if v != version {
d.Fatalf("unexpected version %d (expected %d), is this really a negentropy message?", v, version)
}
d.FieldStructArrayLoop("bounds", "bound", d.NotEnd, func(d *decode.D) {
d.FieldSintFn("timestamp_delta", decodeVarInt, tt)
size := d.FieldSintFn("id_prefix_size", decodeVarInt)
if size > 32 {
d.Fatalf("unexpected id prefix size bigger than 32: %d", size)
}
if size > 0 {
d.FieldRawLen("id_prefix", size*8, scalar.RawHex)
}
mode := d.FieldSintFn("mode", decodeVarInt, modeMapper)
switch mode {
case 0:
return
case 1:
d.FieldRawLen("fingerprint", fingerprintSize*8, scalar.RawHex)
return
case 2:
d.FieldStruct("idlist", func(d *decode.D) {
num := d.FieldSintFn("num", decodeVarInt)
d.FieldArray("ids", func(d *decode.D) {
for i := 0; i < int(num); i++ {
d.FieldRawLen("id", 32*8, scalar.RawHex)
}
})
})
default:
d.Fatalf("unexpected mode %d", mode)
}
})
return nil
}
func decodeVarInt(d *decode.D) int64 {
var res int64 = 0
for {
b := d.U8()
res = (res << 7) | (int64(b) & 127)
if (b & 128) == 0 {
break
}
}
return res
}

View File

@ -0,0 +1,23 @@
### View a full Negentropy message
```
$ fq -d negentropy dd file
```
### Check how many ranges the message has and how many of those are of 'fingerprint' mode
```
$ fq -d negentropy '.bounds | length as $total | map(select(.mode == "fingerprint")) | length | {$total, fingerprint: .}' message
```
### Check get all ids in all idlists
```
$ fq -d negentropy '.bounds | map(select(.mode == "idlist") | .idlist | .ids) | flatten' message
```
### Authors
- fiatjaf, https://fiatjaf.com
### References
- https://github.com/hoytech/negentropy

Binary file not shown.

File diff suppressed because it is too large Load Diff

Binary file not shown.

View File

@ -0,0 +1,110 @@
$ fq -d negentropy dv smalljustfingerprints
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: smalljustfingerprints (negentropy) 0x0-0x140 (320)
0x000|61 |a | version: 97 0x0-0x1 (1)
| | | bounds[0:16]: 0x1-0x140 (319)
| | | [0]{}: bound 0x1-0x18 (23)
0x000| 86 b7 ab b4 7c | ....| | timestamp_delta: 1726667387 (2024-09-18 13:49:46) 0x1-0x6 (5)
0x000| 00 | . | id_prefix_size: 0 0x6-0x7 (1)
0x000| 01 | . | mode: "fingerprint" (1) 0x7-0x8 (1)
0x000| 10 8e 42 06 82 8e e3 bf| ..B.....| fingerprint: "108e4206828ee3bf34258465809a337c" (raw bits) 0x8-0x18 (16)
0x010|34 25 84 65 80 9a 33 7c |4%.e..3| |
| | | [1]{}: bound 0x18-0x2b (19)
0x010| 6c | l | timestamp_delta: 107 (2024-09-18 13:51:32) 0x18-0x19 (1)
0x010| 00 | . | id_prefix_size: 0 0x19-0x1a (1)
0x010| 01 | . | mode: "fingerprint" (1) 0x1a-0x1b (1)
0x010| 9a 68 e3 7b 17| .h.{.| fingerprint: "9a68e37b177a50b3ae7164ccc628b962" (raw bits) 0x1b-0x2b (16)
0x020|7a 50 b3 ae 71 64 cc c6 28 b9 62 |zP..qd..(.b |
| | | [2]{}: bound 0x2b-0x3f (20)
0x020| 02 | . | timestamp_delta: 1 (2024-09-18 13:51:32) 0x2b-0x2c (1)
0x020| 01 | . | id_prefix_size: 1 0x2c-0x2d (1)
0x020| 14 | . | id_prefix: "14" (raw bits) 0x2d-0x2e (1)
0x020| 01 | . | mode: "fingerprint" (1) 0x2e-0x2f (1)
0x020| 9c| .| fingerprint: "9c1381281c9e3849d5fbd514b7bb65ad" (raw bits) 0x2f-0x3f (16)
0x030|13 81 28 1c 9e 38 49 d5 fb d5 14 b7 bb 65 ad |..(..8I......e. |
| | | [3]{}: bound 0x3f-0x53 (20)
0x030| 01| .| timestamp_delta: 0 (2024-09-18 13:51:31) 0x3f-0x40 (1)
0x040|01 |. | id_prefix_size: 1 0x40-0x41 (1)
0x040| e6 | . | id_prefix: "e6" (raw bits) 0x41-0x42 (1)
0x040| 01 | . | mode: "fingerprint" (1) 0x42-0x43 (1)
0x040| fb f7 45 1f 5d 22 e7 fa 36 ae 3e 91 0e| ..E.]"..6.>..| fingerprint: "fbf7451f5d22e7fa36ae3e910e9f5215" (raw bits) 0x43-0x53 (16)
0x050|9f 52 15 |.R. |
| | | [4]{}: bound 0x53-0x67 (20)
0x050| 02 | . | timestamp_delta: 1 (2024-09-18 13:51:31) 0x53-0x54 (1)
0x050| 01 | . | id_prefix_size: 1 0x54-0x55 (1)
0x050| 57 | W | id_prefix: "57" (raw bits) 0x55-0x56 (1)
0x050| 01 | . | mode: "fingerprint" (1) 0x56-0x57 (1)
0x050| 4a 1b 26 85 3e 06 e9 c3 2e| J.&.>....| fingerprint: "4a1b26853e06e9c32eb41b1df4f9ab30" (raw bits) 0x57-0x67 (16)
0x060|b4 1b 1d f4 f9 ab 30 |......0 |
| | | [5]{}: bound 0x67-0x7b (20)
0x060| 02 | . | timestamp_delta: 1 (2024-09-18 13:51:31) 0x67-0x68 (1)
0x060| 01 | . | id_prefix_size: 1 0x68-0x69 (1)
0x060| e6 | . | id_prefix: "e6" (raw bits) 0x69-0x6a (1)
0x060| 01 | . | mode: "fingerprint" (1) 0x6a-0x6b (1)
0x060| 18 40 e2 73 c8| .@.s.| fingerprint: "1840e273c84bb1344f1d4e15d9aa6792" (raw bits) 0x6b-0x7b (16)
0x070|4b b1 34 4f 1d 4e 15 d9 aa 67 92 |K.4O.N...g. |
| | | [6]{}: bound 0x7b-0x8e (19)
0x070| 02 | . | timestamp_delta: 1 (2024-09-18 13:51:31) 0x7b-0x7c (1)
0x070| 00 | . | id_prefix_size: 0 0x7c-0x7d (1)
0x070| 01 | . | mode: "fingerprint" (1) 0x7d-0x7e (1)
0x070| 6f 12| o.| fingerprint: "6f12ee2340888653f10b0ec2d438ac9f" (raw bits) 0x7e-0x8e (16)
0x080|ee 23 40 88 86 53 f1 0b 0e c2 d4 38 ac 9f |.#@..S.....8.. |
| | | [7]{}: bound 0x8e-0xa2 (20)
0x080| 01 | . | timestamp_delta: 0 (2024-09-18 13:51:30) 0x8e-0x8f (1)
0x080| 01| .| id_prefix_size: 1 0x8f-0x90 (1)
0x090|84 |. | id_prefix: "84" (raw bits) 0x90-0x91 (1)
0x090| 01 | . | mode: "fingerprint" (1) 0x91-0x92 (1)
0x090| 56 d2 d7 96 f4 df f0 04 ab 36 9b 9b cf a4| V........6....| fingerprint: "56d2d796f4dff004ab369b9bcfa4d81e" (raw bits) 0x92-0xa2 (16)
0x0a0|d8 1e |.. |
| | | [8]{}: bound 0xa2-0xb6 (20)
0x0a0| 02 | . | timestamp_delta: 1 (2024-09-18 13:51:30) 0xa2-0xa3 (1)
0x0a0| 01 | . | id_prefix_size: 1 0xa3-0xa4 (1)
0x0a0| 87 | . | id_prefix: "87" (raw bits) 0xa4-0xa5 (1)
0x0a0| 01 | . | mode: "fingerprint" (1) 0xa5-0xa6 (1)
0x0a0| 3f 1b 3c 8a 01 98 00 d5 76 4e| ?.<.....vN| fingerprint: "3f1b3c8a019800d5764e2de6bdfd2785" (raw bits) 0xa6-0xb6 (16)
0x0b0|2d e6 bd fd 27 85 |-...'. |
| | | [9]{}: bound 0xb6-0xca (20)
0x0b0| 02 | . | timestamp_delta: 1 (2024-09-18 13:51:30) 0xb6-0xb7 (1)
0x0b0| 01 | . | id_prefix_size: 1 0xb7-0xb8 (1)
0x0b0| 14 | . | id_prefix: "14" (raw bits) 0xb8-0xb9 (1)
0x0b0| 01 | . | mode: "fingerprint" (1) 0xb9-0xba (1)
0x0b0| 7c aa f0 ac b5 df| |.....| fingerprint: "7caaf0acb5dfe249aa0f7f742402168a" (raw bits) 0xba-0xca (16)
0x0c0|e2 49 aa 0f 7f 74 24 02 16 8a |.I...t$... |
| | | [10]{}: bound 0xca-0xde (20)
0x0c0| 01 | . | timestamp_delta: 0 (2024-09-18 13:51:29) 0xca-0xcb (1)
0x0c0| 01 | . | id_prefix_size: 1 0xcb-0xcc (1)
0x0c0| 83 | . | id_prefix: "83" (raw bits) 0xcc-0xcd (1)
0x0c0| 01 | . | mode: "fingerprint" (1) 0xcd-0xce (1)
0x0c0| e7 b8| ..| fingerprint: "e7b8c4decb1eae455ca5714281e32453" (raw bits) 0xce-0xde (16)
0x0d0|c4 de cb 1e ae 45 5c a5 71 42 81 e3 24 53 |.....E\.qB..$S |
| | | [11]{}: bound 0xde-0xf2 (20)
0x0d0| 02 | . | timestamp_delta: 1 (2024-09-18 13:51:29) 0xde-0xdf (1)
0x0d0| 01| .| id_prefix_size: 1 0xdf-0xe0 (1)
0x0e0|7a |z | id_prefix: "7a" (raw bits) 0xe0-0xe1 (1)
0x0e0| 01 | . | mode: "fingerprint" (1) 0xe1-0xe2 (1)
0x0e0| 40 9c 22 63 6b 09 73 62 df 12 5d df fb 6d| @."ck.sb..]..m| fingerprint: "409c22636b097362df125ddffb6d9443" (raw bits) 0xe2-0xf2 (16)
0x0f0|94 43 |.C |
| | | [12]{}: bound 0xf2-0x106 (20)
0x0f0| 02 | . | timestamp_delta: 1 (2024-09-18 13:51:29) 0xf2-0xf3 (1)
0x0f0| 01 | . | id_prefix_size: 1 0xf3-0xf4 (1)
0x0f0| 5b | [ | id_prefix: "5b" (raw bits) 0xf4-0xf5 (1)
0x0f0| 01 | . | mode: "fingerprint" (1) 0xf5-0xf6 (1)
0x0f0| f3 32 20 8b ee 82 ac f8 ed 92| .2 .......| fingerprint: "f332208bee82acf8ed922853ee54057f" (raw bits) 0xf6-0x106 (16)
0x100|28 53 ee 54 05 7f |(S.T.. |
| | | [13]{}: bound 0x106-0x119 (19)
0x100| 02 | . | timestamp_delta: 1 (2024-09-18 13:51:29) 0x106-0x107 (1)
0x100| 00 | . | id_prefix_size: 0 0x107-0x108 (1)
0x100| 01 | . | mode: "fingerprint" (1) 0x108-0x109 (1)
0x100| fc 3e 51 fd b0 b9 29| .>Q...)| fingerprint: "fc3e51fdb0b92966e38017f795990385" (raw bits) 0x109-0x119 (16)
0x110|66 e3 80 17 f7 95 99 03 85 |f........ |
| | | [14]{}: bound 0x119-0x12d (20)
0x110| 01 | . | timestamp_delta: 0 (2024-09-18 13:51:28) 0x119-0x11a (1)
0x110| 01 | . | id_prefix_size: 1 0x11a-0x11b (1)
0x110| cc | . | id_prefix: "cc" (raw bits) 0x11b-0x11c (1)
0x110| 01 | . | mode: "fingerprint" (1) 0x11c-0x11d (1)
0x110| 42 8c e0| B..| fingerprint: "428ce0c96d49f15b50143e4fb228cb93" (raw bits) 0x11d-0x12d (16)
0x120|c9 6d 49 f1 5b 50 14 3e 4f b2 28 cb 93 |.mI.[P.>O.(.. |
| | | [15]{}: bound 0x12d-0x140 (19)
0x120| 00 | . | timestamp_delta: -1 (infinity) 0x12d-0x12e (1)
0x120| 00 | . | id_prefix_size: 0 0x12e-0x12f (1)
0x120| 01| .| mode: "fingerprint" (1) 0x12f-0x130 (1)
0x130|31 71 2d 30 e5 29 6a 7a 45 d0 7b ba 45 2d 61 cd|1q-0.)jzE.{.E-a.| fingerprint: "31712d30e5296a7a45d07bba452d61cd" (raw bits) 0x130-0x140 (16)